Merge branch '42-statusservlet-should-display-correct-depositor-totals' into 'develop'

Resolve StatusServlet should display correct depositor totals

See merge request !8

ace-am/src/main/java/edu/umiacs/ace/monitor/access/CollectionCountContext.java

@@ -44,8 +44,9 @@ import javax.sql.DataSource;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
-import java.util.HashMap;
 import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
@@ -62,16 +63,16 @@ public class CollectionCountContext implements ServletContextListener {
     public static final String CTX_STARTUP = "startup_complete";
     private static final Logger LOG = Logger.getLogger(
             CollectionCountContext.class);
-    private static Map<Collection, Long> fileCountMap = new HashMap<Collection, Long>();
-    private static Map<Collection, Long> fileActiveMap = new HashMap<Collection, Long>();
-    private static Map<Collection, Long> fileCorruptMap = new HashMap<Collection, Long>();
-    private static Map<Collection, Long> fileMissingMap = new HashMap<Collection, Long>();
-    private static Map<Collection, Long> fileMissingTokenMap = new HashMap<Collection, Long>();
-    private static Map<Collection, Long> fileTokenMismatchMap = new HashMap<Collection, Long>();
-    private static Map<Collection, Long> totalErrorMap = new HashMap<Collection, Long>();
-    private static Map<Collection, Long> totalSizeMap = new HashMap<Collection, Long>();
-    private static Map<Collection, Long> fileRemoteMissing = new HashMap<Collection, Long>();
-    private static Map<Collection, Long> fileRemoteCorrupt = new HashMap<Collection, Long>();
+    private static Map<Collection, Long> fileCountMap = new ConcurrentHashMap<>();
+    private static Map<Collection, Long> fileActiveMap = new ConcurrentHashMap<>();
+    private static Map<Collection, Long> fileCorruptMap = new ConcurrentHashMap<>();
+    private static Map<Collection, Long> fileMissingMap = new ConcurrentHashMap<>();
+    private static Map<Collection, Long> fileMissingTokenMap = new ConcurrentHashMap<>();
+    private static Map<Collection, Long> fileTokenMismatchMap = new ConcurrentHashMap<>();
+    private static Map<Collection, Long> totalErrorMap = new ConcurrentHashMap<>();
+    private static Map<Collection, Long> totalSizeMap = new ConcurrentHashMap<>();
+    private static Map<Collection, Long> fileRemoteMissing = new ConcurrentHashMap<>();
+    private static Map<Collection, Long> fileRemoteCorrupt = new ConcurrentHashMap<>();
     private static AtomicInteger totalCollections = new AtomicInteger(0);
     private static Lock lock = new ReentrantLock();
     private static boolean abort = false;
@@ -216,8 +217,20 @@ public class CollectionCountContext implements ServletContextListener {
         totalCollections.incrementAndGet();
     }
 
-    public static void decrementTotalCollections() {
+    public static void decrementTotalCollections(Collection collection) {
         totalCollections.decrementAndGet();
+
+        fileCountMap.remove(collection);
+        fileActiveMap.remove(collection);
+        fileCorruptMap.remove(collection);
+        fileMissingMap.remove(collection);
+        fileMissingTokenMap.remove(collection);
+        fileTokenMismatchMap.remove(collection);
+        totalErrorMap.remove(collection);
+        totalSizeMap.remove(collection);
+        fileRemoteMissing.remove(collection);
+        fileRemoteCorrupt.remove(collection);
+        GroupSummaryContext.updateGroup(collection.getGroup());
     }
 
     public static int getTotalCollections() {
@@ -226,10 +239,11 @@ public class CollectionCountContext implements ServletContextListener {
 
     /**
      * Update statistics for a collection.
-     * 
+     *
      * @param c
      */
-    private static void queryCollection(Collection c) {
+    private static boolean queryCollection(Collection c) {
+        boolean update = false;
         Connection connection = null;
         PreparedStatement ps = null;
         ResultSet rs = null;
@@ -249,14 +263,13 @@ public class CollectionCountContext implements ServletContextListener {
 
             while (rs.next()) {
                 if (abort) {
-                    return;
+                    return update;
                 }
                 char state = rs.getString(1).charAt(0);
                 long count = rs.getLong(2);
 
                 total += count;
 
-
                 switch (state) {
                     case 'A':
                         fileActiveMap.put(c, count);
@@ -288,7 +301,7 @@ public class CollectionCountContext implements ServletContextListener {
                 }
             }
 
-            fileCountMap.put(c, total);
+            update = !Objects.equals(fileCountMap.put(c, total), total);
             totalErrorMap.put(c, totalErrors);
             SQL.release(rs);
             SQL.release(ps);
@@ -303,8 +316,7 @@ public class CollectionCountContext implements ServletContextListener {
             rs = ps.executeQuery();
             rs.next();
             long totalSize = rs.getLong(1);
-            totalSizeMap.put(c, totalSize);
-
+            update = update | !Objects.equals(totalSizeMap.put(c, totalSize), totalSize);
         } catch (Exception e) {
             LOG.error("Error starting up, collection count", e);
         } finally {
@@ -312,8 +324,9 @@ public class CollectionCountContext implements ServletContextListener {
             SQL.release(ps);
             SQL.release(connection);
             LOG.trace("Finished count on " + c.getName());
-
         }
+
+        return update;
     }
 
     public static void updateCollection(final Collection c) {
@@ -325,7 +338,10 @@ public class CollectionCountContext implements ServletContextListener {
                 lock.lock();
 
                 try {
-                    queryCollection(c);
+                    boolean update = queryCollection(c);
+                    if (update) {
+                        GroupSummaryContext.updateGroup(c.getGroup());
+                    }
                 } finally {
                     lock.unlock();
                 }

ace-am/src/main/java/edu/umiacs/ace/monitor/access/GroupSummary.java

+package edu.umiacs.ace.monitor.access;
+
+import java.math.BigDecimal;
+
+/**
+ *
+ * Created by shake on 4/4/17.
+ */
+public final class GroupSummary {
+
+    private final String group;
+    private final BigDecimal size;
+    private final BigDecimal count;
+
+    public GroupSummary(String group, BigDecimal size, BigDecimal count) {
+        this.group = group;
+        this.size = size;
+        this.count = count;
+    }
+
+    public String getGroup() {
+        return group;
+    }
+
+    public BigDecimal getSize() {
+        return size;
+    }
+
+    public BigDecimal getCount() {
+        return count;
+    }
+}

ace-am/src/main/java/edu/umiacs/ace/monitor/access/GroupSummaryContext.java

+package edu.umiacs.ace.monitor.access;
+
+import com.google.common.collect.ImmutableList;
+import edu.umiacs.ace.util.PersistUtil;
+import org.apache.log4j.Logger;
+
+import javax.persistence.EntityManager;
+import javax.persistence.Query;
+import javax.servlet.ServletContextEvent;
+import javax.servlet.ServletContextListener;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ *
+ * Created by shake on 4/4/17.
+ */
+public class GroupSummaryContext implements ServletContextListener {
+    private static final Logger log = Logger.getLogger(GroupSummaryContext.class);
+
+    public static Map<String, GroupSummary> summaries;
+
+    /**
+     * Query to get ALL group summaries
+     */
+    private static final String SUMMARY_QUERY_ALL = "SELECT c.colgroup, sum(m.size) AS size, sum(m.count) AS count " +
+            "FROM collection c " +
+            "JOIN (  " +
+            "SELECT sum(size) AS size, count(id) AS count, parentcollection_id   " +
+            "FROM monitored_item   " +
+            "WHERE directory = 0   " +
+            "GROUP BY parentcollection_id " +
+            ") AS m ON c.id = m.parentcollection_id " +
+            "WHERE c.colgroup IS NOT NULL " +
+            "GROUP BY c.colgroup";
+
+    /**
+     * Query to get the group summary for a single group
+     */
+    private static final String SUMMARY_QUERY_GROUP = "select c.colgroup, sum(m.size) AS size, count(m.id) AS count " +
+            "FROM monitored_item m " +
+            "JOIN ( " +
+            "  select colgroup, id " +
+            "  FROM collection " +
+            "  WHERE colgroup = ? " +
+            ") c ON c.id = m.parentcollection_id WHERE directory = 0";
+
+    @Override
+    public void contextInitialized(ServletContextEvent servletContextEvent) {
+        summaries = new HashMap<>();
+        updateSummaries(ImmutableList.of(SUMMARY_QUERY_ALL),
+                ImmutableList.<String>of());
+    }
+
+    @Override
+    public void contextDestroyed(ServletContextEvent servletContextEvent) {
+        summaries.clear();
+    }
+
+    /**
+     * Public api method to update a single group
+     *
+     * @param group the group to update
+     */
+    @SuppressWarnings("WeakerAccess")
+    public static void updateGroup(String group) {
+        if (group != null) {
+            log.debug("Updating group summary for " + group);
+            updateSummaries(
+                    ImmutableList.of(SUMMARY_QUERY_GROUP),
+                    ImmutableList.of(group));
+        }
+    }
+
+    /**
+     * Static method to allow us to update the summary of a group
+     *
+     * @param sql the sql query to build
+     * @param params the parameters to pass along to the query
+     */
+    private static void updateSummaries(List<String> sql, List<String> params) {
+        EntityManager em = PersistUtil.getEntityManager();
+
+        StringBuilder query = new StringBuilder();
+        for (String s : sql) {
+            query.append(s);
+        }
+
+        Query groupSummary = em.createNativeQuery(query.toString(), "GroupSummaryMapping");
+        int i = 1;
+        for (String param : params) {
+            groupSummary.setParameter(i, param);
+            i++;
+        }
+
+        List<GroupSummary> results = (List<GroupSummary>)groupSummary.getResultList();
+        for (GroupSummary result : results) {
+            log.info("Result: group " + result.getGroup() + ", size: " + result.getSize() + ", count: " + result.getCount());
+            summaries.put(result.getGroup(), result);
+        }
+    }
+}

ace-am/src/main/java/edu/umiacs/ace/monitor/access/StatusServlet.java

@@ -210,6 +210,7 @@ public class StatusServlet extends EntityManagerServlet {
         request.setAttribute(PAGE_STATES, ImmutableList.copyOf(CStateBean.values()));
         request.setAttribute(PAGE_COUNT, count);
         request.setAttribute(PAGE_NUMBER, pb);
+        request.setAttribute("groups", GroupSummaryContext.summaries);
         if (hasJson(request)) {
             dispatcher = request.getRequestDispatcher("status-json.jsp");
         } else if (hasCsv(request)) {

ace-am/src/main/java/edu/umiacs/ace/monitor/core/Collection.java

@@ -75,7 +75,7 @@ import java.util.Map;
     @NamedQuery(name = "Collection.getCollectionByName", query =
     "SELECT c FROM Collection c WHERE c.name = :name"),
     @NamedQuery(name = "Collection.listGroups", query =
-    "SELECT DISTINCT c.group FROM Collection c"),
+    "SELECT DISTINCT c.group FROM Collection c WHERE c.group IS NOT NULL"),
     @NamedQuery(name = "Collection.getCollectionById", query =
     "SELECT c FROM Collection c WHERE c.id = :id"),
     @NamedQuery(name = "Collection.getCollectionsInGroup", query =

ace-am/src/main/java/edu/umiacs/ace/monitor/core/ManageCollectionServlet.java

@@ -96,7 +96,7 @@ public class ManageCollectionServlet extends EntityManagerServlet {
         String paramCheckResponse = null;
 
 
-        /**
+        /*
          * Modification, view, or removal of an existing collection       
          *  if we have an int, and its > 0, and it is the key for a resource
          */
@@ -109,16 +109,16 @@ public class ManageCollectionServlet extends EntityManagerServlet {
             }
 
 
-            /**
+            /*
              * Tst to see if we're removing a collection
              */
             if (!Strings.isEmpty(request.getParameter(PARAM_REMOVE))
                     && request.getParameter(PARAM_REMOVE).toLowerCase().equals("yes")) {
-                LOG.debug("removing collection" + collection.getName());
+                LOG.debug("removing collection " + collection.getName());
                 removeCollection(em, collection, storage);
                 response.sendRedirect("Status?collectionid=-1");
                 return;
-            } /**
+            } /*
              * otherwise, are we updating?
              */
             else if (checkParameters(request)
@@ -132,7 +132,7 @@ public class ManageCollectionServlet extends EntityManagerServlet {
                 }
                 trans.commit();
                 dispatcher = request.getRequestDispatcher("collectionfinish.jsp");
-            } /**
+            } /*
              * ok, we're just opening an existing collection for modification
              *  - this should display the storage page since storage will be set
              */
@@ -140,7 +140,7 @@ public class ManageCollectionServlet extends EntityManagerServlet {
                 LOG.debug("loading existing collection: " + collection.getName());
                 dispatcher = request.getRequestDispatcher("collectionmodify.jsp");
             }
-        } /**
+        } /*
          * its either a new submission, or requesting a blank page.
          *  We shouldn't set storage information here yet, just create the 
          *  new collection and create a blank storage
@@ -209,7 +209,7 @@ public class ManageCollectionServlet extends EntityManagerServlet {
             storage.remove(em);
         }
         em.remove(collection);
-        CollectionCountContext.decrementTotalCollections();
+        CollectionCountContext.decrementTotalCollections(collection);
         trans.commit();
     }
 

ace-am/src/main/resources/META-INF/orm.xml

@@ -14,6 +14,13 @@
             <column name="size"/>
         </constructor-result>
     </sql-result-set-mapping>
+    <sql-result-set-mapping name="GroupSummaryMapping">
+        <constructor-result target-class="edu.umiacs.ace.monitor.access.GroupSummary">
+            <column name="colgroup"/>
+            <column name="size"/>
+            <column name="count"/>
+        </constructor-result>
+    </sql-result-set-mapping>
 
     <converter class="edu.umiacs.ace.monitor.core.CollectionStateConverter"/>
 

ace-am/src/main/webapp/WEB-INF/web.xml

@@ -65,6 +65,9 @@
     <listener>
         <listener-class>edu.umiacs.ace.monitor.access.CollectionCountContext</listener-class>
     </listener>
+    <listener>
+        <listener-class>edu.umiacs.ace.monitor.access.GroupSummaryContext</listener-class>
+    </listener>
     <listener>
         <description>HttpSessionListener</description>
         <listener-class>edu.umiacs.ace.monitor.users.UserSessionTracker</listener-class>

ace-am/src/main/webapp/status.jsp

@@ -186,25 +186,22 @@
             </thead>
             <c:set var="count" value="0" />
             <jsp:useBean id="today" class="java.util.Date"/>
-            <c:set var="counttotal" value="0" />
-            <c:set var="sizetotal" value="0" />
             <c:forEach var="item" items="${collections}">
 
                 <c:if test="${currgroup != item.collection.group && item.collection.group != null}">
-                    <script type="text/javascript">
-                        myp=document.getElementById("group${currgroup}");
-                        if (myp != null)
-                        {
-                            myp.innerHTML='${counttotal} / <c:choose><c:when test="${sizetotal > 0}"><d:FileSize value="${sizetotal}" /></c:when><c:otherwise>0 B</c:otherwise></c:choose>';
-                        }
-                    </script>
+                    <c:set var="group" value="${item.collection.group}"/>
+                    <c:set var="group_count" value="${groups[group].count}"/>
+                    <c:set var="size" value="${groups[group].size}"/>
+
                     <tr>
-                        <td class="groupheader" colspan="3" onclick="toggleVisibility('spexpand${item.collection.group}','inline'); toggleVisibility('sphide${item.collection.group}','inline');">
-                            <span onclick="showGroup('grouptr${item.collection.group}')" id="spexpand${item.collection.group}" style="display:none;float: left;width: 25px;" >[+]</span>
-                            <span onclick="hideGroup('grouptr${item.collection.group}')" id="sphide${item.collection.group}" style="display:inline;float: left;width: 25px;" >[-]</span>
-                            ${item.collection.group}
+                        <td class="groupheader" colspan="3" onclick="toggleVisibility('spexpand${group}','inline'); toggleVisibility('sphide${group}','inline');">
+                            <span onclick="showGroup('grouptr${group}')" id="spexpand${group}" style="display:none;float: left;width: 25px;" >[+]</span>
+                            <span onclick="hideGroup('grouptr${group}')" id="sphide${group}" style="display:inline;float: left;width: 25px;" >[-]</span>
+                            ${group}
+                        </td>
+                        <td class="groupheader" colspan="3" id="group${group}">
+                                ${group_count} /<c:choose><c:when test="${size > 0}"><d:FileSize value="${size}" /></c:when><c:otherwise>0 B</c:otherwise></c:choose>
                         </td>
-                        <td class="groupheader" colspan="3" id="group${item.collection.group}"></td>
                     </tr>
                     <c:set var="counttotal" value="0" />
                     <c:set var="sizetotal" value="0" />
@@ -271,13 +268,7 @@
                 </tr>
                 <c:set var="count" value="${count + 1}" />
                 <c:set var="currgroup" value="${item.collection.group}" />
-                <c:set var="counttotal" value="${counttotal + item.totalFiles}" />
-                <c:set var="sizetotal" value="${sizetotal + item.totalSize}" />
             </c:forEach>
-            <script type="text/javascript">
-                myp=document.getElementById("group${currgroup}");
-                myp.innerHTML='${counttotal} / <c:choose><c:when test="${sizetotal > 0}"><d:FileSize value="${sizetotal}" /></c:when><c:otherwise>0 B</c:otherwise></c:choose>';
-            </script>
 
             <tr><td colspan="5"><br/><d:Auth role="Collection Modify" showUnauthenticated="true"><a href="ManageCollection">Add Collection</a></d:Auth> &nbsp;&nbsp;&nbsp&nbsp;&nbsp;
                     <d:Auth role="Audit">