Commit 1c0f0db0 authored by Michael Ritter's avatar Michael Ritter
Browse files

Merge branch '42-statusservlet-should-display-correct-depositor-totals' into 'develop'

Resolve StatusServlet should display correct depositor totals

See merge request !8
parents 6e8c4d72 c94b3630
...@@ -44,8 +44,9 @@ import javax.sql.DataSource; ...@@ -44,8 +44,9 @@ import javax.sql.DataSource;
import java.sql.Connection; import java.sql.Connection;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantLock;
...@@ -62,16 +63,16 @@ public class CollectionCountContext implements ServletContextListener { ...@@ -62,16 +63,16 @@ public class CollectionCountContext implements ServletContextListener {
public static final String CTX_STARTUP = "startup_complete"; public static final String CTX_STARTUP = "startup_complete";
private static final Logger LOG = Logger.getLogger( private static final Logger LOG = Logger.getLogger(
CollectionCountContext.class); CollectionCountContext.class);
private static Map<Collection, Long> fileCountMap = new HashMap<Collection, Long>(); private static Map<Collection, Long> fileCountMap = new ConcurrentHashMap<>();
private static Map<Collection, Long> fileActiveMap = new HashMap<Collection, Long>(); private static Map<Collection, Long> fileActiveMap = new ConcurrentHashMap<>();
private static Map<Collection, Long> fileCorruptMap = new HashMap<Collection, Long>(); private static Map<Collection, Long> fileCorruptMap = new ConcurrentHashMap<>();
private static Map<Collection, Long> fileMissingMap = new HashMap<Collection, Long>(); private static Map<Collection, Long> fileMissingMap = new ConcurrentHashMap<>();
private static Map<Collection, Long> fileMissingTokenMap = new HashMap<Collection, Long>(); private static Map<Collection, Long> fileMissingTokenMap = new ConcurrentHashMap<>();
private static Map<Collection, Long> fileTokenMismatchMap = new HashMap<Collection, Long>(); private static Map<Collection, Long> fileTokenMismatchMap = new ConcurrentHashMap<>();
private static Map<Collection, Long> totalErrorMap = new HashMap<Collection, Long>(); private static Map<Collection, Long> totalErrorMap = new ConcurrentHashMap<>();
private static Map<Collection, Long> totalSizeMap = new HashMap<Collection, Long>(); private static Map<Collection, Long> totalSizeMap = new ConcurrentHashMap<>();
private static Map<Collection, Long> fileRemoteMissing = new HashMap<Collection, Long>(); private static Map<Collection, Long> fileRemoteMissing = new ConcurrentHashMap<>();
private static Map<Collection, Long> fileRemoteCorrupt = new HashMap<Collection, Long>(); private static Map<Collection, Long> fileRemoteCorrupt = new ConcurrentHashMap<>();
private static AtomicInteger totalCollections = new AtomicInteger(0); private static AtomicInteger totalCollections = new AtomicInteger(0);
private static Lock lock = new ReentrantLock(); private static Lock lock = new ReentrantLock();
private static boolean abort = false; private static boolean abort = false;
...@@ -216,8 +217,20 @@ public class CollectionCountContext implements ServletContextListener { ...@@ -216,8 +217,20 @@ public class CollectionCountContext implements ServletContextListener {
totalCollections.incrementAndGet(); totalCollections.incrementAndGet();
} }
public static void decrementTotalCollections() { public static void decrementTotalCollections(Collection collection) {
totalCollections.decrementAndGet(); totalCollections.decrementAndGet();
fileCountMap.remove(collection);
fileActiveMap.remove(collection);
fileCorruptMap.remove(collection);
fileMissingMap.remove(collection);
fileMissingTokenMap.remove(collection);
fileTokenMismatchMap.remove(collection);
totalErrorMap.remove(collection);
totalSizeMap.remove(collection);
fileRemoteMissing.remove(collection);
fileRemoteCorrupt.remove(collection);
GroupSummaryContext.updateGroup(collection.getGroup());
} }
public static int getTotalCollections() { public static int getTotalCollections() {
...@@ -226,10 +239,11 @@ public class CollectionCountContext implements ServletContextListener { ...@@ -226,10 +239,11 @@ public class CollectionCountContext implements ServletContextListener {
/** /**
* Update statistics for a collection. * Update statistics for a collection.
* *
* @param c * @param c
*/ */
private static void queryCollection(Collection c) { private static boolean queryCollection(Collection c) {
boolean update = false;
Connection connection = null; Connection connection = null;
PreparedStatement ps = null; PreparedStatement ps = null;
ResultSet rs = null; ResultSet rs = null;
...@@ -249,14 +263,13 @@ public class CollectionCountContext implements ServletContextListener { ...@@ -249,14 +263,13 @@ public class CollectionCountContext implements ServletContextListener {
while (rs.next()) { while (rs.next()) {
if (abort) { if (abort) {
return; return update;
} }
char state = rs.getString(1).charAt(0); char state = rs.getString(1).charAt(0);
long count = rs.getLong(2); long count = rs.getLong(2);
total += count; total += count;
switch (state) { switch (state) {
case 'A': case 'A':
fileActiveMap.put(c, count); fileActiveMap.put(c, count);
...@@ -288,7 +301,7 @@ public class CollectionCountContext implements ServletContextListener { ...@@ -288,7 +301,7 @@ public class CollectionCountContext implements ServletContextListener {
} }
} }
fileCountMap.put(c, total); update = !Objects.equals(fileCountMap.put(c, total), total);
totalErrorMap.put(c, totalErrors); totalErrorMap.put(c, totalErrors);
SQL.release(rs); SQL.release(rs);
SQL.release(ps); SQL.release(ps);
...@@ -303,8 +316,7 @@ public class CollectionCountContext implements ServletContextListener { ...@@ -303,8 +316,7 @@ public class CollectionCountContext implements ServletContextListener {
rs = ps.executeQuery(); rs = ps.executeQuery();
rs.next(); rs.next();
long totalSize = rs.getLong(1); long totalSize = rs.getLong(1);
totalSizeMap.put(c, totalSize); update = update | !Objects.equals(totalSizeMap.put(c, totalSize), totalSize);
} catch (Exception e) { } catch (Exception e) {
LOG.error("Error starting up, collection count", e); LOG.error("Error starting up, collection count", e);
} finally { } finally {
...@@ -312,8 +324,9 @@ public class CollectionCountContext implements ServletContextListener { ...@@ -312,8 +324,9 @@ public class CollectionCountContext implements ServletContextListener {
SQL.release(ps); SQL.release(ps);
SQL.release(connection); SQL.release(connection);
LOG.trace("Finished count on " + c.getName()); LOG.trace("Finished count on " + c.getName());
} }
return update;
} }
public static void updateCollection(final Collection c) { public static void updateCollection(final Collection c) {
...@@ -325,7 +338,10 @@ public class CollectionCountContext implements ServletContextListener { ...@@ -325,7 +338,10 @@ public class CollectionCountContext implements ServletContextListener {
lock.lock(); lock.lock();
try { try {
queryCollection(c); boolean update = queryCollection(c);
if (update) {
GroupSummaryContext.updateGroup(c.getGroup());
}
} finally { } finally {
lock.unlock(); lock.unlock();
} }
......
package edu.umiacs.ace.monitor.access;
import java.math.BigDecimal;
/**
*
* Created by shake on 4/4/17.
*/
public final class GroupSummary {
private final String group;
private final BigDecimal size;
private final BigDecimal count;
public GroupSummary(String group, BigDecimal size, BigDecimal count) {
this.group = group;
this.size = size;
this.count = count;
}
public String getGroup() {
return group;
}
public BigDecimal getSize() {
return size;
}
public BigDecimal getCount() {
return count;
}
}
package edu.umiacs.ace.monitor.access;
import com.google.common.collect.ImmutableList;
import edu.umiacs.ace.util.PersistUtil;
import org.apache.log4j.Logger;
import javax.persistence.EntityManager;
import javax.persistence.Query;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
*
* Created by shake on 4/4/17.
*/
public class GroupSummaryContext implements ServletContextListener {
private static final Logger log = Logger.getLogger(GroupSummaryContext.class);
public static Map<String, GroupSummary> summaries;
/**
* Query to get ALL group summaries
*/
private static final String SUMMARY_QUERY_ALL = "SELECT c.colgroup, sum(m.size) AS size, sum(m.count) AS count " +
"FROM collection c " +
"JOIN ( " +
"SELECT sum(size) AS size, count(id) AS count, parentcollection_id " +
"FROM monitored_item " +
"WHERE directory = 0 " +
"GROUP BY parentcollection_id " +
") AS m ON c.id = m.parentcollection_id " +
"WHERE c.colgroup IS NOT NULL " +
"GROUP BY c.colgroup";
/**
* Query to get the group summary for a single group
*/
private static final String SUMMARY_QUERY_GROUP = "select c.colgroup, sum(m.size) AS size, count(m.id) AS count " +
"FROM monitored_item m " +
"JOIN ( " +
" select colgroup, id " +
" FROM collection " +
" WHERE colgroup = ? " +
") c ON c.id = m.parentcollection_id WHERE directory = 0";
@Override
public void contextInitialized(ServletContextEvent servletContextEvent) {
summaries = new HashMap<>();
updateSummaries(ImmutableList.of(SUMMARY_QUERY_ALL),
ImmutableList.<String>of());
}
@Override
public void contextDestroyed(ServletContextEvent servletContextEvent) {
summaries.clear();
}
/**
* Public api method to update a single group
*
* @param group the group to update
*/
@SuppressWarnings("WeakerAccess")
public static void updateGroup(String group) {
if (group != null) {
log.debug("Updating group summary for " + group);
updateSummaries(
ImmutableList.of(SUMMARY_QUERY_GROUP),
ImmutableList.of(group));
}
}
/**
* Static method to allow us to update the summary of a group
*
* @param sql the sql query to build
* @param params the parameters to pass along to the query
*/
private static void updateSummaries(List<String> sql, List<String> params) {
EntityManager em = PersistUtil.getEntityManager();
StringBuilder query = new StringBuilder();
for (String s : sql) {
query.append(s);
}
Query groupSummary = em.createNativeQuery(query.toString(), "GroupSummaryMapping");
int i = 1;
for (String param : params) {
groupSummary.setParameter(i, param);
i++;
}
List<GroupSummary> results = (List<GroupSummary>)groupSummary.getResultList();
for (GroupSummary result : results) {
log.info("Result: group " + result.getGroup() + ", size: " + result.getSize() + ", count: " + result.getCount());
summaries.put(result.getGroup(), result);
}
}
}
...@@ -210,6 +210,7 @@ public class StatusServlet extends EntityManagerServlet { ...@@ -210,6 +210,7 @@ public class StatusServlet extends EntityManagerServlet {
request.setAttribute(PAGE_STATES, ImmutableList.copyOf(CStateBean.values())); request.setAttribute(PAGE_STATES, ImmutableList.copyOf(CStateBean.values()));
request.setAttribute(PAGE_COUNT, count); request.setAttribute(PAGE_COUNT, count);
request.setAttribute(PAGE_NUMBER, pb); request.setAttribute(PAGE_NUMBER, pb);
request.setAttribute("groups", GroupSummaryContext.summaries);
if (hasJson(request)) { if (hasJson(request)) {
dispatcher = request.getRequestDispatcher("status-json.jsp"); dispatcher = request.getRequestDispatcher("status-json.jsp");
} else if (hasCsv(request)) { } else if (hasCsv(request)) {
......
...@@ -75,7 +75,7 @@ import java.util.Map; ...@@ -75,7 +75,7 @@ import java.util.Map;
@NamedQuery(name = "Collection.getCollectionByName", query = @NamedQuery(name = "Collection.getCollectionByName", query =
"SELECT c FROM Collection c WHERE c.name = :name"), "SELECT c FROM Collection c WHERE c.name = :name"),
@NamedQuery(name = "Collection.listGroups", query = @NamedQuery(name = "Collection.listGroups", query =
"SELECT DISTINCT c.group FROM Collection c"), "SELECT DISTINCT c.group FROM Collection c WHERE c.group IS NOT NULL"),
@NamedQuery(name = "Collection.getCollectionById", query = @NamedQuery(name = "Collection.getCollectionById", query =
"SELECT c FROM Collection c WHERE c.id = :id"), "SELECT c FROM Collection c WHERE c.id = :id"),
@NamedQuery(name = "Collection.getCollectionsInGroup", query = @NamedQuery(name = "Collection.getCollectionsInGroup", query =
......
...@@ -96,7 +96,7 @@ public class ManageCollectionServlet extends EntityManagerServlet { ...@@ -96,7 +96,7 @@ public class ManageCollectionServlet extends EntityManagerServlet {
String paramCheckResponse = null; String paramCheckResponse = null;
/** /*
* Modification, view, or removal of an existing collection * Modification, view, or removal of an existing collection
* if we have an int, and its > 0, and it is the key for a resource * if we have an int, and its > 0, and it is the key for a resource
*/ */
...@@ -109,16 +109,16 @@ public class ManageCollectionServlet extends EntityManagerServlet { ...@@ -109,16 +109,16 @@ public class ManageCollectionServlet extends EntityManagerServlet {
} }
/** /*
* Tst to see if we're removing a collection * Tst to see if we're removing a collection
*/ */
if (!Strings.isEmpty(request.getParameter(PARAM_REMOVE)) if (!Strings.isEmpty(request.getParameter(PARAM_REMOVE))
&& request.getParameter(PARAM_REMOVE).toLowerCase().equals("yes")) { && request.getParameter(PARAM_REMOVE).toLowerCase().equals("yes")) {
LOG.debug("removing collection" + collection.getName()); LOG.debug("removing collection " + collection.getName());
removeCollection(em, collection, storage); removeCollection(em, collection, storage);
response.sendRedirect("Status?collectionid=-1"); response.sendRedirect("Status?collectionid=-1");
return; return;
} /** } /*
* otherwise, are we updating? * otherwise, are we updating?
*/ */
else if (checkParameters(request) else if (checkParameters(request)
...@@ -132,7 +132,7 @@ public class ManageCollectionServlet extends EntityManagerServlet { ...@@ -132,7 +132,7 @@ public class ManageCollectionServlet extends EntityManagerServlet {
} }
trans.commit(); trans.commit();
dispatcher = request.getRequestDispatcher("collectionfinish.jsp"); dispatcher = request.getRequestDispatcher("collectionfinish.jsp");
} /** } /*
* ok, we're just opening an existing collection for modification * ok, we're just opening an existing collection for modification
* - this should display the storage page since storage will be set * - this should display the storage page since storage will be set
*/ */
...@@ -140,7 +140,7 @@ public class ManageCollectionServlet extends EntityManagerServlet { ...@@ -140,7 +140,7 @@ public class ManageCollectionServlet extends EntityManagerServlet {
LOG.debug("loading existing collection: " + collection.getName()); LOG.debug("loading existing collection: " + collection.getName());
dispatcher = request.getRequestDispatcher("collectionmodify.jsp"); dispatcher = request.getRequestDispatcher("collectionmodify.jsp");
} }
} /** } /*
* its either a new submission, or requesting a blank page. * its either a new submission, or requesting a blank page.
* We shouldn't set storage information here yet, just create the * We shouldn't set storage information here yet, just create the
* new collection and create a blank storage * new collection and create a blank storage
...@@ -209,7 +209,7 @@ public class ManageCollectionServlet extends EntityManagerServlet { ...@@ -209,7 +209,7 @@ public class ManageCollectionServlet extends EntityManagerServlet {
storage.remove(em); storage.remove(em);
} }
em.remove(collection); em.remove(collection);
CollectionCountContext.decrementTotalCollections(); CollectionCountContext.decrementTotalCollections(collection);
trans.commit(); trans.commit();
} }
......
...@@ -14,6 +14,13 @@ ...@@ -14,6 +14,13 @@
<column name="size"/> <column name="size"/>
</constructor-result> </constructor-result>
</sql-result-set-mapping> </sql-result-set-mapping>
<sql-result-set-mapping name="GroupSummaryMapping">
<constructor-result target-class="edu.umiacs.ace.monitor.access.GroupSummary">
<column name="colgroup"/>
<column name="size"/>
<column name="count"/>
</constructor-result>
</sql-result-set-mapping>
<converter class="edu.umiacs.ace.monitor.core.CollectionStateConverter"/> <converter class="edu.umiacs.ace.monitor.core.CollectionStateConverter"/>
......
...@@ -65,6 +65,9 @@ ...@@ -65,6 +65,9 @@
<listener> <listener>
<listener-class>edu.umiacs.ace.monitor.access.CollectionCountContext</listener-class> <listener-class>edu.umiacs.ace.monitor.access.CollectionCountContext</listener-class>
</listener> </listener>
<listener>
<listener-class>edu.umiacs.ace.monitor.access.GroupSummaryContext</listener-class>
</listener>
<listener> <listener>
<description>HttpSessionListener</description> <description>HttpSessionListener</description>
<listener-class>edu.umiacs.ace.monitor.users.UserSessionTracker</listener-class> <listener-class>edu.umiacs.ace.monitor.users.UserSessionTracker</listener-class>
......
...@@ -186,25 +186,22 @@ ...@@ -186,25 +186,22 @@
</thead> </thead>
<c:set var="count" value="0" /> <c:set var="count" value="0" />
<jsp:useBean id="today" class="java.util.Date"/> <jsp:useBean id="today" class="java.util.Date"/>
<c:set var="counttotal" value="0" />
<c:set var="sizetotal" value="0" />
<c:forEach var="item" items="${collections}"> <c:forEach var="item" items="${collections}">
<c:if test="${currgroup != item.collection.group && item.collection.group != null}"> <c:if test="${currgroup != item.collection.group && item.collection.group != null}">
<script type="text/javascript"> <c:set var="group" value="${item.collection.group}"/>
myp=document.getElementById("group${currgroup}"); <c:set var="group_count" value="${groups[group].count}"/>
if (myp != null) <c:set var="size" value="${groups[group].size}"/>
{
myp.innerHTML='${counttotal} / <c:choose><c:when test="${sizetotal > 0}"><d:FileSize value="${sizetotal}" /></c:when><c:otherwise>0 B</c:otherwise></c:choose>';
}
</script>
<tr> <tr>
<td class="groupheader" colspan="3" onclick="toggleVisibility('spexpand${item.collection.group}','inline'); toggleVisibility('sphide${item.collection.group}','inline');"> <td class="groupheader" colspan="3" onclick="toggleVisibility('spexpand${group}','inline'); toggleVisibility('sphide${group}','inline');">
<span onclick="showGroup('grouptr${item.collection.group}')" id="spexpand${item.collection.group}" style="display:none;float: left;width: 25px;" >[+]</span> <span onclick="showGroup('grouptr${group}')" id="spexpand${group}" style="display:none;float: left;width: 25px;" >[+]</span>
<span onclick="hideGroup('grouptr${item.collection.group}')" id="sphide${item.collection.group}" style="display:inline;float: left;width: 25px;" >[-]</span> <span onclick="hideGroup('grouptr${group}')" id="sphide${group}" style="display:inline;float: left;width: 25px;" >[-]</span>
${item.collection.group} ${group}
</td>
<td class="groupheader" colspan="3" id="group${group}">
${group_count} /<c:choose><c:when test="${size > 0}"><d:FileSize value="${size}" /></c:when><c:otherwise>0 B</c:otherwise></c:choose>
</td> </td>
<td class="groupheader" colspan="3" id="group${item.collection.group}"></td>
</tr> </tr>
<c:set var="counttotal" value="0" /> <c:set var="counttotal" value="0" />
<c:set var="sizetotal" value="0" /> <c:set var="sizetotal" value="0" />
...@@ -271,13 +268,7 @@ ...@@ -271,13 +268,7 @@
</tr> </tr>
<c:set var="count" value="${count + 1}" /> <c:set var="count" value="${count + 1}" />
<c:set var="currgroup" value="${item.collection.group}" /> <c:set var="currgroup" value="${item.collection.group}" />
<c:set var="counttotal" value="${counttotal + item.totalFiles}" />
<c:set var="sizetotal" value="${sizetotal + item.totalSize}" />
</c:forEach> </c:forEach>
<script type="text/javascript">
myp=document.getElementById("group${currgroup}");
myp.innerHTML='${counttotal} / <c:choose><c:when test="${sizetotal > 0}"><d:FileSize value="${sizetotal}" /></c:when><c:otherwise>0 B</c:otherwise></c:choose>';
</script>
<tr><td colspan="5"><br/><d:Auth role="Collection Modify" showUnauthenticated="true"><a href="ManageCollection">Add Collection</a></d:Auth> &nbsp;&nbsp;&nbsp&nbsp;&nbsp; <tr><td colspan="5"><br/><d:Auth role="Collection Modify" showUnauthenticated="true"><a href="ManageCollection">Add Collection</a></d:Auth> &nbsp;&nbsp;&nbsp&nbsp;&nbsp;
<d:Auth role="Audit"> <d:Auth role="Audit">
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment