Commit b2f5ecb3 authored by Michael Ritter's avatar Michael Ritter
Browse files

Merge branch 'release-1.10'

parents 06cc89a4 c66b09c5
...@@ -4,7 +4,7 @@ ...@@ -4,7 +4,7 @@
<parent> <parent>
<artifactId>ace</artifactId> <artifactId>ace</artifactId>
<groupId>edu.umiacs.ace</groupId> <groupId>edu.umiacs.ace</groupId>
<version>1.10-SNAPSHOT</version> <version>1.10</version>
<relativePath>../pom.xml</relativePath> <relativePath>../pom.xml</relativePath>
</parent> </parent>
<groupId>edu.umiacs.ace</groupId> <groupId>edu.umiacs.ace</groupId>
...@@ -252,7 +252,7 @@ ...@@ -252,7 +252,7 @@
<dependency> <dependency>
<groupId>edu.umiacs.ace</groupId> <groupId>edu.umiacs.ace</groupId>
<artifactId>ace-ims-ws</artifactId> <artifactId>ace-ims-ws</artifactId>
<version>1.10-SNAPSHOT</version> <version>1.10</version>
<type>jar</type> <type>jar</type>
</dependency> </dependency>
<dependency> <dependency>
......
...@@ -30,23 +30,25 @@ ...@@ -30,23 +30,25 @@
// $Id$ // $Id$
package edu.umiacs.ace.monitor.access; package edu.umiacs.ace.monitor.access;
import edu.umiacs.ace.util.PersistUtil;
import edu.umiacs.ace.monitor.core.Collection; import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.util.PersistUtil;
import edu.umiacs.sql.SQL; import edu.umiacs.sql.SQL;
import org.apache.log4j.Logger;
import org.apache.log4j.NDC;
import javax.persistence.EntityManager;
import javax.persistence.Query;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import javax.sql.DataSource;
import java.sql.Connection; import java.sql.Connection;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantLock;
import javax.persistence.EntityManager;
import javax.persistence.Query;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import javax.sql.DataSource;
import org.apache.log4j.Logger;
import org.apache.log4j.NDC;
/** /**
* Class to gather a count of all collections at startup. * Class to gather a count of all collections at startup.
...@@ -70,6 +72,7 @@ public class CollectionCountContext implements ServletContextListener { ...@@ -70,6 +72,7 @@ public class CollectionCountContext implements ServletContextListener {
private static Map<Collection, Long> totalSizeMap = new HashMap<Collection, Long>(); private static Map<Collection, Long> totalSizeMap = new HashMap<Collection, Long>();
private static Map<Collection, Long> fileRemoteMissing = new HashMap<Collection, Long>(); private static Map<Collection, Long> fileRemoteMissing = new HashMap<Collection, Long>();
private static Map<Collection, Long> fileRemoteCorrupt = new HashMap<Collection, Long>(); private static Map<Collection, Long> fileRemoteCorrupt = new HashMap<Collection, Long>();
private static AtomicInteger totalCollections = new AtomicInteger(0);
private static Lock lock = new ReentrantLock(); private static Lock lock = new ReentrantLock();
private static boolean abort = false; private static boolean abort = false;
...@@ -99,6 +102,7 @@ public class CollectionCountContext implements ServletContextListener { ...@@ -99,6 +102,7 @@ public class CollectionCountContext implements ServletContextListener {
return; return;
} }
queryCollection((Collection) o); queryCollection((Collection) o);
incrementTotalCollections();
} }
em.close(); em.close();
} catch (Exception e) { } catch (Exception e) {
...@@ -208,6 +212,18 @@ public class CollectionCountContext implements ServletContextListener { ...@@ -208,6 +212,18 @@ public class CollectionCountContext implements ServletContextListener {
return -1; return -1;
} }
public static void incrementTotalCollections() {
totalCollections.incrementAndGet();
}
public static void decrementTotalCollections() {
totalCollections.decrementAndGet();
}
public static int getTotalCollections() {
return totalCollections.get();
}
/** /**
* Update statistics for a collection. * Update statistics for a collection.
* *
......
...@@ -30,55 +30,214 @@ ...@@ -30,55 +30,214 @@
// $Id$ // $Id$
package edu.umiacs.ace.monitor.access; package edu.umiacs.ace.monitor.access;
import edu.umiacs.ace.util.EntityManagerServlet; import com.google.common.collect.ImmutableList;
import edu.umiacs.ace.monitor.core.Collection; import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.support.PageBean;
import edu.umiacs.ace.monitor.support.CStateBean;
import edu.umiacs.ace.util.EntityManagerServlet;
import edu.umiacs.util.Strings; import edu.umiacs.util.Strings;
import java.io.IOException; import org.apache.log4j.Logger;
import java.util.ArrayList;
import java.util.List;
import javax.persistence.EntityManager; import javax.persistence.EntityManager;
import javax.persistence.Query; import javax.persistence.Query;
import javax.servlet.RequestDispatcher; import javax.servlet.RequestDispatcher;
import javax.servlet.ServletException; import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
/** /**
*
* @author toaster * @author toaster
*/ */
public class StatusServlet extends EntityManagerServlet { public class StatusServlet extends EntityManagerServlet {
private static final Logger LOG = Logger.getLogger(StatusServlet.class);
private static final String PAGE_COLLECTIONS = "collections"; private static final String PAGE_COLLECTIONS = "collections";
private static final String PAGE_STATES = "states";
private static final String PAGE_COUNT = "count";
private static final String PAGE_NUMBER = "page";
private static final long DEFAULT_PAGE = 0;
private static final int DEFAULT_COUNT = 100;
// TODO: Add session fields for pagination and search params
private static final String SESSION_WORKINGCOLLECTION = "workingCollection"; private static final String SESSION_WORKINGCOLLECTION = "workingCollection";
private static final String PARAM_CSV = "csv"; private static final String PARAM_CSV = "csv";
// Pagination stuff
private static final String PARAM_COUNT = "count";
private static final String PARAM_PAGE = "page";
// Search Params
private static final String PARAM_GROUP = "group";
private static final String PARAM_STATE = "state";
private static final String PARAM_COLLECTION_LIKE = "collection";
private static final String PARAM_AUDIT_DATE = "audit";
// Filter params?
// ...group
// ...name (maybe some regex?)
/** /**
* Processes requests for both HTTP <code>GET</code> and <code>POST</code> methods. * Processes requests for both HTTP <code>GET</code> and <code>POST</code> methods.
*
* @param request servlet request * @param request servlet request
* @param response servlet response * @param response servlet response
*/ */
@Override @Override
protected void processRequest( HttpServletRequest request, protected void processRequest(HttpServletRequest request,
HttpServletResponse response, EntityManager em ) HttpServletResponse response, EntityManager em)
throws ServletException, IOException { throws ServletException, IOException {
RequestDispatcher dispatcher; RequestDispatcher dispatcher;
List<CollectionSummaryBean> collections; List<CollectionSummaryBean> collections;
List<Collection> items; List<Collection> items;
LOG.info("Processing /Status request");
LOG.info("Processing /Status additional info");
long page = getParameter(request, PARAM_PAGE, DEFAULT_PAGE);
int count = (int) getParameter(request, PARAM_COUNT, DEFAULT_COUNT);
String group = getParameter(request, PARAM_GROUP, null);
String collection = getParameter(request, PARAM_COLLECTION_LIKE, null);
String state = getParameter(request, PARAM_STATE, null);
// String date = getParameter(request, PARAM_GROUP, null);
PageBean pb = new PageBean((int) page, count, "Status");
long offset = page * count;
// our main query
StringBuilder queryString = new StringBuilder();
// for getting a count of items
StringBuilder countString = new StringBuilder();
// our parameters
StringBuilder params = new StringBuilder();
List<String> queries = new ArrayList<>();
if (!Strings.isEmpty(group)) {
queries.add("c.group LIKE :group");
pb.addParam(PARAM_GROUP, group);
}
if (!Strings.isEmpty(collection)) {
queries.add("c.name LIKE :collection");
pb.addParam(PARAM_COLLECTION_LIKE, collection);
}
// Enforce that the state is not empty, or larger than 1 character
if (!Strings.isEmpty(state) && state.length() == 1) {
queries.add("c.state = :state");
pb.addParam(PARAM_STATE, state);
}
queryString.append("SELECT c FROM Collection c");
countString.append("SELECT COUNT(c.id) FROM Collection c");
if (queries.size() > 0) {
params.append(" WHERE");
}
Iterator<String> it = queries.iterator();
while (it.hasNext()) {
String query = it.next();
params.append(" ")
.append(query);
if (it.hasNext()) {
params.append(" AND");
}
}
queryString.append(params);
// allows us to keep a consistent order when displaying collections
queryString.append(" ORDER BY c.group ASC, c.name ASC");
countString.append(params);
Query query = Query query =
em.createNamedQuery("Collection.listAllCollections"); em.createQuery(queryString.toString());
// em.createNamedQuery("Collection.listAllCollections");
query.setFirstResult((int) offset);
query.setMaxResults(count);
Query countQuery = em.createQuery(countString.toString());
if (!Strings.isEmpty(group)) {
query.setParameter(PARAM_GROUP, "%" + group + "%");
countQuery.setParameter(PARAM_GROUP, "%" + group + "%");
}
if (!Strings.isEmpty(collection)) {
query.setParameter(PARAM_COLLECTION_LIKE, "%" + collection + "%");
countQuery.setParameter(PARAM_COLLECTION_LIKE, "%" + collection + "%");
}
if (!Strings.isEmpty(state) && state.length() == 1) {
query.setParameter(PARAM_STATE, state.charAt(0));
countQuery.setParameter(PARAM_STATE, state.charAt(0));
}
items = query.getResultList(); items = query.getResultList();
collections = new ArrayList<CollectionSummaryBean>(); // We only need to execute this query if we have parameters
// and need to update the total count of collections
if (queries.size() > 0) {
long totalResults = (long) countQuery.getSingleResult();
LOG.info("Total results from query: " + totalResults);
pb.update(totalResults);
}
collections = new ArrayList<>();
long collectionId;
String idParam = request.getParameter(PARAM_COLLECTION_ID);
collectionId = Strings.isValidLong(idParam) ? Long.parseLong(idParam) : -1;
CollectionSummaryBean workingCollectionBean = (CollectionSummaryBean) request.getSession().getAttribute(SESSION_WORKINGCOLLECTION);
if ( Strings.isValidLong(request.getParameter(PARAM_COLLECTION_ID)) && -1 == Long.parseLong(request.getParameter( if (Strings.isValidLong(idParam) && -1 == collectionId) {
PARAM_COLLECTION_ID)) ) { // clear the working collection
request.getSession().removeAttribute(SESSION_WORKINGCOLLECTION); request.getSession().removeAttribute(SESSION_WORKINGCOLLECTION);
} else if (Strings.isValidLong(idParam) // valid param
&& (workingCollectionBean == null // no working collection
|| !workingCollectionBean.getCollection().getId().equals(collectionId))) { // or one which does not equal our requested collection
// Get the requested collection from the db and set it as our working collection
Collection workingCollection = em.find(Collection.class, collectionId);
if (workingCollection != null) {
workingCollectionBean = createCollectionSummary(workingCollection);
request.getSession().setAttribute(SESSION_WORKINGCOLLECTION, workingCollectionBean);
}
} else {
// Continue using the current working collection
request.getSession().setAttribute(SESSION_WORKINGCOLLECTION, workingCollectionBean);
}
for (Collection col : items) {
CollectionSummaryBean csb = createCollectionSummary(col);
collections.add(csb);
} }
for ( Collection col : items ) {
request.setAttribute(PAGE_COLLECTIONS, collections);
request.setAttribute(PAGE_STATES, ImmutableList.copyOf(CStateBean.values()));
request.setAttribute(PAGE_COUNT, count);
request.setAttribute(PAGE_NUMBER, pb);
if (hasJson(request)) {
dispatcher = request.getRequestDispatcher("status-json.jsp");
} else if (hasCsv(request)) {
dispatcher = request.getRequestDispatcher("status-csv.jsp");
} else {
dispatcher = request.getRequestDispatcher("status.jsp");
}
dispatcher.forward(request, response);
}
private CollectionSummaryBean createCollectionSummary(Collection col) {
CollectionSummaryBean csb = new CollectionSummaryBean(); CollectionSummaryBean csb = new CollectionSummaryBean();
csb.setCollection(col); csb.setCollection(col);
csb.setTotalFiles(CollectionCountContext.getFileCount(col)); csb.setTotalFiles(CollectionCountContext.getFileCount(col));
...@@ -92,34 +251,10 @@ public class StatusServlet extends EntityManagerServlet { ...@@ -92,34 +251,10 @@ public class StatusServlet extends EntityManagerServlet {
csb.setTotalErrors(CollectionCountContext.getTotalErrors(col)); csb.setTotalErrors(CollectionCountContext.getTotalErrors(col));
csb.setRemoteMissing(CollectionCountContext.getRemoteMissing(col)); csb.setRemoteMissing(CollectionCountContext.getRemoteMissing(col));
csb.setRemoteCorrupt(CollectionCountContext.getRemoteCorrupt(col)); csb.setRemoteCorrupt(CollectionCountContext.getRemoteCorrupt(col));
collections.add(csb); return csb;
// if param_collection was supplied as a parameter see if this bean
// should be set as working/details bean
if ( Strings.isValidLong(request.getParameter(PARAM_COLLECTION_ID)) && col.getId() == Long.parseLong(request.getParameter(
PARAM_COLLECTION_ID)) ) {
request.getSession().setAttribute(SESSION_WORKINGCOLLECTION, csb);
} else if ( request.getSession().getAttribute(
SESSION_WORKINGCOLLECTION) != null && ((CollectionSummaryBean) request.getSession().getAttribute(
SESSION_WORKINGCOLLECTION)).getCollection().getId().equals(col.getId()) ) {
request.getSession().setAttribute(SESSION_WORKINGCOLLECTION, csb);
}
}
request.setAttribute(PAGE_COLLECTIONS, collections);
if ( hasJson(request) ) {
dispatcher = request.getRequestDispatcher("status-json.jsp");
} else if ( hasCsv(request) ) {
dispatcher = request.getRequestDispatcher("status-csv.jsp");
} else {
dispatcher = request.getRequestDispatcher("status.jsp");
}
dispatcher.forward(request, response);
} }
private boolean hasCsv( HttpServletRequest request ) { private boolean hasCsv(HttpServletRequest request) {
String value = (String) request.getParameter(PARAM_CSV); String value = (String) request.getParameter(PARAM_CSV);
return !Strings.isEmpty(value); return !Strings.isEmpty(value);
} }
......
...@@ -33,17 +33,16 @@ package edu.umiacs.ace.monitor.access.browse; ...@@ -33,17 +33,16 @@ package edu.umiacs.ace.monitor.access.browse;
import edu.umiacs.ace.hashtree.Proof; import edu.umiacs.ace.hashtree.Proof;
import edu.umiacs.ace.hashtree.ProofValidator; import edu.umiacs.ace.hashtree.ProofValidator;
import edu.umiacs.ace.monitor.core.MonitoredItem; import edu.umiacs.ace.monitor.access.browse.DirectoryTree.DirectoryNode;
import edu.umiacs.ace.util.EntityManagerServlet;
import edu.umiacs.ace.monitor.audit.AuditThreadFactory; import edu.umiacs.ace.monitor.audit.AuditThreadFactory;
import edu.umiacs.ace.monitor.audit.AuditTokens; import edu.umiacs.ace.monitor.audit.AuditTokens;
import edu.umiacs.ace.monitor.access.browse.DirectoryTree.DirectoryNode;
import edu.umiacs.ace.monitor.core.Collection; import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.core.MonitoredItem;
import edu.umiacs.ace.util.EntityManagerServlet;
import edu.umiacs.ace.util.HashValue; import edu.umiacs.ace.util.HashValue;
import edu.umiacs.ace.util.TokenUtil; import edu.umiacs.ace.util.TokenUtil;
import java.io.IOException; import org.apache.log4j.Logger;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import javax.persistence.EntityManager; import javax.persistence.EntityManager;
import javax.persistence.EntityNotFoundException; import javax.persistence.EntityNotFoundException;
import javax.servlet.RequestDispatcher; import javax.servlet.RequestDispatcher;
...@@ -51,7 +50,9 @@ import javax.servlet.ServletException; ...@@ -51,7 +50,9 @@ import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.HttpSession; import javax.servlet.http.HttpSession;
import org.apache.log4j.Logger; import java.io.IOException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
/** /**
* Servlet to manage browsing a collection. This will store a directorytree in * Servlet to manage browsing a collection. This will store a directorytree in
...@@ -102,6 +103,7 @@ public class BrowseServlet extends EntityManagerServlet { ...@@ -102,6 +103,7 @@ public class BrowseServlet extends EntityManagerServlet {
isRunning = true; isRunning = true;
} }
dt = new DirectoryTree(c); dt = new DirectoryTree(c);
request.setAttribute("collection", c);
session.setAttribute(SESSION_DIRECTORY_TREE, dt); session.setAttribute(SESSION_DIRECTORY_TREE, dt);
} else if ( itemId > 0 ) { } else if ( itemId > 0 ) {
LOG.trace("Toggling item: " + itemId); LOG.trace("Toggling item: " + itemId);
...@@ -114,6 +116,8 @@ public class BrowseServlet extends EntityManagerServlet { ...@@ -114,6 +116,8 @@ public class BrowseServlet extends EntityManagerServlet {
AuditThreadFactory.isQueued(c)) { AuditThreadFactory.isQueued(c)) {
isRunning = true; isRunning = true;
} }
request.setAttribute("collection", c);
session.setAttribute(SESSION_FILE, session.setAttribute(SESSION_FILE,
loadFileBean(dt.getDirectoryNode(itemId), em,c)); loadFileBean(dt.getDirectoryNode(itemId), em,c));
if ( dt.getDirectoryNode(itemId).isDirectory() ) { if ( dt.getDirectoryNode(itemId).isDirectory() ) {
...@@ -131,7 +135,12 @@ public class BrowseServlet extends EntityManagerServlet { ...@@ -131,7 +135,12 @@ public class BrowseServlet extends EntityManagerServlet {
dispatcher.forward(request, response); dispatcher.forward(request, response);
} }
private FileBean loadFileBean( DirectoryNode node, EntityManager em,Collection c ) { private FileBean loadFileBean( DirectoryNode node, EntityManager em, Collection c ) {
// avoid possible null references below
if (node == null) {
return null;
}
FileBean retBean = new FileBean(); FileBean retBean = new FileBean();
try { try {
......
...@@ -191,6 +191,7 @@ public final class AuditThread extends Thread implements CancelCallback { ...@@ -191,6 +191,7 @@ public final class AuditThread extends Thread implements CancelCallback {
iterableItems.cancel(); iterableItems.cancel();
} }
/*
if (batch != null) { if (batch != null) {
batch.close(); batch.close();
} }
...@@ -198,10 +199,12 @@ public final class AuditThread extends Thread implements CancelCallback { ...@@ -198,10 +199,12 @@ public final class AuditThread extends Thread implements CancelCallback {
if (validator != null) { if (validator != null) {
validator.close(); validator.close();
} }
*/
if (AuditThreadFactory.isRunning(coll) || AuditThreadFactory.isQueued(coll)) { if (AuditThreadFactory.isRunning(coll) || AuditThreadFactory.isQueued(coll)) {
AuditThreadFactory.finished(coll); AuditThreadFactory.finished(coll);
} }
this.interrupt(); this.interrupt();
} }
......
...@@ -63,7 +63,7 @@ public final class AuditTokens extends Thread implements CancelCallback { ...@@ -63,7 +63,7 @@ public final class AuditTokens extends Thread implements CancelCallback {
new ConcurrentHashMap<>(); new ConcurrentHashMap<>();
// private Map<TokenResponse, Token> tokenMap = new ConcurrentHashMap<TokenResponse, Token>(); // private Map<TokenResponse, Token> tokenMap = new ConcurrentHashMap<TokenResponse, Token>();
private Map<AceToken, MonitoredItem> itemMap = private Map<AceToken, MonitoredItem> itemMap =
new ConcurrentHashMap<AceToken, MonitoredItem>(); new ConcurrentHashMap<>();
private static final Logger LOG = Logger.getLogger(AuditTokens.class); private static final Logger LOG = Logger.getLogger(AuditTokens.class);
private Collection collection; private Collection collection;
private boolean cancel = false; private boolean cancel = false;
...@@ -186,7 +186,7 @@ public final class AuditTokens extends Thread implements CancelCallback { ...@@ -186,7 +186,7 @@ public final class AuditTokens extends Thread implements CancelCallback {
} catch ( Throwable e ) { } catch ( Throwable e ) {
LOG.fatal("UNcaught exception in doWork()", e); LOG.fatal("UNcaught exception in doWork()", e);
} finally { } finally {
itemMap.clear(); // free memory in case this gets stuck hianging around itemMap.clear(); // free memory in case this gets stuck hanging around
} }