Merge branch 'release-1.10'

ace-am/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <artifactId>ace</artifactId>
         <groupId>edu.umiacs.ace</groupId>
-        <version>1.10-SNAPSHOT</version>
+        <version>1.10</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <groupId>edu.umiacs.ace</groupId>
@@ -252,7 +252,7 @@
         <dependency>
             <groupId>edu.umiacs.ace</groupId>
             <artifactId>ace-ims-ws</artifactId>
-            <version>1.10-SNAPSHOT</version>
+            <version>1.10</version>
             <type>jar</type>
         </dependency>
         <dependency>

ace-am/src/main/java/edu/umiacs/ace/monitor/access/CollectionCountContext.java

@@ -30,23 +30,25 @@
 // $Id$
 package edu.umiacs.ace.monitor.access;
 
-import edu.umiacs.ace.util.PersistUtil;
 import edu.umiacs.ace.monitor.core.Collection;
+import edu.umiacs.ace.util.PersistUtil;
 import edu.umiacs.sql.SQL;
+import org.apache.log4j.Logger;
+import org.apache.log4j.NDC;
+
+import javax.persistence.EntityManager;
+import javax.persistence.Query;
+import javax.servlet.ServletContextEvent;
+import javax.servlet.ServletContextListener;
+import javax.sql.DataSource;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
-import javax.persistence.EntityManager;
-import javax.persistence.Query;
-import javax.servlet.ServletContextEvent;
-import javax.servlet.ServletContextListener;
-import javax.sql.DataSource;
-import org.apache.log4j.Logger;
-import org.apache.log4j.NDC;
 
 /**
  * Class to gather a count of all collections at startup.
@@ -70,6 +72,7 @@ public class CollectionCountContext implements ServletContextListener {
     private static Map<Collection, Long> totalSizeMap = new HashMap<Collection, Long>();
     private static Map<Collection, Long> fileRemoteMissing = new HashMap<Collection, Long>();
     private static Map<Collection, Long> fileRemoteCorrupt = new HashMap<Collection, Long>();
+    private static AtomicInteger totalCollections = new AtomicInteger(0);
     private static Lock lock = new ReentrantLock();
     private static boolean abort = false;
 
@@ -99,6 +102,7 @@ public class CollectionCountContext implements ServletContextListener {
                                 return;
                             }
                             queryCollection((Collection) o);
+                            incrementTotalCollections();
                         }
                         em.close();
                     } catch (Exception e) {
@@ -208,6 +212,18 @@ public class CollectionCountContext implements ServletContextListener {
         return -1;
     }
 
+    public static void incrementTotalCollections() {
+        totalCollections.incrementAndGet();
+    }
+
+    public static void decrementTotalCollections() {
+        totalCollections.decrementAndGet();
+    }
+
+    public static int getTotalCollections() {
+        return totalCollections.get();
+    }
+
     /**
      * Update statistics for a collection.
      * 

ace-am/src/main/java/edu/umiacs/ace/monitor/access/StatusServlet.java

@@ -30,88 +30,206 @@
 // $Id$
 package edu.umiacs.ace.monitor.access;
 
-import edu.umiacs.ace.util.EntityManagerServlet;
+import com.google.common.collect.ImmutableList;
 import edu.umiacs.ace.monitor.core.Collection;
+import edu.umiacs.ace.monitor.support.PageBean;
+import edu.umiacs.ace.monitor.support.CStateBean;
+import edu.umiacs.ace.util.EntityManagerServlet;
 import edu.umiacs.util.Strings;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
+import org.apache.log4j.Logger;
+
 import javax.persistence.EntityManager;
 import javax.persistence.Query;
 import javax.servlet.RequestDispatcher;
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
 
 /**
- *
  * @author toaster
  */
 public class StatusServlet extends EntityManagerServlet {
 
+    private static final Logger LOG = Logger.getLogger(StatusServlet.class);
+
     private static final String PAGE_COLLECTIONS = "collections";
+    private static final String PAGE_STATES = "states";
+    private static final String PAGE_COUNT = "count";
+    private static final String PAGE_NUMBER = "page";
+
+    private static final long DEFAULT_PAGE = 0;
+    private static final int DEFAULT_COUNT = 100;
+
+    // TODO: Add session fields for pagination and search params
     private static final String SESSION_WORKINGCOLLECTION = "workingCollection";
+
     private static final String PARAM_CSV = "csv";
 
-    /** 
+    // Pagination stuff
+    private static final String PARAM_COUNT = "count";
+    private static final String PARAM_PAGE = "page";
+
+    // Search Params
+    private static final String PARAM_GROUP = "group";
+    private static final String PARAM_STATE = "state";
+    private static final String PARAM_COLLECTION_LIKE = "collection";
+    private static final String PARAM_AUDIT_DATE = "audit";
+
+    // Filter params?
+    // ...group
+    // ...name (maybe some regex?)
+
+    /**
      * Processes requests for both HTTP <code>GET</code> and <code>POST</code> methods.
-     * @param request servlet request
+     *
+     * @param request  servlet request
      * @param response servlet response
      */
     @Override
-    protected void processRequest( HttpServletRequest request,
-            HttpServletResponse response, EntityManager em )
+    protected void processRequest(HttpServletRequest request,
+                                  HttpServletResponse response, EntityManager em)
             throws ServletException, IOException {
         RequestDispatcher dispatcher;
         List<CollectionSummaryBean> collections;
         List<Collection> items;
+        LOG.info("Processing /Status request");
+        LOG.info("Processing /Status additional info");
+
+        long page = getParameter(request, PARAM_PAGE, DEFAULT_PAGE);
+        int count = (int) getParameter(request, PARAM_COUNT, DEFAULT_COUNT);
+        String group = getParameter(request, PARAM_GROUP, null);
+        String collection = getParameter(request, PARAM_COLLECTION_LIKE, null);
+        String state = getParameter(request, PARAM_STATE, null);
+        // String date = getParameter(request, PARAM_GROUP, null);
+        PageBean pb = new PageBean((int) page, count, "Status");
+
+        long offset = page * count;
+
+        // our main query
+        StringBuilder queryString = new StringBuilder();
+
+        // for getting a count of items
+        StringBuilder countString = new StringBuilder();
 
+        // our parameters
+        StringBuilder params = new StringBuilder();
+
+        List<String> queries = new ArrayList<>();
+
+        if (!Strings.isEmpty(group)) {
+            queries.add("c.group LIKE :group");
+            pb.addParam(PARAM_GROUP, group);
+        }
+
+        if (!Strings.isEmpty(collection)) {
+            queries.add("c.name LIKE :collection");
+            pb.addParam(PARAM_COLLECTION_LIKE, collection);
+        }
+
+        // Enforce that the state is not empty, or larger than 1 character
+        if (!Strings.isEmpty(state) && state.length() == 1) {
+            queries.add("c.state = :state");
+            pb.addParam(PARAM_STATE, state);
+        }
+
+        queryString.append("SELECT c FROM Collection c");
+        countString.append("SELECT COUNT(c.id) FROM Collection c");
+
+        if (queries.size() > 0) {
+            params.append(" WHERE");
+        }
+
+        Iterator<String> it = queries.iterator();
+        while (it.hasNext()) {
+            String query = it.next();
+            params.append(" ")
+                  .append(query);
+            if (it.hasNext()) {
+                params.append(" AND");
+            }
+        }
+
+        queryString.append(params);
+        // allows us to keep a consistent order when displaying collections
+        queryString.append(" ORDER BY c.group ASC, c.name ASC");
+        countString.append(params);
 
         Query query =
-                em.createNamedQuery("Collection.listAllCollections");
+                em.createQuery(queryString.toString());
+        // em.createNamedQuery("Collection.listAllCollections");
+        query.setFirstResult((int) offset);
+        query.setMaxResults(count);
 
-        items = query.getResultList();
+        Query countQuery = em.createQuery(countString.toString());
 
-        collections = new ArrayList<CollectionSummaryBean>();
+        if (!Strings.isEmpty(group)) {
+            query.setParameter(PARAM_GROUP, "%" + group + "%");
+            countQuery.setParameter(PARAM_GROUP, "%" + group + "%");
+        }
 
-        if ( Strings.isValidLong(request.getParameter(PARAM_COLLECTION_ID)) && -1 == Long.parseLong(request.getParameter(
-                PARAM_COLLECTION_ID)) ) {
-            request.getSession().removeAttribute(SESSION_WORKINGCOLLECTION);
+        if (!Strings.isEmpty(collection)) {
+            query.setParameter(PARAM_COLLECTION_LIKE, "%" + collection + "%");
+            countQuery.setParameter(PARAM_COLLECTION_LIKE, "%" + collection + "%");
         }
-        for ( Collection col : items ) {
-            CollectionSummaryBean csb = new CollectionSummaryBean();
-            csb.setCollection(col);
-            csb.setTotalFiles(CollectionCountContext.getFileCount(col));
-            csb.setActiveFiles(CollectionCountContext.getActiveCount(col));
-            csb.setCorruptFiles(CollectionCountContext.getCorruptCount(col));
-            csb.setInvalidDigests(CollectionCountContext.getTokenMismatchCount(
-                    col));
-            csb.setMissingFiles(CollectionCountContext.getMissingCount(col));
-            csb.setMissingTokens(CollectionCountContext.getMissingTokenCount(col));
-            csb.setTotalSize(CollectionCountContext.getTotelSize(col));
-            csb.setTotalErrors(CollectionCountContext.getTotalErrors(col));
-            csb.setRemoteMissing(CollectionCountContext.getRemoteMissing(col));
-            csb.setRemoteCorrupt(CollectionCountContext.getRemoteCorrupt(col));
-            collections.add(csb);
 
-            // if param_collection was supplied as a parameter see if this bean
-            // should be set as working/details bean
-            if ( Strings.isValidLong(request.getParameter(PARAM_COLLECTION_ID)) && col.getId() == Long.parseLong(request.getParameter(
-                    PARAM_COLLECTION_ID)) ) {
-                request.getSession().setAttribute(SESSION_WORKINGCOLLECTION, csb);
-            } else if ( request.getSession().getAttribute(
-                    SESSION_WORKINGCOLLECTION) != null && ((CollectionSummaryBean) request.getSession().getAttribute(
-                    SESSION_WORKINGCOLLECTION)).getCollection().getId().equals(col.getId()) ) {
-                request.getSession().setAttribute(SESSION_WORKINGCOLLECTION, csb);
+        if (!Strings.isEmpty(state) && state.length() == 1) {
+            query.setParameter(PARAM_STATE, state.charAt(0));
+            countQuery.setParameter(PARAM_STATE, state.charAt(0));
+        }
+
+        items = query.getResultList();
 
+        // We only need to execute this query if we have parameters
+        // and need to update the total count of collections
+        if (queries.size() > 0) {
+            long totalResults = (long) countQuery.getSingleResult();
+            LOG.info("Total results from query: " + totalResults);
+            pb.update(totalResults);
+        }
+
+        collections = new ArrayList<>();
+
+        long collectionId;
+        String idParam = request.getParameter(PARAM_COLLECTION_ID);
+        collectionId = Strings.isValidLong(idParam) ? Long.parseLong(idParam) : -1;
+        CollectionSummaryBean workingCollectionBean = (CollectionSummaryBean) request.getSession().getAttribute(SESSION_WORKINGCOLLECTION);
+
+        if (Strings.isValidLong(idParam) && -1 == collectionId) {
+            // clear the working collection
+            request.getSession().removeAttribute(SESSION_WORKINGCOLLECTION);
+
+        } else if (Strings.isValidLong(idParam)                                            // valid param
+                && (workingCollectionBean == null                                          // no working collection
+                || !workingCollectionBean.getCollection().getId().equals(collectionId))) { // or one which does not equal our requested collection
+            // Get the requested collection from the db and set it as our working collection
+            Collection workingCollection = em.find(Collection.class, collectionId);
+            if (workingCollection != null) {
+                workingCollectionBean = createCollectionSummary(workingCollection);
+                request.getSession().setAttribute(SESSION_WORKINGCOLLECTION, workingCollectionBean);
             }
 
+        } else {
+            // Continue using the current working collection
+            request.getSession().setAttribute(SESSION_WORKINGCOLLECTION, workingCollectionBean);
+        }
+
+
+        for (Collection col : items) {
+            CollectionSummaryBean csb = createCollectionSummary(col);
+            collections.add(csb);
         }
 
         request.setAttribute(PAGE_COLLECTIONS, collections);
-        if ( hasJson(request) ) {
+        request.setAttribute(PAGE_STATES, ImmutableList.copyOf(CStateBean.values()));
+        request.setAttribute(PAGE_COUNT, count);
+        request.setAttribute(PAGE_NUMBER, pb);
+        if (hasJson(request)) {
             dispatcher = request.getRequestDispatcher("status-json.jsp");
-        } else if ( hasCsv(request) ) {
+        } else if (hasCsv(request)) {
             dispatcher = request.getRequestDispatcher("status-csv.jsp");
         } else {
             dispatcher = request.getRequestDispatcher("status.jsp");
@@ -119,7 +237,24 @@ public class StatusServlet extends EntityManagerServlet {
         dispatcher.forward(request, response);
     }
 
-    private boolean hasCsv( HttpServletRequest request ) {
+    private CollectionSummaryBean createCollectionSummary(Collection col) {
+        CollectionSummaryBean csb = new CollectionSummaryBean();
+        csb.setCollection(col);
+        csb.setTotalFiles(CollectionCountContext.getFileCount(col));
+        csb.setActiveFiles(CollectionCountContext.getActiveCount(col));
+        csb.setCorruptFiles(CollectionCountContext.getCorruptCount(col));
+        csb.setInvalidDigests(CollectionCountContext.getTokenMismatchCount(
+                col));
+        csb.setMissingFiles(CollectionCountContext.getMissingCount(col));
+        csb.setMissingTokens(CollectionCountContext.getMissingTokenCount(col));
+        csb.setTotalSize(CollectionCountContext.getTotelSize(col));
+        csb.setTotalErrors(CollectionCountContext.getTotalErrors(col));
+        csb.setRemoteMissing(CollectionCountContext.getRemoteMissing(col));
+        csb.setRemoteCorrupt(CollectionCountContext.getRemoteCorrupt(col));
+        return csb;
+    }
+
+    private boolean hasCsv(HttpServletRequest request) {
         String value = (String) request.getParameter(PARAM_CSV);
         return !Strings.isEmpty(value);
     }

ace-am/src/main/java/edu/umiacs/ace/monitor/access/browse/BrowseServlet.java

@@ -33,17 +33,16 @@ package edu.umiacs.ace.monitor.access.browse;
 
 import edu.umiacs.ace.hashtree.Proof;
 import edu.umiacs.ace.hashtree.ProofValidator;
-import edu.umiacs.ace.monitor.core.MonitoredItem;
-import edu.umiacs.ace.util.EntityManagerServlet;
+import edu.umiacs.ace.monitor.access.browse.DirectoryTree.DirectoryNode;
 import edu.umiacs.ace.monitor.audit.AuditThreadFactory;
 import edu.umiacs.ace.monitor.audit.AuditTokens;
-import edu.umiacs.ace.monitor.access.browse.DirectoryTree.DirectoryNode;
 import edu.umiacs.ace.monitor.core.Collection;
+import edu.umiacs.ace.monitor.core.MonitoredItem;
+import edu.umiacs.ace.util.EntityManagerServlet;
 import edu.umiacs.ace.util.HashValue;
 import edu.umiacs.ace.util.TokenUtil;
-import java.io.IOException;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
+import org.apache.log4j.Logger;
+
 import javax.persistence.EntityManager;
 import javax.persistence.EntityNotFoundException;
 import javax.servlet.RequestDispatcher;
@@ -51,7 +50,9 @@ import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import javax.servlet.http.HttpSession;
-import org.apache.log4j.Logger;
+import java.io.IOException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 
 /**
  * Servlet to manage browsing a collection. This will store a directorytree in
@@ -102,6 +103,7 @@ public class BrowseServlet extends EntityManagerServlet {
                 isRunning = true;
             }
             dt = new DirectoryTree(c);
+            request.setAttribute("collection", c);
             session.setAttribute(SESSION_DIRECTORY_TREE, dt);
         } else if ( itemId > 0 ) {
             LOG.trace("Toggling item: " + itemId);
@@ -114,6 +116,8 @@ public class BrowseServlet extends EntityManagerServlet {
                     AuditThreadFactory.isQueued(c)) {
                 isRunning = true;
             }
+
+            request.setAttribute("collection", c);
             session.setAttribute(SESSION_FILE,
                     loadFileBean(dt.getDirectoryNode(itemId), em,c));
             if ( dt.getDirectoryNode(itemId).isDirectory() ) {
@@ -131,7 +135,12 @@ public class BrowseServlet extends EntityManagerServlet {
         dispatcher.forward(request, response);
     }
 
-    private FileBean loadFileBean( DirectoryNode node, EntityManager em,Collection c ) {
+    private FileBean loadFileBean( DirectoryNode node, EntityManager em, Collection c ) {
+        // avoid possible null references below
+        if (node == null) {
+            return null;
+        }
+
         FileBean retBean = new FileBean();
 
         try {

ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java

@@ -191,6 +191,7 @@ public final class AuditThread extends Thread implements CancelCallback {
             iterableItems.cancel();
         }
 
+        /*
         if (batch != null) {
             batch.close();
         }
@@ -198,10 +199,12 @@ public final class AuditThread extends Thread implements CancelCallback {
         if (validator != null) {
             validator.close();
         }
+        */
 
         if (AuditThreadFactory.isRunning(coll) || AuditThreadFactory.isQueued(coll)) {
             AuditThreadFactory.finished(coll);
         }
+
         this.interrupt();
     }
 

ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditTokens.java

@@ -63,7 +63,7 @@ public final class AuditTokens extends Thread implements CancelCallback {
             new ConcurrentHashMap<>();
 //    private Map<TokenResponse, Token> tokenMap = new ConcurrentHashMap<TokenResponse, Token>();
     private Map<AceToken, MonitoredItem> itemMap =
-            new ConcurrentHashMap<AceToken, MonitoredItem>();
+        new ConcurrentHashMap<>();
     private static final Logger LOG = Logger.getLogger(AuditTokens.class);
     private Collection collection;
     private boolean cancel = false;
@@ -186,7 +186,7 @@ public final class AuditTokens extends Thread implements CancelCallback {
         } catch ( Throwable e ) {
             LOG.fatal("UNcaught exception in doWork()", e);
         } finally {
-            itemMap.clear(); // free memory in case this gets stuck hianging around
+            itemMap.clear(); // free memory in case this gets stuck hanging around
         }
 
     }
@@ -235,6 +235,7 @@ public final class AuditTokens extends Thread implements CancelCallback {
         TokenValidator validator = openIms();
 
         if ( validator == null ) {
+            runningThreads.remove(collection);
             return;
         }
         em = PersistUtil.getEntityManager();

ace-am/src/main/java/edu/umiacs/ace/monitor/core/ManageCollectionServlet.java

@@ -30,15 +30,14 @@
 // $Id$
 package edu.umiacs.ace.monitor.core;
 
-import edu.umiacs.ace.util.EntityManagerServlet;
-import edu.umiacs.ace.util.PersistUtil;
 import edu.umiacs.ace.driver.StorageDriver;
 import edu.umiacs.ace.driver.StorageDriverFactory;
+import edu.umiacs.ace.monitor.access.CollectionCountContext;
+import edu.umiacs.ace.util.EntityManagerServlet;
+import edu.umiacs.ace.util.PersistUtil;
 import edu.umiacs.util.Strings;
-import java.io.IOException;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.HashMap;
+import org.apache.log4j.Logger;
+
 import javax.persistence.EntityManager;
 import javax.persistence.EntityTransaction;
 import javax.persistence.Query;
@@ -47,7 +46,10 @@ import javax.servlet.ServletException;
 import javax.servlet.ServletRequest;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
-import org.apache.log4j.Logger;
+import java.io.IOException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.HashMap;
 
 /**
  * add,modify,remove the settings of a collection
@@ -154,6 +156,7 @@ public class ManageCollectionServlet extends EntityManagerServlet {
                 PersistUtil.persist(collection);
                 storage = StorageDriverFactory.createStorageAccess(collection,
                         em);
+                CollectionCountContext.incrementTotalCollections();
             }
 
             dispatcher = request.getRequestDispatcher("collectionmodify.jsp");
@@ -206,6 +209,7 @@ public class ManageCollectionServlet extends EntityManagerServlet {
             storage.remove(em);
         }
         em.remove(collection);
+        CollectionCountContext.decrementTotalCollections();
         trans.commit();
     }
 

ace-am/src/main/java/edu/umiacs/ace/monitor/support/CStateBean.java

+package edu.umiacs.ace.monitor.support;
+
+/**
+ * A Bean to hold each collection status as well as its definition
+ *
+ * Created by shake on 4/5/16.
+ */
+public enum CStateBean {
+
+    NO_FILTER(""),
+    ACTIVE("A"),
+    ERROR("E"),
+    NEVER_SCANNED("N");
+
+    private final String state;
+
+    CStateBean(String state) {
+        this.state = state;
+    }
+
+    public String getState() {
+        return state;
+    }
+}

ace-am/src/main/java/edu/umiacs/ace/monitor/support/PageBean.java

+package edu.umiacs.ace.monitor.support;
+
+import edu.umiacs.ace.monitor.access.CollectionCountContext;
+import org.apache.log4j.Logger;
+
+/**
+ *
+ * Created by shake on 1/28/16.
+ */
+public class PageBean {
+
+    private static Logger LOG = Logger.getLogger(PageBean.class);
+
+    // current page
+    int page;
+
+    // page number
+    int count;
+
+    // item offset (for db queries)
+    int offset;
+
+    int previous;
+    int next;
+    int end;
+
+    StringBuilder url;
+    private boolean firstParam;
+
+    public PageBean(int page, int count, String root) {
+        this.firstParam = true;
+        this.page = page;
+        this.count = count;
+        this.offset = page*count;