Merge branch 'release-1.11'

ace-am/pom.xml

@@ -4,10 +4,9 @@
     <parent>
         <artifactId>ace</artifactId>
         <groupId>edu.umiacs.ace</groupId>
-        <version>1.10</version>
+        <version>1.11-RELEASE</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
-    <groupId>edu.umiacs.ace</groupId>
     <artifactId>ace-am</artifactId>
     <name>ace-am</name>
     <packaging>war</packaging>
@@ -102,12 +101,12 @@
         <dependency>
             <groupId>org.eclipse.persistence</groupId>
             <artifactId>eclipselink</artifactId>
-            <version>2.2.0</version>
+            <version>2.6.3</version>
         </dependency>
         <dependency>
             <groupId>org.eclipse.persistence</groupId>
-            <artifactId>javax.persistence</artifactId>
-            <version>2.0.3</version>
+            <artifactId>org.eclipse.persistence.jpa</artifactId>
+            <version>2.6.3</version>
         </dependency>
 
         <dependency>
@@ -118,9 +117,9 @@
 
         <!-- for json -->
         <dependency>
-            <groupId>org.codehaus.jackson</groupId>
-            <artifactId>jackson-mapper-asl</artifactId>
-            <version>1.8.1</version>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-core</artifactId>
+            <version>2.8.1</version>
         </dependency>
 
         <dependency>
@@ -178,6 +177,12 @@
             <version>3.1</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-csv</artifactId>
+            <version>1.4</version>
+        </dependency>
+
         <dependency>
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>
@@ -238,7 +243,7 @@
             </exclusions>
         </dependency> -->
         
-        <!-- web services -->
+        <!-- web services
         <dependency>
             <groupId>com.sun.jersey</groupId>
             <artifactId>jersey-server</artifactId>
@@ -249,10 +254,21 @@
             <artifactId>jersey-json</artifactId>
             <version>1.6</version>
         </dependency>
+        -->
+        <dependency>
+            <groupId>org.glassfish.jersey.containers</groupId>
+            <artifactId>jersey-container-servlet</artifactId>
+            <version>2.24</version>
+        </dependency>
+        <dependency>
+            <groupId>org.glassfish.jersey.media</groupId>
+            <artifactId>jersey-media-json-jackson</artifactId>
+            <version>2.24</version>
+        </dependency>
         <dependency>
             <groupId>edu.umiacs.ace</groupId>
             <artifactId>ace-ims-ws</artifactId>
-            <version>1.10</version>
+            <version>1.11-RELEASE</version>
             <type>jar</type>
         </dependency>
         <dependency>

ace-am/src/main/java/edu/umiacs/ace/driver/FileBean.java

@@ -83,7 +83,6 @@ public class FileBean {
      * [0] /1k-10/d7/file-19
      * [1] /1k-10/d7
      * [2] /1k-10
-     * @param parentPathList
      */
     public void setPathList( String[] pathList ) {
         if ( pathList != null ) {

ace-am/src/main/java/edu/umiacs/ace/driver/StorageDriverFactory.java

@@ -33,16 +33,17 @@ package edu.umiacs.ace.driver;
 import edu.umiacs.ace.driver.benchmark.BenchmarkAccess;
 import edu.umiacs.ace.driver.irods.IrodsAccess;
 import edu.umiacs.ace.driver.localfile.LocalFileAccess;
-import edu.umiacs.ace.monitor.core.Collection;
 import edu.umiacs.ace.driver.srb.SrbAccess;
 import edu.umiacs.ace.driver.swap.SwapFileAccess;
+import edu.umiacs.ace.monitor.core.Collection;
+
+import javax.persistence.EntityManager;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import javax.persistence.EntityManager;
 
 /**
  * Factory to register all storage resources supported by this monitor.
@@ -56,7 +57,7 @@ public final class StorageDriverFactory {
      * Map to track descriptive storage type name to implementing class
      */
     private static final Map<String, Class<? extends StorageDriver>> implementationMap =
-            new HashMap<String, Class<? extends StorageDriver>>();
+            new HashMap<>();
 
     /**
      * All available storage gateways must be registered here
@@ -71,7 +72,7 @@ public final class StorageDriverFactory {
     }
 
     public static final List<String> listResources() {
-        List<String> lt = new ArrayList<String>(implementationMap.keySet());
+        List<String> lt = new ArrayList<>(implementationMap.keySet());
         Collections.sort(lt,comp);
         return lt;
     }

ace-am/src/main/java/edu/umiacs/ace/driver/irods/IrodsHandler.java

@@ -36,11 +36,12 @@ import edu.umiacs.ace.driver.filter.PathFilter;
 import edu.umiacs.ace.util.HashValue;
 import edu.umiacs.irods.operation.BulkFileHandler;
 import edu.umiacs.srb.util.StringUtil;
+import org.apache.log4j.Logger;
+
 import java.io.IOException;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.concurrent.LinkedBlockingQueue;
-import org.apache.log4j.Logger;
 
 /**
  *
@@ -61,7 +62,6 @@ public class IrodsHandler implements BulkFileHandler {
      * @param readyList
      * @param filter
      * @param digestAlgorithm
-     * @param base prefix to add to returned files (no trailing /)
      */
     public IrodsHandler( LinkedBlockingQueue<FileBean> readyList,
             PathFilter filter, String digestAlgorithm ) {

ace-am/src/main/java/edu/umiacs/ace/driver/swap/SwapSettings.java

@@ -133,7 +133,6 @@ public class SwapSettings implements Serializable {
     }
 
     /**
-     * @param server the server to set
      */
     public void setServers( String servers ) {
         this.servers = servers;

ace-am/src/main/java/edu/umiacs/ace/monitor/access/StatusServlet.java

@@ -32,8 +32,8 @@ package edu.umiacs.ace.monitor.access;
 
 import com.google.common.collect.ImmutableList;
 import edu.umiacs.ace.monitor.core.Collection;
-import edu.umiacs.ace.monitor.support.PageBean;
 import edu.umiacs.ace.monitor.support.CStateBean;
+import edu.umiacs.ace.monitor.support.PageBean;
 import edu.umiacs.ace.util.EntityManagerServlet;
 import edu.umiacs.util.Strings;
 import org.apache.log4j.Logger;
@@ -101,11 +101,13 @@ public class StatusServlet extends EntityManagerServlet {
 
         long page = getParameter(request, PARAM_PAGE, DEFAULT_PAGE);
         int count = (int) getParameter(request, PARAM_COUNT, DEFAULT_COUNT);
+
+        // local getParameter so that the session is checked as well
         String group = getParameter(request, PARAM_GROUP, null);
         String collection = getParameter(request, PARAM_COLLECTION_LIKE, null);
         String state = getParameter(request, PARAM_STATE, null);
         // String date = getParameter(request, PARAM_GROUP, null);
-        PageBean pb = new PageBean((int) page, count, "Status");
+        PageBean pb = new PageBean((int) page, count, "");
 
         long offset = page * count;
 
@@ -120,20 +122,24 @@ public class StatusServlet extends EntityManagerServlet {
 
         List<String> queries = new ArrayList<>();
 
+        // TODO: Can probably tidy this up a bit
         if (!Strings.isEmpty(group)) {
             queries.add("c.group LIKE :group");
             pb.addParam(PARAM_GROUP, group);
+            request.setAttribute(PARAM_GROUP, group);
         }
 
         if (!Strings.isEmpty(collection)) {
             queries.add("c.name LIKE :collection");
             pb.addParam(PARAM_COLLECTION_LIKE, collection);
+            request.setAttribute(PARAM_COLLECTION_LIKE, collection);
         }
 
         // Enforce that the state is not empty, or larger than 1 character
         if (!Strings.isEmpty(state) && state.length() == 1) {
             queries.add("c.state = :state");
             pb.addParam(PARAM_STATE, state);
+            request.setAttribute(PARAM_STATE, state);
         }
 
         queryString.append("SELECT c FROM Collection c");
@@ -166,6 +172,7 @@ public class StatusServlet extends EntityManagerServlet {
 
         Query countQuery = em.createQuery(countString.toString());
 
+        // TODO: Can probably tidy this up a bit
         if (!Strings.isEmpty(group)) {
             query.setParameter(PARAM_GROUP, "%" + group + "%");
             countQuery.setParameter(PARAM_GROUP, "%" + group + "%");
@@ -191,8 +198,29 @@ public class StatusServlet extends EntityManagerServlet {
             pb.update(totalResults);
         }
 
+        setWorkingCollection(request, em);
+
         collections = new ArrayList<>();
+        for (Collection col : items) {
+            CollectionSummaryBean csb = createCollectionSummary(col);
+            collections.add(csb);
+        }
 
+        request.setAttribute(PAGE_COLLECTIONS, collections);
+        request.setAttribute(PAGE_STATES, ImmutableList.copyOf(CStateBean.values()));
+        request.setAttribute(PAGE_COUNT, count);
+        request.setAttribute(PAGE_NUMBER, pb);
+        if (hasJson(request)) {
+            dispatcher = request.getRequestDispatcher("status-json.jsp");
+        } else if (hasCsv(request)) {
+            dispatcher = request.getRequestDispatcher("status-csv.jsp");
+        } else {
+            dispatcher = request.getRequestDispatcher("status.jsp");
+        }
+        dispatcher.forward(request, response);
+    }
+
+    private void setWorkingCollection(HttpServletRequest request, EntityManager em) {
         long collectionId;
         String idParam = request.getParameter(PARAM_COLLECTION_ID);
         collectionId = Strings.isValidLong(idParam) ? Long.parseLong(idParam) : -1;
@@ -201,7 +229,6 @@ public class StatusServlet extends EntityManagerServlet {
         if (Strings.isValidLong(idParam) && -1 == collectionId) {
             // clear the working collection
             request.getSession().removeAttribute(SESSION_WORKINGCOLLECTION);
-
         } else if (Strings.isValidLong(idParam)                                            // valid param
                 && (workingCollectionBean == null                                          // no working collection
                 || !workingCollectionBean.getCollection().getId().equals(collectionId))) { // or one which does not equal our requested collection
@@ -216,25 +243,6 @@ public class StatusServlet extends EntityManagerServlet {
             // Continue using the current working collection
             request.getSession().setAttribute(SESSION_WORKINGCOLLECTION, workingCollectionBean);
         }
-
-
-        for (Collection col : items) {
-            CollectionSummaryBean csb = createCollectionSummary(col);
-            collections.add(csb);
-        }
-
-        request.setAttribute(PAGE_COLLECTIONS, collections);
-        request.setAttribute(PAGE_STATES, ImmutableList.copyOf(CStateBean.values()));
-        request.setAttribute(PAGE_COUNT, count);
-        request.setAttribute(PAGE_NUMBER, pb);
-        if (hasJson(request)) {
-            dispatcher = request.getRequestDispatcher("status-json.jsp");
-        } else if (hasCsv(request)) {
-            dispatcher = request.getRequestDispatcher("status-csv.jsp");
-        } else {
-            dispatcher = request.getRequestDispatcher("status.jsp");
-        }
-        dispatcher.forward(request, response);
     }
 
     private CollectionSummaryBean createCollectionSummary(Collection col) {
@@ -255,7 +263,23 @@ public class StatusServlet extends EntityManagerServlet {
     }
 
     private boolean hasCsv(HttpServletRequest request) {
-        String value = (String) request.getParameter(PARAM_CSV);
+        String value = request.getParameter(PARAM_CSV);
         return !Strings.isEmpty(value);
     }
+
+    @Override
+    public String getParameter(HttpServletRequest request, String paramName, String defaultValue) {
+        String requestParam = request.getParameter(paramName);
+
+        // We could remove the session attributes here, not sure if that's a good idea though
+
+        // req param: not null and not empty -> use
+        // session attr not null and req param null -> use
+        // session attr not null and req param empty -> default
+        if (!Strings.isEmpty(requestParam)) {
+            return requestParam;
+        } else {
+            return defaultValue;
+        }
+    }
 }

ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java

@@ -32,48 +32,54 @@ package edu.umiacs.ace.monitor.audit;
 
 import edu.umiacs.ace.driver.AuditIterable;
 import edu.umiacs.ace.driver.DriverStateBean;
-import edu.umiacs.ace.driver.filter.PathFilter;
 import edu.umiacs.ace.driver.FileBean;
 import edu.umiacs.ace.driver.StorageDriver;
+import edu.umiacs.ace.driver.filter.PathFilter;
+import edu.umiacs.ace.driver.filter.SimpleFilter;
 import edu.umiacs.ace.ims.api.IMSException;
 import edu.umiacs.ace.ims.api.IMSService;
 import edu.umiacs.ace.ims.api.TokenRequestBatch;
 import edu.umiacs.ace.ims.api.TokenValidator;
 import edu.umiacs.ace.ims.ws.TokenRequest;
 import edu.umiacs.ace.monitor.access.CollectionCountContext;
-import edu.umiacs.ace.monitor.register.IngestThreadPool;
-import edu.umiacs.ace.util.PersistUtil;
-import edu.umiacs.ace.driver.filter.SimpleFilter;
 import edu.umiacs.ace.monitor.compare.CollectionCompare2;
 import edu.umiacs.ace.monitor.compare.CompareResults;
-import edu.umiacs.ace.remote.JsonGateway;
+import edu.umiacs.ace.monitor.core.Collection;
+import edu.umiacs.ace.monitor.core.ConfigConstants;
 import edu.umiacs.ace.monitor.core.MonitoredItem;
 import edu.umiacs.ace.monitor.core.MonitoredItemManager;
+import edu.umiacs.ace.monitor.log.LogEnum;
+import edu.umiacs.ace.monitor.log.LogEvent;
 import edu.umiacs.ace.monitor.log.LogEventManager;
+import edu.umiacs.ace.monitor.peers.PeerCollection;
 import edu.umiacs.ace.monitor.reporting.ReportSummary;
 import edu.umiacs.ace.monitor.reporting.SchedulerContextListener;
 import edu.umiacs.ace.monitor.reporting.SummaryGenerator;
-import edu.umiacs.ace.monitor.core.Collection;
-import edu.umiacs.ace.monitor.core.ConfigConstants;
 import edu.umiacs.ace.monitor.settings.SettingsUtil;
-import edu.umiacs.ace.monitor.log.LogEnum;
-import edu.umiacs.ace.monitor.log.LogEvent;
-import edu.umiacs.ace.monitor.peers.PeerCollection;
+import edu.umiacs.ace.remote.JsonGateway;
 import edu.umiacs.ace.token.AceToken;
+import edu.umiacs.ace.util.PersistUtil;
 import edu.umiacs.ace.util.TokenUtil;
 import edu.umiacs.util.Strings;
+import org.apache.log4j.Logger;
+import org.apache.log4j.NDC;
+
+import javax.mail.MessagingException;
+import javax.persistence.EntityManager;
+import javax.persistence.EntityTransaction;
+import javax.persistence.Query;
+import javax.sql.DataSource;
 import java.io.InputStream;
 import java.security.MessageDigest;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Timestamp;
 import java.util.Arrays;
 import java.util.Date;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
-import javax.mail.MessagingException;
-import javax.persistence.EntityManager;
-import javax.persistence.EntityTransaction;
-import org.apache.log4j.Logger;
-import org.apache.log4j.NDC;
 
 /**
  *
@@ -191,16 +197,6 @@ public final class AuditThread extends Thread implements CancelCallback {
             iterableItems.cancel();
         }
 
-        /*
-        if (batch != null) {
-            batch.close();
-        }
-
-        if (validator != null) {
-            validator.close();
-        }
-        */
-
         if (AuditThreadFactory.isRunning(coll) || AuditThreadFactory.isQueued(coll)) {
             AuditThreadFactory.finished(coll);
         }
@@ -222,12 +218,6 @@ public final class AuditThread extends Thread implements CancelCallback {
                 session = System.currentTimeMillis();
                 logManager = new LogEventManager(session, coll);
                 logAuditStart();
-                /*
-                while (IngestThreadPool.isIngesting(coll)) {
-                    LOG.debug("Waiting for ingest to finish");
-                    Thread.sleep(500);
-                }
-                */
 
                 callback = new FileAuditCallback(coll, session, this);
                 boolean auditTokens = SettingsUtil.getBoolean(coll,
@@ -254,7 +244,7 @@ public final class AuditThread extends Thread implements CancelCallback {
                 // Let outstanding tokens finish, TODO, de-hackify this.
                 sleep(2000);
             } catch (Throwable e) {
-                LOG.fatal("UNcaught exception in performAudit()", e);
+                LOG.fatal("Uncaught exception in performAudit()", e);
                 if (abortException != null) {
                     abortException = e;
                 }
@@ -595,7 +585,6 @@ public final class AuditThread extends Thread implements CancelCallback {
         }
 
         TokenRequest request = new TokenRequest();
-        //request.setName(item.getPath());