Merge branch 'release-1.14'

.gitlab-ci.yml

@@ -18,7 +18,7 @@
 variables:
   # This will supress any download for dependencies and plugins or upload messages which would clutter the console log.
   # `showDateTime` will show the passed time in milliseconds. You need to specify `--batch-mode` to make this work.
-  MAVEN_OPTS: "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Dorg.slf4j.simpleLogger.showDateTime=true -Djava.awt.headless=true"
+  MAVEN_OPTS: "-Dmaven.repo.local=$CI_PROJECT_DIR/.m2/repository -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Dorg.slf4j.simpleLogger.showDateTime=true -Djava.awt.headless=true"
   # As of Maven 3.3.0 instead of this you may define these options in `.mvn/maven.config` so the same config is used
   # when running from the command line.
   # `installAtEnd` and `deployAtEnd`are only effective with recent version of the corresponding plugins.
@@ -27,63 +27,42 @@ variables:
 # Cache downloaded dependencies and plugins between builds.
 cache:
   paths:
-    - /root/.m2/repository/
+    - .m2/repository/
 
-# This will only validate and compile stuff and run e.g. maven-enforcer-plugin. 
-# Because some enforcer rules might check dependency convergence and class duplications
-# we use `test-compile` here instead of `validate`, so the correct classpath is picked up.
-.validate: &validate
-  stage: build
-  script:
-    - 'mvn $MAVEN_CLI_OPTS test-compile'
-
-# For merge requests do not `deploy` but only run `verify`.
-# See https://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html
-.verify: &verify
-  stage: test
-  script:
-    - 'mvn $MAVEN_CLI_OPTS verify'
-  except:
-    - master
+image: maven:3-jdk-8
 
 # Validate merge requests using JDK8
 validate:jdk8:
-  <<: *validate
-  image: maven:3.3.9-jdk-8
+  stage: build
+  script:
+    - mvn $MAVEN_CLI_OPTS test-compile
 
 # Verify merge requests using JDK8
 verify:jdk8:
-  <<: *verify
-  image: maven:3.3.9-jdk-8
-
+  stage: test
+  script:
+    - mvn $MAVEN_CLI_OPTS verify
 
-# For `master` branch run `mvn deploy` automatically.
-# Here you need to decide whether you want to use JDK7 or 8.
-# To get this working you need to define a volume while configuring your gitlab-ci-multi-runner.
-# Mount your `settings.xml` as `/root/.m2/settings.xml` which holds your secrets.
-# See https://maven.apache.org/settings.html
+# Call mvn deploy to upload artifacts for devlop and releases
+# Requires settings.xml to be mounted in /root/.m2/settings.xml
 deploy:jdk8:
-  # Use stage test here, so the pages job may later pickup the created site.
-  stage: test
+  stage: deploy
   script:
-    - 'mvn $MAVEN_CLI_OPTS deploy'
+    - mvn $MAVEN_CLI_OPTS deploy
   only:
-    - master
-  # Archive up the built documentation site.
-  image: maven:3.3.9-jdk-8
+    - develop
+    - tags
+    - /^release-.*$/
 
 upload:
-  image: maven:3.3.9-jdk-8
   stage: deploy
   only:
-    - master
     - develop
+    - tags
+    - /^release-.*$/
   script:
     - 'mvn package'
-    - 'for f in `find ace-am/src/main/sql -type f`; do curl -X POST -H "Authorization: token $BUILD_TOKEN" --data-binary @$f $SERVER/artifacts/ace/$CI_BUILD_REF_NAME/$(echo $CI_BUILD_REF | cut -c -7)/$(basename $f); done'
-    - 'for f in `find ace-am/target -type f -name "*.war"`; do curl -X POST -H "Authorization: token $BUILD_TOKEN" --data-binary @$f $SERVER/artifacts/ace/$CI_BUILD_REF_NAME/$(echo $CI_BUILD_REF | cut -c -7)/$(basename $f); done'
-    - 'for f in `find ace-ims-ear/target -type f -name "*.ear"`; do curl -X POST -H "Authorization: token $BUILD_TOKEN" --data-binary @$f $SERVER/artifacts/ace/$CI_BUILD_REF_NAME/$(echo $CI_BUILD_REF | cut -c -7)/$(basename $f); done'
+    - 'for f in `find ace-am/src/main/sql -type f`; do curl -X POST -H "Content-Type: application/octet-stream" -H "Authorization: $BUILD_TOKEN" --data-binary @$f $SERVER/resource/ace/$CI_BUILD_REF_NAME/$(echo $CI_BUILD_REF | cut -c -7)/$(basename $f); done'
+    - 'for f in `find ace-am/target -type f -name "*.war"`; do curl -X POST -H "Content-Type: application/octet-stream" -H "Authorization: $BUILD_TOKEN" --data-binary @$f $SERVER/resource/ace/$CI_BUILD_REF_NAME/$(echo $CI_BUILD_REF | cut -c -7)/$(basename $f); done'
+    - 'for f in `find ace-ims-ear/target -type f -name "*.ear"`; do curl -X POST -H "Content-Type: application/octet-stream" -H "Authorization: $BUILD_TOKEN" --data-binary @$f $SERVER/resource/ace/$CI_BUILD_REF_NAME/$(echo $CI_BUILD_REF | cut -c -7)/$(basename $f); done'
 
-# release:
-#   only:
-#    - master

ace-am/pom.xml

@@ -4,8 +4,7 @@
     <parent>
         <artifactId>ace</artifactId>
         <groupId>edu.umiacs.ace</groupId>
-        <version>1.13-RELEASE</version>
-        <relativePath>../pom.xml</relativePath>
+        <version>1.14-RELEASE</version>
     </parent>
     <artifactId>ace-am</artifactId>
     <name>ace-am</name>
@@ -21,6 +20,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-war-plugin</artifactId>
+                <version>3.2.2</version>
                 <configuration>
                     <warSourceExcludes>**/footer.jsp</warSourceExcludes>
                     <webResources>
@@ -31,13 +31,6 @@
                                 <include>**/footer.jsp</include>
                             </includes>
                         </resource>
-                        <!--<resource>
-                          <directory>src/main/webapp</directory>
-                          <filtering>false</filtering>
-                          <excludes>
-                            <exclude>**/footer.jsp</exclude>
-                          </excludes>
-                        </resource>-->
                     </webResources>
                 </configuration>
             </plugin>
@@ -111,7 +104,7 @@
             <scope>provided</scope>
         </dependency>
         <dependency>
-            <groupId>jstl</groupId>
+            <groupId>javax.servlet</groupId>
             <artifactId>jstl</artifactId>
             <version>1.1.2</version>
         </dependency>
@@ -202,34 +195,8 @@
             <artifactId>irods-api</artifactId>
             <version>1.6</version>
         </dependency>
-        <!--<dependency>
-            <groupId>org.apache.jackrabbit</groupId>
-            <artifactId>jackrabbit-core</artifactId>
-            <version>2.2.7</version>
-            <exclusions>
-                <exclusion>
-                    <artifactId>slf4j-api</artifactId>
-                    <groupId>org.slf4j</groupId>
-                </exclusion>
-                <exclusion>
-                    <artifactId>jcl-over-slf4j</artifactId>
-                    <groupId>org.slf4j</groupId>
-                </exclusion>
-            </exclusions>
-        </dependency> -->
-        
-        <!-- web services
-        <dependency>
-            <groupId>com.sun.jersey</groupId>
-            <artifactId>jersey-server</artifactId>
-            <version>1.6</version>
-        </dependency>
-        <dependency>
-            <groupId>com.sun.jersey</groupId>
-            <artifactId>jersey-json</artifactId>
-            <version>1.6</version>
-        </dependency>
-        -->
+
+        <!-- web services -->
         <dependency>
             <groupId>org.glassfish.jersey.containers</groupId>
             <artifactId>jersey-container-servlet</artifactId>
@@ -240,12 +207,6 @@
             <artifactId>jersey-media-json-jackson</artifactId>
             <version>2.24</version>
         </dependency>
-        <dependency>
-            <groupId>edu.umiacs.ace</groupId>
-            <artifactId>ace-ims-ws</artifactId>
-            <version>1.13-RELEASE</version>
-            <type>jar</type>
-        </dependency>
         <dependency>
             <groupId>org.apache.httpcomponents</groupId>
             <artifactId>httpclient</artifactId>

ace-am/src/main/java/edu/umiacs/ace/driver/localfile/LocalFileAccess.java

@@ -34,22 +34,23 @@ import edu.umiacs.ace.driver.AuditIterable;
 import edu.umiacs.ace.driver.DriverStateBean;
 import edu.umiacs.ace.driver.DriverStateBean.State;
 import edu.umiacs.ace.driver.FileBean;
-import edu.umiacs.ace.driver.filter.PathFilter;
-import edu.umiacs.ace.driver.StorageDriver;
-import edu.umiacs.ace.monitor.core.MonitoredItem;
-import edu.umiacs.ace.monitor.core.Collection;
 import edu.umiacs.ace.driver.QueryThrottle;
 import edu.umiacs.ace.driver.StateBeanDigestListener;
+import edu.umiacs.ace.driver.StorageDriver;
+import edu.umiacs.ace.driver.filter.PathFilter;
+import edu.umiacs.ace.monitor.core.Collection;
+import edu.umiacs.ace.monitor.core.MonitoredItem;
 import edu.umiacs.ace.util.HashValue;
 import edu.umiacs.ace.util.ThreadedDigestStream;
 import edu.umiacs.ace.util.ThrottledInputStream;
-import edu.umiacs.io.IO;
 import edu.umiacs.util.Strings;
+import org.apache.log4j.Logger;
+
+import javax.persistence.EntityManager;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.security.DigestInputStream;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
@@ -58,12 +59,10 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Queue;
-import javax.persistence.EntityManager;
-import org.apache.log4j.Logger;
 
 /**
  * Storage driver for accessing files stored on a local file system (ie, java.io.File)
- * 
+ *
  * @author toaster
  */
 public class LocalFileAccess extends StorageDriver {
@@ -138,18 +137,19 @@ public class LocalFileAccess extends StorageDriver {
     class MyIterator implements Iterator<FileBean> {
 
         private FileBean next;
-        private Queue<File> dirsToProcess = new LinkedList<File>();
-        private Queue<File> filesToProcess = new LinkedList<File>();
+        private Queue<File> dirsToProcess = new LinkedList<>();
+        private Queue<File> filesToProcess = new LinkedList<>();
         private MessageDigest digest;
-//        private byte[] buffer = new byte[BLOCK_SIZE];
         private File rootFile;
         private PathFilter filter;
         private DriverStateBean statebean;
         private ThreadedDigestStream reader;
         private boolean cancel = false;
 
-        public MyIterator(MonitoredItem[] startPath, PathFilter filter,
-                String digestAlgorithm, DriverStateBean statebean) {
+        public MyIterator(MonitoredItem[] startPath,
+                          PathFilter filter,
+                          String digestAlgorithm,
+                          DriverStateBean statebean) {
             this.statebean = statebean;
             this.filter = filter;
             try {
@@ -163,8 +163,7 @@ public class LocalFileAccess extends StorageDriver {
                 if (startPath != null) {
                     for (MonitoredItem mi : startPath) {
                         File startFile;
-                        startFile = new File(
-                                getCollection().getDirectory() + mi.getPath());
+                        startFile = new File(getCollection().getDirectory() + mi.getPath());
                         if (startFile.isDirectory()) {
                             dirsToProcess.add(startFile);
 
@@ -224,13 +223,13 @@ public class LocalFileAccess extends StorageDriver {
                 if (fileList == null) {
                     LOG.info("Could not read directory, skipping: " + directory);
                 } else {
-                    for (File f : directory.listFiles()) {
+                    for (File f : fileList) {
                         LOG.trace("Found item " + f);
-                        if ( f.isDirectory() && 
+                        if ( f.isDirectory() &&
                              filter.process(extractPathList(f), true)) {
                             LOG.trace("Adding matching directory: " + f);
                             dirsToProcess.add(f);
-                        } else if ( f.isFile() && 
+                        } else if ( f.isFile() &&
                                     filter.process(extractPathList(f), false)) {
                             LOG.trace("Adding matching file: " + f);
                             filesToProcess.add(f);
@@ -254,11 +253,9 @@ public class LocalFileAccess extends StorageDriver {
             int substrLength = rootFile.getPath().length();
 
             // build directory path
-            List<String> dirPathList = new ArrayList<String>();
+            List<String> dirPathList = new ArrayList<>();
             File currFile = file;
             while (!currFile.equals(rootFile)) {
-//                LOG.trace("Adding dir to path: " + currFile.getPath().substring(
-//                        substrLength));
                 String pathToAdd = currFile.getPath().substring(substrLength);
                 pathToAdd = pathToAdd.replace(File.separatorChar, '/');
                 dirPathList.add(pathToAdd);
@@ -269,11 +266,7 @@ public class LocalFileAccess extends StorageDriver {
 
         @SuppressWarnings("empty-statement")
         private FileBean processFile(File file) {
-
-
-            DigestInputStream dis = null;
             FileBean fb = new FileBean();
-
             fb.setPathList(extractPathList(file));
 
             LOG.trace("Processing file: " + file);
@@ -307,7 +300,6 @@ public class LocalFileAccess extends StorageDriver {
                 fb.setError(true);
                 fb.setErrorMessage(Strings.exceptionAsString(ie));
             } finally {
-                IO.release(dis);
                 statebean.setStateAndReset(State.IDLE);
                 if (cancel) {
                     return null;

ace-am/src/main/java/edu/umiacs/ace/monitor/access/CollectionSummaryServlet.java

@@ -36,7 +36,9 @@ import edu.umiacs.ace.util.EntityManagerServlet;
 import org.apache.log4j.Logger;
 
 import javax.persistence.EntityManager;
+import javax.persistence.NoResultException;
 import javax.persistence.Query;
+import javax.persistence.TypedQuery;
 import javax.servlet.RequestDispatcher;
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
@@ -47,7 +49,7 @@ import java.util.List;
 
 /**
  * Show the current status of a collection, listing any non-active files
- * 
+ *
  * @author toaster
  */
 public class CollectionSummaryServlet extends EntityManagerServlet {
@@ -63,15 +65,16 @@ public class CollectionSummaryServlet extends EntityManagerServlet {
     public static final String PAGE_COUNT = "count";
     public static final String PAGE_SESSION = "session";
 
-    /** 
+    /**
      * Processes requests for both HTTP <code>GET</code> and <code>POST</code> methods.
-     * @param request servlet request
+     *
+     * @param request  servlet request
      * @param response servlet response
      */
     @Override
-    protected void processRequest( HttpServletRequest request,
-            HttpServletResponse response, EntityManager em )
-            throws ServletException, IOException {
+    protected void processRequest(HttpServletRequest request,
+                                  HttpServletResponse response,
+                                  EntityManager em) throws ServletException, IOException {
         RequestDispatcher dispatch;
         long startid = getParameter(request, PARAM_START, 0);
         long topid = getParameter(request, PARAM_TOP, 0);
@@ -84,41 +87,47 @@ public class CollectionSummaryServlet extends EntityManagerServlet {
 
         // determine start
         String startquery = "";
-        if ( startid > 0 ) {
+        if (startid > 0) {
             startquery = "AND m.id > :id ";
-        } else if ( topid > 0 ) {
+        } else if (topid > 0) {
             startquery = "AND m.id < :id ";
         }
 
         // build query
 
-        Query q;
+        TypedQuery<MonitoredItem> q;
         Collection c = getCollection(request, em);
+        // todo: if null should we return all error'd items from the db?
+        // probably a bad idea for this servlet
+        if (c == null) {
+            response.sendError(400, "collectionid must be present!");
+            return;
+        }
+
         query += "m.parentCollection = :coll " + startquery;
-        q = em.createQuery(query);
+        q = em.createQuery(query, MonitoredItem.class);
         q.setParameter("coll", c);
 
         LOG.debug("query: " + query);
         // fill in start ids if necessary
-        if ( startid > 0 ) {
+        if (startid > 0) {
             q.setParameter("id", startid);
-        } else if ( topid > 0 ) {
+        } else if (topid > 0) {
             q.setParameter("id", topid);
         }
 
-        if ( count != -1 ) {
+        if (count != -1) {
             q.setMaxResults(count);
         }
         List<MonitoredItem> miList = q.getResultList();
-        if ( miList != null && miList.size() > 0 ) {
-            request.setAttribute(PAGE_NEXT,
-                    miList.get(miList.size() - 1).getId() + 1);
+        if (miList != null && miList.size() > 0) {
+            request.setAttribute(PAGE_NEXT, miList.get(miList.size() - 1).getId() + 1);
         }
 
-        if ( listOnly ) {
+        if (listOnly) {
             Writer writer = response.getWriter();
             response.setContentType("text/plain");
-            for ( MonitoredItem mi : miList ) {
+            for (MonitoredItem mi : miList) {
                 writer.write(mi.getState() + ":" + mi.getPath() + "\r\n");
             }
             return;
@@ -132,8 +141,7 @@ public class CollectionSummaryServlet extends EntityManagerServlet {
         csb.setTotalFiles(CollectionCountContext.getFileCount(c));
         csb.setActiveFiles(CollectionCountContext.getActiveCount(c));
         csb.setCorruptFiles(CollectionCountContext.getCorruptCount(c));
-        csb.setInvalidDigests(CollectionCountContext.getTokenMismatchCount(
-                c));
+        csb.setInvalidDigests(CollectionCountContext.getTokenMismatchCount(c));
         csb.setMissingFiles(CollectionCountContext.getMissingCount(c));
         csb.setMissingTokens(CollectionCountContext.getMissingTokenCount(c));
         csb.setTotalErrors(CollectionCountContext.getTotalErrors(c));
@@ -145,7 +153,7 @@ public class CollectionSummaryServlet extends EntityManagerServlet {
         request.setAttribute(PAGE_COLLECTION, csb);
         request.setAttribute(PAGE_COUNT, count);
 
-        if ( hasJson(request) ) {
+        if (hasJson(request)) {
             dispatch = request.getRequestDispatcher("report-json.jsp");
         } else {
             dispatch = request.getRequestDispatcher("report.jsp");
@@ -160,6 +168,11 @@ public class CollectionSummaryServlet extends EntityManagerServlet {
         Query q = em.createQuery(query);
         q.setParameter("c", c);
         q.setMaxResults(1);
-        return (Long) q.getSingleResult();
+        try {
+            return (Long)