Fixes for 1.14 regressions

ace-am/
  - update jersey dependencies
  - fix redirect for TokenImportStatus
  - catch possible NPE when shutting down the IngestThreadPool
  - start to incorporate the ForkJoin common pool for tasks which need to be run
  - in the AuditThread, if a MonitoredItem has no digest set the storedDigest to be the hash generated
  - in the IngestThread, use a non-locking method when querying for MonitoredItems. The MonitoredItemManager has the potential to lock threads out which have open database transactions, which can cause deadlocks.

ace-am/pom.xml

 <?xml version="1.0"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <parent>
         <artifactId>ace</artifactId>
@@ -72,10 +74,12 @@
         </dependency>
 
         <!-- for json -->
+        <!--
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-core</artifactId>
         </dependency>
+        -->
 
         <dependency>
             <groupId>net.java.dev.rome</groupId>
@@ -185,12 +189,14 @@
         <dependency>
             <groupId>org.glassfish.jersey.containers</groupId>
             <artifactId>jersey-container-servlet</artifactId>
-            <version>2.24</version>
         </dependency>
         <dependency>
             <groupId>org.glassfish.jersey.media</groupId>
             <artifactId>jersey-media-json-jackson</artifactId>
-            <version>2.24</version>
+        </dependency>
+        <dependency>
+            <groupId>org.glassfish.jersey.inject</groupId>
+            <artifactId>jersey-hk2</artifactId>
         </dependency>
         <dependency>
             <groupId>org.apache.httpcomponents</groupId>

ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java

@@ -593,9 +593,12 @@ public final class AuditThread extends Thread implements CancelCallback {
         LOG.trace("Generated checksum: " + currentFileHash + " expected checksum: " + storedDigest);
         LogEvent event;
 
-        // If we have a registered file, set the digested value
+        // If we have a registered file:
+        // set the stored digest to be the generated hash
+        // set the file digest of the current monitored_item to the generated hash
         if (null == storedDigest) {
             LOG.trace("Setting digest for registered file " + item.getPath());
+            storedDigest = currentFileHash;
             item.setFileDigest(currentFileHash);
             item.setLastSeen(new Date());
         }

ace-am/src/main/java/edu/umiacs/ace/monitor/core/TokenMigrationContextListener.java

@@ -35,6 +35,12 @@ import edu.umiacs.ace.monitor.audit.AuditConfigurationContext;
 import edu.umiacs.ace.monitor.audit.AuditConfigurationContext.PauseBean;
 import edu.umiacs.ace.util.PersistUtil;
 import edu.umiacs.sql.SQL;
+import org.apache.log4j.Logger;
+import org.apache.log4j.NDC;
+
+import javax.servlet.ServletContextEvent;
+import javax.servlet.ServletContextListener;
+import javax.sql.DataSource;
 import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.sql.Blob;
@@ -43,11 +49,6 @@ import java.sql.DatabaseMetaData;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import javax.sql.DataSource;
-import javax.servlet.ServletContextEvent;
-import javax.servlet.ServletContextListener;
-import org.apache.log4j.Logger;
-import org.apache.log4j.NDC;
 
 /**
  * Token migration listner to migrate tokens from the older pre-1.6 version to the current 1.6+
@@ -225,7 +226,7 @@ public class TokenMigrationContextListener implements ServletContextListener {
                         + writeStmt.executeBatch().length);
                 writeConn.commit();
 
-                LOG.info("Entried processed: " + i);
+                LOG.info("Entries processed: " + i);
             } catch ( SQLException e ) {
                 writeConn.rollback();
                 LOG.error("SQL Exception while moving tokens");

ace-am/src/main/java/edu/umiacs/ace/monitor/register/IngestStore.java

@@ -139,7 +139,7 @@ public class IngestStore extends EntityManagerServlet {
 
         IngestThreadPool tPool = IngestThreadPool.getInstance();
         tPool.submitTokens(batchTokens, coll);
-        response.sendRedirect("/TokenImportStatus?active=" + coll.getId());
+        response.sendRedirect("TokenImportStatus?active=" + coll.getId());
     }
 
 }

ace-am/src/main/java/edu/umiacs/ace/monitor/register/IngestSupervisor.java

@@ -33,7 +33,7 @@ public class IngestSupervisor implements Runnable {
     public IngestSupervisor(final Map<String, Token> tokens, final Collection coll) {
         this.tokens = tokens;
         this.coll = coll;
-        this.pool = new ForkJoinPool();
+        this.pool = ForkJoinPool.commonPool();
         this.states = new ConcurrentHashMap<>();
 
         // so we don't have to worry about npes
@@ -58,7 +58,7 @@ public class IngestSupervisor implements Runnable {
 
         dirTask.quietlyJoin();
         fileTask.quietlyJoin();
-        pool.shutdown();
+        // pool.shutdown();
         LOG.info("Leaving Supervisor");
     }
 

ace-am/src/main/java/edu/umiacs/ace/monitor/register/IngestThread.java

@@ -32,7 +32,6 @@ package edu.umiacs.ace.monitor.register;
 
 import edu.umiacs.ace.monitor.core.Collection;
 import edu.umiacs.ace.monitor.core.MonitoredItem;
-import edu.umiacs.ace.monitor.core.MonitoredItemManager;
 import edu.umiacs.ace.monitor.core.Token;
 import edu.umiacs.ace.monitor.log.LogEnum;
 import edu.umiacs.ace.monitor.log.LogEvent;
@@ -45,6 +44,9 @@ import org.apache.log4j.Logger;
 
 import javax.persistence.EntityManager;
 import javax.persistence.EntityTransaction;
+import javax.persistence.TypedQuery;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.Date;
 import java.util.LinkedList;
 import java.util.List;
@@ -93,7 +95,7 @@ public class IngestThread extends RecursiveAction {
             return;
         }
 
-        if (identifiers.size() < 7000) {
+        if (identifiers.size() < 2500) {
             // TODO: I still want to play around with rollbacks in case of failure
             em = PersistUtil.getEntityManager();
             EntityTransaction trans = em.getTransaction();
@@ -115,7 +117,6 @@ public class IngestThread extends RecursiveAction {
         MonitoredItem item;
         session = System.currentTimeMillis();
         logManager = new LogEventManager(session, coll);
-        MonitoredItemManager mim = new MonitoredItemManager(em);
 
         // Cycle through all items read in and add/update tokens
         // Commit only if there are no errors in all transactions
@@ -123,7 +124,7 @@ public class IngestThread extends RecursiveAction {
         for (String identifier : identifiers) {
             queued.remove(identifier);
             Token token = tokens.get(identifier);
-            item = mim.getItemByPath(identifier, coll);
+            item = getItemByPath(identifier, coll);
             if (item == null) {
                 LOG.debug("[Ingest Thread " + Thread.currentThread().getId()
                         + "] Adding new item " + identifier);
@@ -256,4 +257,28 @@ public class IngestThread extends RecursiveAction {
         mi.setSize(size);
         return mi;
     }
+
+
+    /**
+     * From MIM but without locking
+     *
+     * @param path path to look for
+     * @return MonitoredItem if exist, null otherwise
+     */
+    public MonitoredItem getItemByPath(String path, Collection c) {
+        MonitoredItem tmp = new MonitoredItem();
+        tmp.setPath(path);
+
+        TypedQuery<MonitoredItem> q =
+                em.createNamedQuery("MonitoredItem.getItemByPath", MonitoredItem.class);
+        q.setParameter("path", path);
+        q.setParameter("coll", c);
+
+        // Make sure we have the correct item
+        List<MonitoredItem> li = q.getResultList();
+        li.sort(Comparator.comparing(MonitoredItem::getPath));
+        int idx = Collections.binarySearch(li, tmp, Comparator.comparing(MonitoredItem::getPath));
+        return (idx >= 0 ? li.get(idx) : null);
+    }
+
 }
\ No newline at end of file

ace-am/src/main/java/edu/umiacs/ace/monitor/register/IngestThreadPool.java

@@ -141,7 +141,9 @@ public class IngestThreadPool {
 
     protected static void shutdownPools() {
         LOG.debug("[Ingest] Shutting down thread pools.");
-        cache.invalidateAll();
+        if (cache != null) {
+            cache.invalidateAll();
+        }
     }
 
 }

ace-am/src/main/webapp/collectionremove.jsp

@@ -7,20 +7,29 @@ Author     : toaster
 <%@page contentType="text/html" pageEncoding="UTF-8"%>
 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
 "http://www.w3.org/TR/html4/loose.dtd">
-<%@taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"%> 
+<%@taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"%>
 <html>
     <head>
         <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-        <title>Replica Added</title>
+        <title>Collection Remove</title>
         <link rel="stylesheet" type="text/css" href="style.css" />
     </head>
     <body>
         <jsp:include page="header.jsp" />
         <div class="standardBody">
         <h3>Confirm Remove</h3>
-        All replica, file, and log information for the collection ${workingCollection.collection.name} will be removed forever. Click Remove to proceed.
-        
-        <BR><a href="ManageCollection?remove=yes&collectionid=${workingCollection.collection.id}">Remove</a>&nbsp;&nbsp;&nbsp;<a href="Status">Back</a>
+            <p>
+                All replica, file, and log information for the collection
+                ${workingCollection.collection.name} will be removed forever. This process can take
+                time for larger collections, which may continue to appear until the remove is
+                complete.
+            </p>
+
+            <p>Click Remove to proceed.</p>
+
+        <a href="ManageCollection?remove=yes&collectionid=${workingCollection.collection.id}">
+            Remove</a>&nbsp;&nbsp;&nbsp;
+        <a href="Status">Back</a>
     </div>
         <jsp:include page="footer.jsp" />
 

pom.xml

@@ -57,14 +57,13 @@
         <!-- dependency bom -->
         <jaxb.version>2.3.1</jaxb.version>
         <jaxws.version>2.3.1</jaxws.version>
-        <jersey.version>2.24</jersey.version>
+        <jersey.version>2.28</jersey.version>
         <guava.version>28.0-jre</guava.version>
         <commons-io.version>2.6</commons-io.version>
         <commons-fu.version>1.4</commons-fu.version>
         <commons-csv.version>1.6</commons-csv.version>
         <commons-collections.version>4.3</commons-collections.version>
         <log4j2.version>2.11.2</log4j2.version>
-        <jackson.version>2.9.9</jackson.version>
         <eclipselink.version>2.7.4</eclipselink.version>
     </properties>
 
@@ -132,16 +131,6 @@
             </dependency>
 
             <!-- audit manager misc -->
-            <dependency>
-                <groupId>com.fasterxml.jackson.core</groupId>
-                <artifactId>jackson-databind</artifactId>
-                <version>${jackson.version}</version>
-            </dependency>
-            <dependency>
-                <groupId>com.fasterxml.jackson.core</groupId>
-                <artifactId>jackson-core</artifactId>
-                <version>${jackson.version}</version>
-            </dependency>
             <dependency>
                 <groupId>com.sun.mail</groupId>
                 <artifactId>javax.mail</artifactId>
@@ -153,6 +142,23 @@
                 <version>1.2.5</version>
             </dependency>
 
+            <!-- audit manager jersey -->
+            <dependency>
+                <groupId>org.glassfish.jersey.containers</groupId>
+                <artifactId>jersey-container-servlet</artifactId>
+                <version>${jersey.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.glassfish.jersey.media</groupId>
+                <artifactId>jersey-media-json-jackson</artifactId>
+                <version>${jersey.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.glassfish.jersey.inject</groupId>
+                <artifactId>jersey-hk2</artifactId>
+                <version>${jersey.version}</version>
+            </dependency>
+
             <!-- apache commons + guava -->
             <dependency>
                 <groupId>org.apache.commons</groupId>