Commit 7864362c authored by Michael Ritter's avatar Michael Ritter
Browse files

Merge branch 'release-1.11'

parents b2f5ecb3 1c7c40be
...@@ -4,10 +4,9 @@ ...@@ -4,10 +4,9 @@
<parent> <parent>
<artifactId>ace</artifactId> <artifactId>ace</artifactId>
<groupId>edu.umiacs.ace</groupId> <groupId>edu.umiacs.ace</groupId>
<version>1.10</version> <version>1.11-RELEASE</version>
<relativePath>../pom.xml</relativePath> <relativePath>../pom.xml</relativePath>
</parent> </parent>
<groupId>edu.umiacs.ace</groupId>
<artifactId>ace-am</artifactId> <artifactId>ace-am</artifactId>
<name>ace-am</name> <name>ace-am</name>
<packaging>war</packaging> <packaging>war</packaging>
...@@ -102,12 +101,12 @@ ...@@ -102,12 +101,12 @@
<dependency> <dependency>
<groupId>org.eclipse.persistence</groupId> <groupId>org.eclipse.persistence</groupId>
<artifactId>eclipselink</artifactId> <artifactId>eclipselink</artifactId>
<version>2.2.0</version> <version>2.6.3</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.eclipse.persistence</groupId> <groupId>org.eclipse.persistence</groupId>
<artifactId>javax.persistence</artifactId> <artifactId>org.eclipse.persistence.jpa</artifactId>
<version>2.0.3</version> <version>2.6.3</version>
</dependency> </dependency>
<dependency> <dependency>
...@@ -118,9 +117,9 @@ ...@@ -118,9 +117,9 @@
<!-- for json --> <!-- for json -->
<dependency> <dependency>
<groupId>org.codehaus.jackson</groupId> <groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-mapper-asl</artifactId> <artifactId>jackson-core</artifactId>
<version>1.8.1</version> <version>2.8.1</version>
</dependency> </dependency>
<dependency> <dependency>
...@@ -178,6 +177,12 @@ ...@@ -178,6 +177,12 @@
<version>3.1</version> <version>3.1</version>
</dependency> </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
<version>1.4</version>
</dependency>
<dependency> <dependency>
<groupId>com.google.guava</groupId> <groupId>com.google.guava</groupId>
<artifactId>guava</artifactId> <artifactId>guava</artifactId>
...@@ -238,7 +243,7 @@ ...@@ -238,7 +243,7 @@
</exclusions> </exclusions>
</dependency> --> </dependency> -->
<!-- web services --> <!-- web services
<dependency> <dependency>
<groupId>com.sun.jersey</groupId> <groupId>com.sun.jersey</groupId>
<artifactId>jersey-server</artifactId> <artifactId>jersey-server</artifactId>
...@@ -249,10 +254,21 @@ ...@@ -249,10 +254,21 @@
<artifactId>jersey-json</artifactId> <artifactId>jersey-json</artifactId>
<version>1.6</version> <version>1.6</version>
</dependency> </dependency>
-->
<dependency>
<groupId>org.glassfish.jersey.containers</groupId>
<artifactId>jersey-container-servlet</artifactId>
<version>2.24</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-json-jackson</artifactId>
<version>2.24</version>
</dependency>
<dependency> <dependency>
<groupId>edu.umiacs.ace</groupId> <groupId>edu.umiacs.ace</groupId>
<artifactId>ace-ims-ws</artifactId> <artifactId>ace-ims-ws</artifactId>
<version>1.10</version> <version>1.11-RELEASE</version>
<type>jar</type> <type>jar</type>
</dependency> </dependency>
<dependency> <dependency>
......
...@@ -83,7 +83,6 @@ public class FileBean { ...@@ -83,7 +83,6 @@ public class FileBean {
* [0] /1k-10/d7/file-19 * [0] /1k-10/d7/file-19
* [1] /1k-10/d7 * [1] /1k-10/d7
* [2] /1k-10 * [2] /1k-10
* @param parentPathList
*/ */
public void setPathList( String[] pathList ) { public void setPathList( String[] pathList ) {
if ( pathList != null ) { if ( pathList != null ) {
......
...@@ -33,16 +33,17 @@ package edu.umiacs.ace.driver; ...@@ -33,16 +33,17 @@ package edu.umiacs.ace.driver;
import edu.umiacs.ace.driver.benchmark.BenchmarkAccess; import edu.umiacs.ace.driver.benchmark.BenchmarkAccess;
import edu.umiacs.ace.driver.irods.IrodsAccess; import edu.umiacs.ace.driver.irods.IrodsAccess;
import edu.umiacs.ace.driver.localfile.LocalFileAccess; import edu.umiacs.ace.driver.localfile.LocalFileAccess;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.driver.srb.SrbAccess; import edu.umiacs.ace.driver.srb.SrbAccess;
import edu.umiacs.ace.driver.swap.SwapFileAccess; import edu.umiacs.ace.driver.swap.SwapFileAccess;
import edu.umiacs.ace.monitor.core.Collection;
import javax.persistence.EntityManager;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator; import java.util.Comparator;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import javax.persistence.EntityManager;
/** /**
* Factory to register all storage resources supported by this monitor. * Factory to register all storage resources supported by this monitor.
...@@ -56,7 +57,7 @@ public final class StorageDriverFactory { ...@@ -56,7 +57,7 @@ public final class StorageDriverFactory {
* Map to track descriptive storage type name to implementing class * Map to track descriptive storage type name to implementing class
*/ */
private static final Map<String, Class<? extends StorageDriver>> implementationMap = private static final Map<String, Class<? extends StorageDriver>> implementationMap =
new HashMap<String, Class<? extends StorageDriver>>(); new HashMap<>();
/** /**
* All available storage gateways must be registered here * All available storage gateways must be registered here
...@@ -71,7 +72,7 @@ public final class StorageDriverFactory { ...@@ -71,7 +72,7 @@ public final class StorageDriverFactory {
} }
public static final List<String> listResources() { public static final List<String> listResources() {
List<String> lt = new ArrayList<String>(implementationMap.keySet()); List<String> lt = new ArrayList<>(implementationMap.keySet());
Collections.sort(lt,comp); Collections.sort(lt,comp);
return lt; return lt;
} }
......
...@@ -36,11 +36,12 @@ import edu.umiacs.ace.driver.filter.PathFilter; ...@@ -36,11 +36,12 @@ import edu.umiacs.ace.driver.filter.PathFilter;
import edu.umiacs.ace.util.HashValue; import edu.umiacs.ace.util.HashValue;
import edu.umiacs.irods.operation.BulkFileHandler; import edu.umiacs.irods.operation.BulkFileHandler;
import edu.umiacs.srb.util.StringUtil; import edu.umiacs.srb.util.StringUtil;
import org.apache.log4j.Logger;
import java.io.IOException; import java.io.IOException;
import java.security.MessageDigest; import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException; import java.security.NoSuchAlgorithmException;
import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.LinkedBlockingQueue;
import org.apache.log4j.Logger;
/** /**
* *
...@@ -61,7 +62,6 @@ public class IrodsHandler implements BulkFileHandler { ...@@ -61,7 +62,6 @@ public class IrodsHandler implements BulkFileHandler {
* @param readyList * @param readyList
* @param filter * @param filter
* @param digestAlgorithm * @param digestAlgorithm
* @param base prefix to add to returned files (no trailing /)
*/ */
public IrodsHandler( LinkedBlockingQueue<FileBean> readyList, public IrodsHandler( LinkedBlockingQueue<FileBean> readyList,
PathFilter filter, String digestAlgorithm ) { PathFilter filter, String digestAlgorithm ) {
......
...@@ -133,7 +133,6 @@ public class SwapSettings implements Serializable { ...@@ -133,7 +133,6 @@ public class SwapSettings implements Serializable {
} }
/** /**
* @param server the server to set
*/ */
public void setServers( String servers ) { public void setServers( String servers ) {
this.servers = servers; this.servers = servers;
......
...@@ -32,8 +32,8 @@ package edu.umiacs.ace.monitor.access; ...@@ -32,8 +32,8 @@ package edu.umiacs.ace.monitor.access;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import edu.umiacs.ace.monitor.core.Collection; import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.support.PageBean;
import edu.umiacs.ace.monitor.support.CStateBean; import edu.umiacs.ace.monitor.support.CStateBean;
import edu.umiacs.ace.monitor.support.PageBean;
import edu.umiacs.ace.util.EntityManagerServlet; import edu.umiacs.ace.util.EntityManagerServlet;
import edu.umiacs.util.Strings; import edu.umiacs.util.Strings;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
...@@ -101,11 +101,13 @@ public class StatusServlet extends EntityManagerServlet { ...@@ -101,11 +101,13 @@ public class StatusServlet extends EntityManagerServlet {
long page = getParameter(request, PARAM_PAGE, DEFAULT_PAGE); long page = getParameter(request, PARAM_PAGE, DEFAULT_PAGE);
int count = (int) getParameter(request, PARAM_COUNT, DEFAULT_COUNT); int count = (int) getParameter(request, PARAM_COUNT, DEFAULT_COUNT);
// local getParameter so that the session is checked as well
String group = getParameter(request, PARAM_GROUP, null); String group = getParameter(request, PARAM_GROUP, null);
String collection = getParameter(request, PARAM_COLLECTION_LIKE, null); String collection = getParameter(request, PARAM_COLLECTION_LIKE, null);
String state = getParameter(request, PARAM_STATE, null); String state = getParameter(request, PARAM_STATE, null);
// String date = getParameter(request, PARAM_GROUP, null); // String date = getParameter(request, PARAM_GROUP, null);
PageBean pb = new PageBean((int) page, count, "Status"); PageBean pb = new PageBean((int) page, count, "");
long offset = page * count; long offset = page * count;
...@@ -120,20 +122,24 @@ public class StatusServlet extends EntityManagerServlet { ...@@ -120,20 +122,24 @@ public class StatusServlet extends EntityManagerServlet {
List<String> queries = new ArrayList<>(); List<String> queries = new ArrayList<>();
// TODO: Can probably tidy this up a bit
if (!Strings.isEmpty(group)) { if (!Strings.isEmpty(group)) {
queries.add("c.group LIKE :group"); queries.add("c.group LIKE :group");
pb.addParam(PARAM_GROUP, group); pb.addParam(PARAM_GROUP, group);
request.setAttribute(PARAM_GROUP, group);
} }
if (!Strings.isEmpty(collection)) { if (!Strings.isEmpty(collection)) {
queries.add("c.name LIKE :collection"); queries.add("c.name LIKE :collection");
pb.addParam(PARAM_COLLECTION_LIKE, collection); pb.addParam(PARAM_COLLECTION_LIKE, collection);
request.setAttribute(PARAM_COLLECTION_LIKE, collection);
} }
// Enforce that the state is not empty, or larger than 1 character // Enforce that the state is not empty, or larger than 1 character
if (!Strings.isEmpty(state) && state.length() == 1) { if (!Strings.isEmpty(state) && state.length() == 1) {
queries.add("c.state = :state"); queries.add("c.state = :state");
pb.addParam(PARAM_STATE, state); pb.addParam(PARAM_STATE, state);
request.setAttribute(PARAM_STATE, state);
} }
queryString.append("SELECT c FROM Collection c"); queryString.append("SELECT c FROM Collection c");
...@@ -166,6 +172,7 @@ public class StatusServlet extends EntityManagerServlet { ...@@ -166,6 +172,7 @@ public class StatusServlet extends EntityManagerServlet {
Query countQuery = em.createQuery(countString.toString()); Query countQuery = em.createQuery(countString.toString());
// TODO: Can probably tidy this up a bit
if (!Strings.isEmpty(group)) { if (!Strings.isEmpty(group)) {
query.setParameter(PARAM_GROUP, "%" + group + "%"); query.setParameter(PARAM_GROUP, "%" + group + "%");
countQuery.setParameter(PARAM_GROUP, "%" + group + "%"); countQuery.setParameter(PARAM_GROUP, "%" + group + "%");
...@@ -191,8 +198,29 @@ public class StatusServlet extends EntityManagerServlet { ...@@ -191,8 +198,29 @@ public class StatusServlet extends EntityManagerServlet {
pb.update(totalResults); pb.update(totalResults);
} }
setWorkingCollection(request, em);
collections = new ArrayList<>(); collections = new ArrayList<>();
for (Collection col : items) {
CollectionSummaryBean csb = createCollectionSummary(col);
collections.add(csb);
}
request.setAttribute(PAGE_COLLECTIONS, collections);
request.setAttribute(PAGE_STATES, ImmutableList.copyOf(CStateBean.values()));
request.setAttribute(PAGE_COUNT, count);
request.setAttribute(PAGE_NUMBER, pb);
if (hasJson(request)) {
dispatcher = request.getRequestDispatcher("status-json.jsp");
} else if (hasCsv(request)) {
dispatcher = request.getRequestDispatcher("status-csv.jsp");
} else {
dispatcher = request.getRequestDispatcher("status.jsp");
}
dispatcher.forward(request, response);
}
private void setWorkingCollection(HttpServletRequest request, EntityManager em) {
long collectionId; long collectionId;
String idParam = request.getParameter(PARAM_COLLECTION_ID); String idParam = request.getParameter(PARAM_COLLECTION_ID);
collectionId = Strings.isValidLong(idParam) ? Long.parseLong(idParam) : -1; collectionId = Strings.isValidLong(idParam) ? Long.parseLong(idParam) : -1;
...@@ -201,7 +229,6 @@ public class StatusServlet extends EntityManagerServlet { ...@@ -201,7 +229,6 @@ public class StatusServlet extends EntityManagerServlet {
if (Strings.isValidLong(idParam) && -1 == collectionId) { if (Strings.isValidLong(idParam) && -1 == collectionId) {
// clear the working collection // clear the working collection
request.getSession().removeAttribute(SESSION_WORKINGCOLLECTION); request.getSession().removeAttribute(SESSION_WORKINGCOLLECTION);
} else if (Strings.isValidLong(idParam) // valid param } else if (Strings.isValidLong(idParam) // valid param
&& (workingCollectionBean == null // no working collection && (workingCollectionBean == null // no working collection
|| !workingCollectionBean.getCollection().getId().equals(collectionId))) { // or one which does not equal our requested collection || !workingCollectionBean.getCollection().getId().equals(collectionId))) { // or one which does not equal our requested collection
...@@ -216,25 +243,6 @@ public class StatusServlet extends EntityManagerServlet { ...@@ -216,25 +243,6 @@ public class StatusServlet extends EntityManagerServlet {
// Continue using the current working collection // Continue using the current working collection
request.getSession().setAttribute(SESSION_WORKINGCOLLECTION, workingCollectionBean); request.getSession().setAttribute(SESSION_WORKINGCOLLECTION, workingCollectionBean);
} }
for (Collection col : items) {
CollectionSummaryBean csb = createCollectionSummary(col);
collections.add(csb);
}
request.setAttribute(PAGE_COLLECTIONS, collections);
request.setAttribute(PAGE_STATES, ImmutableList.copyOf(CStateBean.values()));
request.setAttribute(PAGE_COUNT, count);
request.setAttribute(PAGE_NUMBER, pb);
if (hasJson(request)) {
dispatcher = request.getRequestDispatcher("status-json.jsp");
} else if (hasCsv(request)) {
dispatcher = request.getRequestDispatcher("status-csv.jsp");
} else {
dispatcher = request.getRequestDispatcher("status.jsp");
}
dispatcher.forward(request, response);
} }
private CollectionSummaryBean createCollectionSummary(Collection col) { private CollectionSummaryBean createCollectionSummary(Collection col) {
...@@ -255,7 +263,23 @@ public class StatusServlet extends EntityManagerServlet { ...@@ -255,7 +263,23 @@ public class StatusServlet extends EntityManagerServlet {
} }
private boolean hasCsv(HttpServletRequest request) { private boolean hasCsv(HttpServletRequest request) {
String value = (String) request.getParameter(PARAM_CSV); String value = request.getParameter(PARAM_CSV);
return !Strings.isEmpty(value); return !Strings.isEmpty(value);
} }
@Override
public String getParameter(HttpServletRequest request, String paramName, String defaultValue) {
String requestParam = request.getParameter(paramName);
// We could remove the session attributes here, not sure if that's a good idea though
// req param: not null and not empty -> use
// session attr not null and req param null -> use
// session attr not null and req param empty -> default
if (!Strings.isEmpty(requestParam)) {
return requestParam;
} else {
return defaultValue;
}
}
} }
...@@ -32,48 +32,54 @@ package edu.umiacs.ace.monitor.audit; ...@@ -32,48 +32,54 @@ package edu.umiacs.ace.monitor.audit;
import edu.umiacs.ace.driver.AuditIterable; import edu.umiacs.ace.driver.AuditIterable;
import edu.umiacs.ace.driver.DriverStateBean; import edu.umiacs.ace.driver.DriverStateBean;
import edu.umiacs.ace.driver.filter.PathFilter;
import edu.umiacs.ace.driver.FileBean; import edu.umiacs.ace.driver.FileBean;
import edu.umiacs.ace.driver.StorageDriver; import edu.umiacs.ace.driver.StorageDriver;
import edu.umiacs.ace.driver.filter.PathFilter;
import edu.umiacs.ace.driver.filter.SimpleFilter;
import edu.umiacs.ace.ims.api.IMSException; import edu.umiacs.ace.ims.api.IMSException;
import edu.umiacs.ace.ims.api.IMSService; import edu.umiacs.ace.ims.api.IMSService;
import edu.umiacs.ace.ims.api.TokenRequestBatch; import edu.umiacs.ace.ims.api.TokenRequestBatch;
import edu.umiacs.ace.ims.api.TokenValidator; import edu.umiacs.ace.ims.api.TokenValidator;
import edu.umiacs.ace.ims.ws.TokenRequest; import edu.umiacs.ace.ims.ws.TokenRequest;
import edu.umiacs.ace.monitor.access.CollectionCountContext; import edu.umiacs.ace.monitor.access.CollectionCountContext;
import edu.umiacs.ace.monitor.register.IngestThreadPool;
import edu.umiacs.ace.util.PersistUtil;
import edu.umiacs.ace.driver.filter.SimpleFilter;
import edu.umiacs.ace.monitor.compare.CollectionCompare2; import edu.umiacs.ace.monitor.compare.CollectionCompare2;
import edu.umiacs.ace.monitor.compare.CompareResults; import edu.umiacs.ace.monitor.compare.CompareResults;
import edu.umiacs.ace.remote.JsonGateway; import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.core.ConfigConstants;
import edu.umiacs.ace.monitor.core.MonitoredItem; import edu.umiacs.ace.monitor.core.MonitoredItem;
import edu.umiacs.ace.monitor.core.MonitoredItemManager; import edu.umiacs.ace.monitor.core.MonitoredItemManager;
import edu.umiacs.ace.monitor.log.LogEnum;
import edu.umiacs.ace.monitor.log.LogEvent;
import edu.umiacs.ace.monitor.log.LogEventManager; import edu.umiacs.ace.monitor.log.LogEventManager;
import edu.umiacs.ace.monitor.peers.PeerCollection;
import edu.umiacs.ace.monitor.reporting.ReportSummary; import edu.umiacs.ace.monitor.reporting.ReportSummary;
import edu.umiacs.ace.monitor.reporting.SchedulerContextListener; import edu.umiacs.ace.monitor.reporting.SchedulerContextListener;
import edu.umiacs.ace.monitor.reporting.SummaryGenerator; import edu.umiacs.ace.monitor.reporting.SummaryGenerator;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.core.ConfigConstants;
import edu.umiacs.ace.monitor.settings.SettingsUtil; import edu.umiacs.ace.monitor.settings.SettingsUtil;
import edu.umiacs.ace.monitor.log.LogEnum; import edu.umiacs.ace.remote.JsonGateway;
import edu.umiacs.ace.monitor.log.LogEvent;
import edu.umiacs.ace.monitor.peers.PeerCollection;
import edu.umiacs.ace.token.AceToken; import edu.umiacs.ace.token.AceToken;
import edu.umiacs.ace.util.PersistUtil;
import edu.umiacs.ace.util.TokenUtil; import edu.umiacs.ace.util.TokenUtil;
import edu.umiacs.util.Strings; import edu.umiacs.util.Strings;
import org.apache.log4j.Logger;
import org.apache.log4j.NDC;
import javax.mail.MessagingException;
import javax.persistence.EntityManager;
import javax.persistence.EntityTransaction;
import javax.persistence.Query;
import javax.sql.DataSource;
import java.io.InputStream; import java.io.InputStream;
import java.security.MessageDigest; import java.security.MessageDigest;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.Arrays; import java.util.Arrays;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import javax.mail.MessagingException;
import javax.persistence.EntityManager;
import javax.persistence.EntityTransaction;
import org.apache.log4j.Logger;
import org.apache.log4j.NDC;
/** /**
* *
...@@ -191,16 +197,6 @@ public final class AuditThread extends Thread implements CancelCallback { ...@@ -191,16 +197,6 @@ public final class AuditThread extends Thread implements CancelCallback {
iterableItems.cancel(); iterableItems.cancel();
} }
/*
if (batch != null) {
batch.close();
}
if (validator != null) {
validator.close();
}
*/
if (AuditThreadFactory.isRunning(coll) || AuditThreadFactory.isQueued(coll)) { if (AuditThreadFactory.isRunning(coll) || AuditThreadFactory.isQueued(coll)) {
AuditThreadFactory.finished(coll); AuditThreadFactory.finished(coll);
} }
...@@ -222,12 +218,6 @@ public final class AuditThread extends Thread implements CancelCallback { ...@@ -222,12 +218,6 @@ public final class AuditThread extends Thread implements CancelCallback {
session = System.currentTimeMillis(); session = System.currentTimeMillis();
logManager = new LogEventManager(session, coll); logManager = new LogEventManager(session, coll);
logAuditStart(); logAuditStart();
/*
while (IngestThreadPool.isIngesting(coll)) {
LOG.debug("Waiting for ingest to finish");
Thread.sleep(500);
}
*/
callback = new FileAuditCallback(coll, session, this); callback = new FileAuditCallback(coll, session, this);
boolean auditTokens = SettingsUtil.getBoolean(coll, boolean auditTokens = SettingsUtil.getBoolean(coll,
...@@ -254,7 +244,7 @@ public final class AuditThread extends Thread implements CancelCallback { ...@@ -254,7 +244,7 @@ public final class AuditThread extends Thread implements CancelCallback {
// Let outstanding tokens finish, TODO, de-hackify this. // Let outstanding tokens finish, TODO, de-hackify this.
sleep(2000); sleep(2000);
} catch (Throwable e) { } catch (Throwable e) {
LOG.fatal("UNcaught exception in performAudit()", e); LOG.fatal("Uncaught exception in performAudit()", e);
if (abortException != null) { if (abortException != null) {
abortException = e; abortException = e;
} }
...@@ -595,7 +585,6 @@ public final class AuditThread extends Thread implements CancelCallback { ...@@ -595,7 +585,6 @@ public final class AuditThread extends Thread implements CancelCallback {
} }
TokenRequest request = new TokenRequest(); TokenRequest request = new TokenRequest();
//request.setName(item.getPath());
request.setName(item.getId().toString()); request.