Commit 7864362c authored by Michael Ritter's avatar Michael Ritter

Merge branch 'release-1.11'

parents b2f5ecb3 1c7c40be
......@@ -4,10 +4,9 @@
<parent>
<artifactId>ace</artifactId>
<groupId>edu.umiacs.ace</groupId>
<version>1.10</version>
<version>1.11-RELEASE</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>edu.umiacs.ace</groupId>
<artifactId>ace-am</artifactId>
<name>ace-am</name>
<packaging>war</packaging>
......@@ -102,12 +101,12 @@
<dependency>
<groupId>org.eclipse.persistence</groupId>
<artifactId>eclipselink</artifactId>
<version>2.2.0</version>
<version>2.6.3</version>
</dependency>
<dependency>
<groupId>org.eclipse.persistence</groupId>
<artifactId>javax.persistence</artifactId>
<version>2.0.3</version>
<artifactId>org.eclipse.persistence.jpa</artifactId>
<version>2.6.3</version>
</dependency>
<dependency>
......@@ -118,9 +117,9 @@
<!-- for json -->
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
<version>1.8.1</version>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.8.1</version>
</dependency>
<dependency>
......@@ -178,6 +177,12 @@
<version>3.1</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
<version>1.4</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
......@@ -238,7 +243,7 @@
</exclusions>
</dependency> -->
<!-- web services -->
<!-- web services
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-server</artifactId>
......@@ -249,10 +254,21 @@
<artifactId>jersey-json</artifactId>
<version>1.6</version>
</dependency>
-->
<dependency>
<groupId>org.glassfish.jersey.containers</groupId>
<artifactId>jersey-container-servlet</artifactId>
<version>2.24</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-json-jackson</artifactId>
<version>2.24</version>
</dependency>
<dependency>
<groupId>edu.umiacs.ace</groupId>
<artifactId>ace-ims-ws</artifactId>
<version>1.10</version>
<version>1.11-RELEASE</version>
<type>jar</type>
</dependency>
<dependency>
......
......@@ -83,7 +83,6 @@ public class FileBean {
* [0] /1k-10/d7/file-19
* [1] /1k-10/d7
* [2] /1k-10
* @param parentPathList
*/
public void setPathList( String[] pathList ) {
if ( pathList != null ) {
......
......@@ -33,16 +33,17 @@ package edu.umiacs.ace.driver;
import edu.umiacs.ace.driver.benchmark.BenchmarkAccess;
import edu.umiacs.ace.driver.irods.IrodsAccess;
import edu.umiacs.ace.driver.localfile.LocalFileAccess;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.driver.srb.SrbAccess;
import edu.umiacs.ace.driver.swap.SwapFileAccess;
import edu.umiacs.ace.monitor.core.Collection;
import javax.persistence.EntityManager;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.persistence.EntityManager;
/**
* Factory to register all storage resources supported by this monitor.
......@@ -56,7 +57,7 @@ public final class StorageDriverFactory {
* Map to track descriptive storage type name to implementing class
*/
private static final Map<String, Class<? extends StorageDriver>> implementationMap =
new HashMap<String, Class<? extends StorageDriver>>();
new HashMap<>();
/**
* All available storage gateways must be registered here
......@@ -71,7 +72,7 @@ public final class StorageDriverFactory {
}
public static final List<String> listResources() {
List<String> lt = new ArrayList<String>(implementationMap.keySet());
List<String> lt = new ArrayList<>(implementationMap.keySet());
Collections.sort(lt,comp);
return lt;
}
......
......@@ -36,11 +36,12 @@ import edu.umiacs.ace.driver.filter.PathFilter;
import edu.umiacs.ace.util.HashValue;
import edu.umiacs.irods.operation.BulkFileHandler;
import edu.umiacs.srb.util.StringUtil;
import org.apache.log4j.Logger;
import java.io.IOException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.concurrent.LinkedBlockingQueue;
import org.apache.log4j.Logger;
/**
*
......@@ -61,7 +62,6 @@ public class IrodsHandler implements BulkFileHandler {
* @param readyList
* @param filter
* @param digestAlgorithm
* @param base prefix to add to returned files (no trailing /)
*/
public IrodsHandler( LinkedBlockingQueue<FileBean> readyList,
PathFilter filter, String digestAlgorithm ) {
......
......@@ -133,7 +133,6 @@ public class SwapSettings implements Serializable {
}
/**
* @param server the server to set
*/
public void setServers( String servers ) {
this.servers = servers;
......
......@@ -32,8 +32,8 @@ package edu.umiacs.ace.monitor.access;
import com.google.common.collect.ImmutableList;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.support.PageBean;
import edu.umiacs.ace.monitor.support.CStateBean;
import edu.umiacs.ace.monitor.support.PageBean;
import edu.umiacs.ace.util.EntityManagerServlet;
import edu.umiacs.util.Strings;
import org.apache.log4j.Logger;
......@@ -101,11 +101,13 @@ public class StatusServlet extends EntityManagerServlet {
long page = getParameter(request, PARAM_PAGE, DEFAULT_PAGE);
int count = (int) getParameter(request, PARAM_COUNT, DEFAULT_COUNT);
// local getParameter so that the session is checked as well
String group = getParameter(request, PARAM_GROUP, null);
String collection = getParameter(request, PARAM_COLLECTION_LIKE, null);
String state = getParameter(request, PARAM_STATE, null);
// String date = getParameter(request, PARAM_GROUP, null);
PageBean pb = new PageBean((int) page, count, "Status");
PageBean pb = new PageBean((int) page, count, "");
long offset = page * count;
......@@ -120,20 +122,24 @@ public class StatusServlet extends EntityManagerServlet {
List<String> queries = new ArrayList<>();
// TODO: Can probably tidy this up a bit
if (!Strings.isEmpty(group)) {
queries.add("c.group LIKE :group");
pb.addParam(PARAM_GROUP, group);
request.setAttribute(PARAM_GROUP, group);
}
if (!Strings.isEmpty(collection)) {
queries.add("c.name LIKE :collection");
pb.addParam(PARAM_COLLECTION_LIKE, collection);
request.setAttribute(PARAM_COLLECTION_LIKE, collection);
}
// Enforce that the state is not empty, or larger than 1 character
if (!Strings.isEmpty(state) && state.length() == 1) {
queries.add("c.state = :state");
pb.addParam(PARAM_STATE, state);
request.setAttribute(PARAM_STATE, state);
}
queryString.append("SELECT c FROM Collection c");
......@@ -166,6 +172,7 @@ public class StatusServlet extends EntityManagerServlet {
Query countQuery = em.createQuery(countString.toString());
// TODO: Can probably tidy this up a bit
if (!Strings.isEmpty(group)) {
query.setParameter(PARAM_GROUP, "%" + group + "%");
countQuery.setParameter(PARAM_GROUP, "%" + group + "%");
......@@ -191,8 +198,29 @@ public class StatusServlet extends EntityManagerServlet {
pb.update(totalResults);
}
setWorkingCollection(request, em);
collections = new ArrayList<>();
for (Collection col : items) {
CollectionSummaryBean csb = createCollectionSummary(col);
collections.add(csb);
}
request.setAttribute(PAGE_COLLECTIONS, collections);
request.setAttribute(PAGE_STATES, ImmutableList.copyOf(CStateBean.values()));
request.setAttribute(PAGE_COUNT, count);
request.setAttribute(PAGE_NUMBER, pb);
if (hasJson(request)) {
dispatcher = request.getRequestDispatcher("status-json.jsp");
} else if (hasCsv(request)) {
dispatcher = request.getRequestDispatcher("status-csv.jsp");
} else {
dispatcher = request.getRequestDispatcher("status.jsp");
}
dispatcher.forward(request, response);
}
private void setWorkingCollection(HttpServletRequest request, EntityManager em) {
long collectionId;
String idParam = request.getParameter(PARAM_COLLECTION_ID);
collectionId = Strings.isValidLong(idParam) ? Long.parseLong(idParam) : -1;
......@@ -201,7 +229,6 @@ public class StatusServlet extends EntityManagerServlet {
if (Strings.isValidLong(idParam) && -1 == collectionId) {
// clear the working collection
request.getSession().removeAttribute(SESSION_WORKINGCOLLECTION);
} else if (Strings.isValidLong(idParam) // valid param
&& (workingCollectionBean == null // no working collection
|| !workingCollectionBean.getCollection().getId().equals(collectionId))) { // or one which does not equal our requested collection
......@@ -216,25 +243,6 @@ public class StatusServlet extends EntityManagerServlet {
// Continue using the current working collection
request.getSession().setAttribute(SESSION_WORKINGCOLLECTION, workingCollectionBean);
}
for (Collection col : items) {
CollectionSummaryBean csb = createCollectionSummary(col);
collections.add(csb);
}
request.setAttribute(PAGE_COLLECTIONS, collections);
request.setAttribute(PAGE_STATES, ImmutableList.copyOf(CStateBean.values()));
request.setAttribute(PAGE_COUNT, count);
request.setAttribute(PAGE_NUMBER, pb);
if (hasJson(request)) {
dispatcher = request.getRequestDispatcher("status-json.jsp");
} else if (hasCsv(request)) {
dispatcher = request.getRequestDispatcher("status-csv.jsp");
} else {
dispatcher = request.getRequestDispatcher("status.jsp");
}
dispatcher.forward(request, response);
}
private CollectionSummaryBean createCollectionSummary(Collection col) {
......@@ -255,7 +263,23 @@ public class StatusServlet extends EntityManagerServlet {
}
private boolean hasCsv(HttpServletRequest request) {
String value = (String) request.getParameter(PARAM_CSV);
String value = request.getParameter(PARAM_CSV);
return !Strings.isEmpty(value);
}
@Override
public String getParameter(HttpServletRequest request, String paramName, String defaultValue) {
String requestParam = request.getParameter(paramName);
// We could remove the session attributes here, not sure if that's a good idea though
// req param: not null and not empty -> use
// session attr not null and req param null -> use
// session attr not null and req param empty -> default
if (!Strings.isEmpty(requestParam)) {
return requestParam;
} else {
return defaultValue;
}
}
}
......@@ -32,48 +32,54 @@ package edu.umiacs.ace.monitor.audit;
import edu.umiacs.ace.driver.AuditIterable;
import edu.umiacs.ace.driver.DriverStateBean;
import edu.umiacs.ace.driver.filter.PathFilter;
import edu.umiacs.ace.driver.FileBean;
import edu.umiacs.ace.driver.StorageDriver;
import edu.umiacs.ace.driver.filter.PathFilter;
import edu.umiacs.ace.driver.filter.SimpleFilter;
import edu.umiacs.ace.ims.api.IMSException;
import edu.umiacs.ace.ims.api.IMSService;
import edu.umiacs.ace.ims.api.TokenRequestBatch;
import edu.umiacs.ace.ims.api.TokenValidator;
import edu.umiacs.ace.ims.ws.TokenRequest;
import edu.umiacs.ace.monitor.access.CollectionCountContext;
import edu.umiacs.ace.monitor.register.IngestThreadPool;
import edu.umiacs.ace.util.PersistUtil;
import edu.umiacs.ace.driver.filter.SimpleFilter;
import edu.umiacs.ace.monitor.compare.CollectionCompare2;
import edu.umiacs.ace.monitor.compare.CompareResults;
import edu.umiacs.ace.remote.JsonGateway;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.core.ConfigConstants;
import edu.umiacs.ace.monitor.core.MonitoredItem;
import edu.umiacs.ace.monitor.core.MonitoredItemManager;
import edu.umiacs.ace.monitor.log.LogEnum;
import edu.umiacs.ace.monitor.log.LogEvent;
import edu.umiacs.ace.monitor.log.LogEventManager;
import edu.umiacs.ace.monitor.peers.PeerCollection;
import edu.umiacs.ace.monitor.reporting.ReportSummary;
import edu.umiacs.ace.monitor.reporting.SchedulerContextListener;
import edu.umiacs.ace.monitor.reporting.SummaryGenerator;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.core.ConfigConstants;
import edu.umiacs.ace.monitor.settings.SettingsUtil;
import edu.umiacs.ace.monitor.log.LogEnum;
import edu.umiacs.ace.monitor.log.LogEvent;
import edu.umiacs.ace.monitor.peers.PeerCollection;
import edu.umiacs.ace.remote.JsonGateway;
import edu.umiacs.ace.token.AceToken;
import edu.umiacs.ace.util.PersistUtil;
import edu.umiacs.ace.util.TokenUtil;
import edu.umiacs.util.Strings;
import org.apache.log4j.Logger;
import org.apache.log4j.NDC;
import javax.mail.MessagingException;
import javax.persistence.EntityManager;
import javax.persistence.EntityTransaction;
import javax.persistence.Query;
import javax.sql.DataSource;
import java.io.InputStream;
import java.security.MessageDigest;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import javax.mail.MessagingException;
import javax.persistence.EntityManager;
import javax.persistence.EntityTransaction;
import org.apache.log4j.Logger;
import org.apache.log4j.NDC;
/**
*
......@@ -191,16 +197,6 @@ public final class AuditThread extends Thread implements CancelCallback {
iterableItems.cancel();
}
/*
if (batch != null) {
batch.close();
}
if (validator != null) {
validator.close();
}
*/
if (AuditThreadFactory.isRunning(coll) || AuditThreadFactory.isQueued(coll)) {
AuditThreadFactory.finished(coll);
}
......@@ -222,12 +218,6 @@ public final class AuditThread extends Thread implements CancelCallback {
session = System.currentTimeMillis();
logManager = new LogEventManager(session, coll);
logAuditStart();
/*
while (IngestThreadPool.isIngesting(coll)) {
LOG.debug("Waiting for ingest to finish");
Thread.sleep(500);
}
*/
callback = new FileAuditCallback(coll, session, this);
boolean auditTokens = SettingsUtil.getBoolean(coll,
......@@ -254,7 +244,7 @@ public final class AuditThread extends Thread implements CancelCallback {
// Let outstanding tokens finish, TODO, de-hackify this.
sleep(2000);
} catch (Throwable e) {
LOG.fatal("UNcaught exception in performAudit()", e);
LOG.fatal("Uncaught exception in performAudit()", e);
if (abortException != null) {
abortException = e;
}
......@@ -595,7 +585,6 @@ public final class AuditThread extends Thread implements CancelCallback {
}
TokenRequest request = new TokenRequest();
//request.setName(item.getPath());
request.setName(item.getId().toString());
request.setHashValue(currentFile.getHash());
if (!Strings.isEmpty(item.getPath()) &&
......@@ -616,7 +605,6 @@ public final class AuditThread extends Thread implements CancelCallback {
LogEvent event;
// If we have a registered file, set the digested value
LOG.trace(null == item.getFileDigest());
if (null == item.getFileDigest()) {
LOG.trace("Setting digest for registered file " + item.getPath());
item.setFileDigest(currentFile.getHash());
......@@ -713,8 +701,8 @@ public final class AuditThread extends Thread implements CancelCallback {
if (parentName != null) {
parentName = Strings.cleanStringForXml(parentName, '_');
for (int i = 1; i < currentFile.getPathList().length; i++) {
String parent = (currentFile.getPathList().length > i + 1
? currentFile.getPathList()[i + 1] : null);
String parent = currentFile.getPathList().length > i + 1 ?
currentFile.getPathList()[i + 1] : null;
parent = Strings.cleanStringForXml(parent, '_');
mim.createDirectory(currentFile.getPathList()[i], parent, coll);
}
......@@ -725,8 +713,7 @@ public final class AuditThread extends Thread implements CancelCallback {
private String[] createMailList() {
String addrs = SettingsUtil.getString(coll, ConfigConstants.ATTR_EMAIL_RECIPIENTS);
String[] maillist = (addrs == null ? null : addrs.split("\\s*,\\s*"));
return maillist;
return addrs == null ? null : addrs.split("\\s*,\\s*");
}
private void setCollectionState() {
......@@ -760,30 +747,41 @@ public final class AuditThread extends Thread implements CancelCallback {
return;
}
DataSource dataSource = PersistUtil.getDataSource();
EntityManager em = PersistUtil.getEntityManager();
MonitoredItemManager mim = new MonitoredItemManager(em);
EntityTransaction trans = em.getTransaction();
trans.begin();
for (MonitoredItem mi : mim.listItemsBefore(coll, d)) {
LOG.trace("Updating missing item: " + mi.getPath());
LOG.trace("Item information: LS= " + mi.getLastSeen()
+ ", LV = " + mi.getLastVisited()
+ ", SC = " + mi.getStateChange()
+ "Audit started on: " + d);
if (mi.getState() != 'M'
&& (mi.getStateChange() == null
|| d.after(mi.getStateChange()))) {
mi.setState('M');
mi.setStateChange(new Date());
em.persist(logManager.createItemEvent(LogEnum.FILE_MISSING,
mi.getPath()));
try {
// Update the monitored item table
Query query = em.createNamedQuery("MonitoredItem.updateMissing")
.setParameter("coll", coll)
.setParameter("date", d);
int i = query.executeUpdate();
if (i > 0) {
LOG.info("Set " + i + " new missing items");
// Add log entries for the new missing items
Connection connection = dataSource.getConnection();
PreparedStatement ps = connection.prepareStatement("INSERT INTO logevent(session, path, date, logtype, collection_id) " +
"SELECT ?, path, NOW(), ?, parentcollection_id FROM monitored_item m WHERE m.parentcollection_id = ? AND m.state = ? AND m.statechange = ?");
ps.setLong(1, session);
ps.setInt(2, LogEnum.FILE_MISSING.getType());
ps.setLong(3, coll.getId());
ps.setString(4, String.valueOf('M'));
ps.setTimestamp(5, new Timestamp(d.getTime()));
ps.executeUpdate();
ps.close();
connection.close();
}
mi.setLastVisited(new Date());
em.merge(mi);
trans.commit();
} catch (SQLException e) {
trans.rollback();
LOG.error("SQL error, rolling back missing items and log entries", e);
} finally {
em.close();
}
trans.commit();
em.close();
}
private void compareToPeers() {
......
......@@ -31,20 +31,20 @@
package edu.umiacs.ace.monitor.audit;
import edu.umiacs.ace.ims.api.ValidationCallback;
import edu.umiacs.ace.util.PersistUtil;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.core.MonitoredItem;
import edu.umiacs.ace.monitor.core.Token;
import edu.umiacs.ace.monitor.log.LogEventManager;
import java.util.Date;
import java.util.Map;
import edu.umiacs.ace.ims.ws.TokenResponse;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.log.LogEnum;
import edu.umiacs.ace.monitor.log.LogEventManager;
import edu.umiacs.ace.token.AceToken;
import edu.umiacs.ace.util.PersistUtil;
import edu.umiacs.util.Strings;
import org.apache.log4j.Logger;
import javax.persistence.EntityManager;
import javax.persistence.EntityTransaction;
import org.apache.log4j.Logger;
import java.util.Date;
import java.util.Map;
/**
*
......@@ -89,7 +89,6 @@ public final class TokenAuditCallback implements ValidationCallback {
// trans.commit();
em.close();
cancel.cancel();
}
@Override
......
......@@ -31,6 +31,7 @@
package edu.umiacs.ace.monitor.core;
import com.fasterxml.jackson.annotation.JsonIgnore;
import edu.umiacs.ace.monitor.peers.PeerCollection;
import edu.umiacs.util.Argument;
......@@ -50,7 +51,6 @@ import javax.persistence.OneToMany;
import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
import javax.xml.bind.annotation.XmlRootElement;
import java.io.Serializable;
import java.util.Date;
import java.util.List;
......@@ -79,11 +79,11 @@ import java.util.Map;
@NamedQuery(name = "Collection.getCollectionsInGroup", query =
"SELECT c FROM Collection c WHERE c.group = :group")
})
@XmlRootElement
public class Collection implements Serializable {
@OneToMany(mappedBy = "parent", cascade = CascadeType.ALL)
private List<PeerCollection> peerCollections;
private static final long serialVersionUID = 1L;
@Id
......@@ -134,6 +134,7 @@ public class Collection implements Serializable {
this.peerCollections = peerCollections;
}
@JsonIgnore
public List<PeerCollection> getPeerCollections() {
return peerCollections;
}
......
......@@ -101,35 +101,48 @@ import javax.persistence.TemporalType;
@NamedQuery(name = "MonitoredItem.listRemoteErrors", query =
"SELECT m FROM MonitoredItem m WHERE (m.state = 'P' OR m.state = 'D') AND m.directory = false AND m.parentCollection = :coll"),
@NamedQuery(name = "MonitoredItem.listLocalErrors", query =
"SELECT m FROM MonitoredItem m WHERE (m.state = 'C' OR m.state = 'M' OR m.state = 'T' OR m.state = 'I') AND m.directory = false AND m.parentCollection = :coll")
"SELECT m FROM MonitoredItem m WHERE (m.state = 'C' OR m.state = 'M' OR m.state = 'T' OR m.state = 'I') AND m.directory = false AND m.parentCollection = :coll"),
@NamedQuery(name = "MonitoredItem.updateMissing", query =
"UPDATE MonitoredItem SET state = 'M', stateChange = :date, lastVisited = :date WHERE parentCollection = :coll AND lastVisited < :date AND state != 'M'")
})
public class MonitoredItem implements Serializable, Comparable {
private static final long serialVersionUID = 1L;
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column(nullable = false, columnDefinition = "VARCHAR(255) COLLATE utf8")
private String path; // path relative to base directory
@Column(columnDefinition = "VARCHAR(255) COLLATE utf8")
private String parentPath;
@Column(nullable = false)
private boolean directory; // true if directory
@Temporal(TemporalType.TIMESTAMP)
private Date lastSeen;
@Temporal(TemporalType.TIMESTAMP)
private Date stateChange;
@Temporal(TemporalType.TIMESTAMP)
private Date lastVisited;
@ManyToOne
@JoinColumn(nullable = false)
private Collection parentCollection;
private String fileDigest;
@Column(nullable = false)
private char state;
@ManyToOne(cascade = CascadeType.ALL)
private Token token;
private long size;
private String fileDigest;
public void setId( Long id ) {
this.id = id;
......
......@@ -34,6 +34,11 @@ package edu.umiacs.ace.monitor.core;
import edu.umiacs.ace.monitor.log.LogEnum;
import edu.umiacs.ace.monitor.log.LogEventManager;