Commit 123c6d17 authored by Michael Ritter's avatar Michael Ritter

Merge branch 'release-1.13' into 'master'

Release 1.13

Closes #44

See merge request !9
parents e8d0f78a bbdacd08
......@@ -4,7 +4,7 @@
<parent>
<artifactId>ace</artifactId>
<groupId>edu.umiacs.ace</groupId>
<version>1.12-RELEASE</version>
<version>1.13-RELEASE</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>ace-am</artifactId>
......@@ -49,44 +49,6 @@
<target>1.8</target>
</configuration>
</plugin>
<!-- debian package made easy -->
<!--
<plugin>
<artifactId>jdeb</artifactId>
<groupId>org.vafer</groupId>
<version>0.11</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>jdeb</goal>
</goals>
<configuration>
<controlDir>${basedir}/src/main/deb/control</controlDir>
<dataSet>
<data>
<src>${project.build.directory}/${project.build.finalName}.war</src>
<type>file</type>
<dst>ace-am.war</dst>
<mapper>
<type>perm</type>
<prefix>/var/lib/tomcat6/webapps</prefix>
</mapper>
</data>
<data>
<src>${basedir}/src/main/sql/ace-am.sql</src>
<type>file</type>
<mapper>
<type>perm</type>
<prefix>/tmp</prefix>
</mapper>
</data>
</dataSet>
</configuration>
</execution>
</executions>
</plugin>
-->
</plugins>
</build>
......@@ -166,7 +128,6 @@
</dependency>
<!-- Various libs for helping with stuff -->
<dependency>
<groupId>commons-fileupload</groupId>
<artifactId>commons-fileupload</artifactId>
......@@ -194,7 +155,13 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>18.0</version>
<version>25.0-jre</version>
</dependency>
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
<version>2.6.2</version>
</dependency>
<dependency>
......@@ -276,7 +243,7 @@
<dependency>
<groupId>edu.umiacs.ace</groupId>
<artifactId>ace-ims-ws</artifactId>
<version>1.12-RELEASE</version>
<version>1.13-RELEASE</version>
<type>jar</type>
</dependency>
<dependency>
......
......@@ -31,15 +31,16 @@
package edu.umiacs.ace.driver;
import edu.umiacs.ace.monitor.settings.SettingsParameter;
import edu.umiacs.ace.monitor.settings.SettingsUtil;
import edu.umiacs.ace.util.PersistUtil;
import edu.umiacs.util.Strings;
import java.util.Timer;
import java.util.TimerTask;
import org.apache.log4j.Logger;
import javax.persistence.EntityManager;
import javax.persistence.Query;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import org.apache.log4j.Logger;
import java.util.Timer;
import java.util.TimerTask;
/**
*
......@@ -94,24 +95,22 @@ public class QueryThrottle implements ServletContextListener {
@Override
public void contextInitialized( ServletContextEvent sce ) {
EntityManager em = PersistUtil.getEntityManager();
Query q = em.createNamedQuery("SettingsParameter.getAttr");
q.setParameter("attr", PARAM_TIME);
SettingsParameter s = (SettingsParameter) q.getSingleResult();
String time = s.getValue();
if ( Strings.isValidInt(time) ) {
setMinWait(Integer.parseInt(time));
// Todo: migrate these out of here
SettingsParameter blockTime = SettingsUtil.getOrDefault(PARAM_TIME,
String.valueOf(minWait), em);
SettingsParameter bps = SettingsUtil.getOrDefault(PARAM_BPS,
String.valueOf(maxBps), em);
if (Strings.isValidInt(blockTime.getValue())) {
setMinWait(Integer.parseInt(blockTime.getValue()));
LOG.info("Setting query throttle minwait to " + minWait);
}
q.setParameter("attr", PARAM_BPS);
s = (SettingsParameter) q.getSingleResult();
String maxBpsString = s.getValue();
if ( Strings.isValidLong(maxBpsString) ) {
setMaxBps(Long.parseLong(maxBpsString));
if (Strings.isValidLong(bps.getValue())) {
setMaxBps(Long.parseLong(bps.getValue()));
}
if ( minWait > 0 ) {
checkTimer = new Timer("srb timer");
checkTimer.schedule(new TimerTask() {
......@@ -122,8 +121,6 @@ public class QueryThrottle implements ServletContextListener {
}
}, 0, minWait);
}
}
@Override
......
......@@ -18,6 +18,12 @@ public final class GroupSummary {
this.count = count;
}
public GroupSummary(String group, BigDecimal size, Long count) {
this.group = group;
this.size = size;
this.count = new BigDecimal(count);
}
public String getGroup() {
return group;
}
......
......@@ -15,9 +15,9 @@ import java.util.Map;
/**
* Servlet context listener which acts as a cache for Summary information about
* groups in ACE
*
* <p>
* If there are many groups, it's conceivable that this could be a strain on memory
*
* <p>
* Created by shake on 4/4/17.
*/
public class GroupSummaryContext implements ServletContextListener {
......@@ -28,33 +28,35 @@ public class GroupSummaryContext implements ServletContextListener {
/**
* Query to get ALL group summaries
*/
private static final String SUMMARY_QUERY_ALL = "SELECT c.colgroup, sum(m.size) AS size, sum(m.count) AS count " +
"FROM collection c " +
"JOIN ( " +
"SELECT sum(size) AS size, count(id) AS count, parentcollection_id " +
"FROM monitored_item " +
"WHERE directory = 0 " +
"GROUP BY parentcollection_id " +
") AS m ON c.id = m.parentcollection_id " +
"WHERE c.colgroup IS NOT NULL " +
"GROUP BY c.colgroup";
private static final String SUMMARY_QUERY_ALL =
"SELECT c.colgroup, sum(m.size) AS size, sum(m.count) AS count " +
"FROM collection c " +
"JOIN ( " +
"SELECT sum(size) AS size, count(id) AS count, parentcollection_id " +
"FROM monitored_item " +
"WHERE directory = 0 " +
"GROUP BY parentcollection_id " +
") AS m ON c.id = m.parentcollection_id " +
"WHERE c.colgroup IS NOT NULL " +
"GROUP BY c.colgroup";
/**
* Query to get the group summary for a single group
*/
private static final String SUMMARY_QUERY_GROUP = "select c.colgroup, sum(m.size) AS size, count(m.id) AS count " +
"FROM monitored_item m " +
"JOIN ( " +
" select colgroup, id " +
" FROM collection " +
" WHERE colgroup = ? " +
") c ON c.id = m.parentcollection_id WHERE directory = 0";
private static final String SUMMARY_QUERY_GROUP =
"select c.colgroup, sum(m.size) AS size, count(m.id) AS count " +
"FROM monitored_item m " +
"JOIN ( " +
" select colgroup, id " +
" FROM collection " +
" WHERE colgroup = ? " +
") c ON c.id = m.parentcollection_id WHERE directory = 0";
@Override
public void contextInitialized(ServletContextEvent servletContextEvent) {
summaries = new HashMap<>();
updateSummaries(ImmutableList.of(SUMMARY_QUERY_ALL),
ImmutableList.<String>of());
ImmutableList.of());
}
@Override
......@@ -80,7 +82,7 @@ public class GroupSummaryContext implements ServletContextListener {
/**
* Static method to allow us to update the summary of a group
*
* @param sql the sql query to build
* @param sql the sql query to build
* @param params the parameters to pass along to the query
*/
private static void updateSummaries(List<String> sql, List<String> params) {
......@@ -98,7 +100,7 @@ public class GroupSummaryContext implements ServletContextListener {
i++;
}
List<GroupSummary> results = (List<GroupSummary>)groupSummary.getResultList();
List<GroupSummary> results = (List<GroupSummary>) groupSummary.getResultList();
for (GroupSummary result : results) {
log.info("Result: group " + result.getGroup() + ", size: " + result.getSize() + ", count: " + result.getCount());
summaries.put(result.getGroup(), result);
......
......@@ -41,6 +41,7 @@ import org.apache.log4j.Logger;
import javax.persistence.EntityManager;
import javax.persistence.Query;
import javax.persistence.TypedQuery;
import javax.servlet.RequestDispatcher;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
......@@ -75,9 +76,9 @@ public class StatusServlet extends EntityManagerServlet {
private static final String PARAM_PAGE = "page";
// Search Params
private static final String PARAM_GROUP = "group";
private static final String PARAM_STATE = "state";
private static final String PARAM_COLLECTION_LIKE = "collection";
private static final String PARAM_GROUP = "status_group";
private static final String PARAM_STATE = "status_state";
private static final String PARAM_COLLECTION_LIKE = "status_collection";
private static final String PARAM_AUDIT_DATE = "audit";
// Filter params?
......@@ -103,8 +104,8 @@ public class StatusServlet extends EntityManagerServlet {
// local getParameter so that the session is checked as well
String group = getParameter(request, PARAM_GROUP, null);
String collection = getParameter(request, PARAM_COLLECTION_LIKE, null);
String state = getParameter(request, PARAM_STATE, null);
String collection = getParameter(request, PARAM_COLLECTION_LIKE, null);
// String date = getParameter(request, PARAM_GROUP, null);
PageBean pb = new PageBean((int) page, count, "");
......@@ -123,20 +124,20 @@ public class StatusServlet extends EntityManagerServlet {
// TODO: Can probably tidy this up a bit
if (!Strings.isEmpty(group)) {
queries.add("c.group LIKE :group");
queries.add("c.group LIKE :status_group");
pb.addParam(PARAM_GROUP, group);
request.setAttribute(PARAM_GROUP, group);
}
if (!Strings.isEmpty(collection)) {
queries.add("c.name LIKE :collection");
queries.add("c.name LIKE :status_collection");
pb.addParam(PARAM_COLLECTION_LIKE, collection);
request.setAttribute(PARAM_COLLECTION_LIKE, collection);
}
// Enforce that the state is not empty, or larger than 1 character
if (!Strings.isEmpty(state) && state.length() == 1) {
queries.add("c.state = :state");
queries.add("c.state = :status_state");
pb.addParam(PARAM_STATE, state);
request.setAttribute(PARAM_STATE, state);
}
......@@ -163,8 +164,8 @@ public class StatusServlet extends EntityManagerServlet {
queryString.append(" ORDER BY c.group ASC, c.name ASC");
countString.append(params);
Query query =
em.createQuery(queryString.toString());
TypedQuery<Collection> query =
em.createQuery(queryString.toString(), Collection.class);
// em.createNamedQuery("Collection.listAllCollections");
query.setFirstResult((int) offset);
query.setMaxResults(count);
......@@ -255,7 +256,7 @@ public class StatusServlet extends EntityManagerServlet {
col));
csb.setMissingFiles(CollectionCountContext.getMissingCount(col));
csb.setMissingTokens(CollectionCountContext.getMissingTokenCount(col));
csb.setTotalSize(CollectionCountContext.getTotelSize(col));
csb.setTotalSize(CollectionCountContext.getTotalSize(col));
csb.setTotalErrors(CollectionCountContext.getTotalErrors(col));
csb.setRemoteMissing(CollectionCountContext.getRemoteMissing(col));
csb.setRemoteCorrupt(CollectionCountContext.getRemoteCorrupt(col));
......
......@@ -120,22 +120,16 @@ public class BrowseServlet extends EntityManagerServlet {
request.setAttribute("collection", c);
session.setAttribute(SESSION_FILE,
loadFileBean(dt.getDirectoryNode(itemId), em,c));
if ( dt.getDirectoryNode(itemId).isDirectory() ) {
if (dt.getDirectoryNode(itemId) != null && dt.getDirectoryNode(itemId).isDirectory()) {
dt.toggleItem(itemId);
}
// else
// {
// session.setAttribute(SESSION_FILE,
// loadFileBean(dt.getDirectoryNode(itemId)));
// }
}
request.setAttribute(PAGE_ISAUDITING, isRunning);
RequestDispatcher dispatcher = request.getRequestDispatcher("browse.jsp");
dispatcher.forward(request, response);
}
private FileBean loadFileBean( DirectoryNode node, EntityManager em, Collection c ) {
private FileBean loadFileBean(DirectoryNode node, EntityManager em, Collection c) {
// avoid possible null references below
if (node == null) {
return null;
......@@ -144,20 +138,20 @@ public class BrowseServlet extends EntityManagerServlet {
FileBean retBean = new FileBean();
try {
MonitoredItem master = em.getReference(MonitoredItem.class,
node.getId());
MonitoredItem master = em.getReference(MonitoredItem.class, node.getId());
retBean.root = master;
retBean.name = node.getName();
if (master.getToken() != null) {
// TokenResponse resp = (TokenResponse)master.getToken().getToken();
MessageDigest digest = MessageDigest.getInstance(master.getToken().getProofAlgorithm());
MessageDigest digest = MessageDigest.getInstance(
master.getToken().getProofAlgorithm());
ProofValidator pv = new ProofValidator();
Proof proof = TokenUtil.extractProof(master.getToken());
String fileDigest = master.getFileDigest();
if ( fileDigest != null ){
byte[] root = pv.rootHash(digest, proof, HashValue.asBytes(master.getFileDigest()));
if (fileDigest != null){
byte[] root = pv.rootHash(digest, proof,
HashValue.asBytes(master.getFileDigest()));
retBean.itemProof = HashValue.asHexString(root);
}else {
retBean.itemProof = null;
......
......@@ -508,10 +508,10 @@ public final class AuditThread extends Thread implements CancelCallback {
MonitoredItemManager mim;
LOG.trace(
"Driver returned Item: " + currentFile.getPathList()[0]
+ " error: " + currentFile.isError() + " error msg: "
+ currentFile.getErrorMessage() + " hash: "
+ currentFile.getHash());
"Driver returned {Item=" + currentFile.getPathList()[0]
+ ";error=" + currentFile.isError() + ";error-msg="
+ currentFile.getErrorMessage() + ";hash="
+ currentFile.getHash() + "}");
mim = new MonitoredItemManager(em);
......
......@@ -33,6 +33,7 @@ package edu.umiacs.ace.monitor.audit;
import edu.umiacs.ace.driver.StorageDriver;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.core.MonitoredItem;
import edu.umiacs.ace.monitor.settings.SettingsConstants;
import edu.umiacs.ace.util.CollectionThreadPoolExecutor;
import edu.umiacs.ace.util.KSFuture;
import edu.umiacs.ace.util.PersistUtil;
......@@ -57,22 +58,22 @@ import static edu.umiacs.ace.util.Submittable.RunState.RUNNING;
public class AuditThreadFactory {
private static final Logger LOG = Logger.getLogger(AuditThreadFactory.class);
private static ConcurrentHashMap<Collection, KSFuture<AuditThread>> audits
= new ConcurrentHashMap<>();
private static ConcurrentHashMap<Collection, KSFuture<AuditThread>> audits =
new ConcurrentHashMap<>();
private static int max_audits = 3;
private static String imsHost = null;
private static int imsPort = 8080;
private static int imsPort = 80;
private static String tokenClass = "SHA-256";
private static boolean auditOnly = false;
private static boolean auditSample = false;
private static boolean ssl = false;
public static boolean blocking = false;
public static int maxBlockTime = 0;
private static boolean blocking = false;
private static int maxBlockTime = 0;
public static void setIMS( String ims ) {
if (Strings.isEmpty(ims)) {
LOG.error("Empty ims string, ignoring");
return;
LOG.error("Empty ims string, setting from default value");
ims = SettingsConstants.ims;
}
imsHost = ims;
}
......@@ -93,10 +94,10 @@ public class AuditThreadFactory {
return imsPort;
}
public static void setImsPort( int imsPort ) {
if ( imsPort < 1 && imsPort > 32768 ) {
LOG.error("ims port must be between 1 and 32768");
return;
public static void setImsPort(int imsPort) {
if (imsPort < 1 && imsPort > 32768) {
LOG.error("ims port must be between 1 and 32768, setting default");
imsPort = Integer.parseInt(SettingsConstants.imsPort);
}
AuditThreadFactory.imsPort = imsPort;
}
......@@ -151,7 +152,7 @@ public class AuditThreadFactory {
newThread.setImsHost(imsHost);
newThread.setImsPort(imsPort);
newThread.setTokenClassName(tokenClass);
KSFuture f = executor.submitFileAudit(c, newThread);
KSFuture<AuditThread> f = executor.submitFileAudit(c, newThread);
if (f != null) {
audits.put(c, f);
}
......@@ -168,11 +169,11 @@ public class AuditThreadFactory {
return null;
}
public static final boolean isQueued(Collection c) {
public static boolean isQueued(Collection c) {
return checkState(c, QUEUED);
}
public static final boolean isRunning(Collection c) {
public static boolean isRunning(Collection c) {
return checkState(c, RUNNING);
}
......@@ -216,22 +217,6 @@ public class AuditThreadFactory {
AuditThreadFactory.ssl = ssl;
}
/**
* Return the current thread for a collection.
* @param c collection to fetch
*
* @return current running thread or null if nothing is running
*
public static AuditThread getThread( Collection c ) {
synchronized ( runningThreads ) {
if ( isRunning(c) ) {
return runningThreads.get(c);
}
}
return null;
}
*/
/**
* Method for AuditThread to notify its finished
*
......
......@@ -55,46 +55,40 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
*
* @author toaster
*/
public final class AuditTokens extends Thread implements CancelCallback {
private static final Map<Collection, KSFuture<AuditTokens>> runningThreads =
new ConcurrentHashMap<>();
// private Map<TokenResponse, Token> tokenMap = new ConcurrentHashMap<TokenResponse, Token>();
private Map<AceToken, MonitoredItem> itemMap =
new ConcurrentHashMap<>();
new ConcurrentHashMap<>();
private static final Logger LOG = Logger.getLogger(AuditTokens.class);
private Collection collection;
private boolean cancel = false;
private boolean hasRun = false;
private long session;
// private long totalErrors = 0;
private long tokensSeen = 0;
// private long validTokens = 0;
private TokenAuditCallback callback;
private String imsHost;
private int imsPort;
LogEventManager logManager;
private LogEventManager logManager;
private AuditTokens( Collection collection ) {
private AuditTokens(Collection collection) {
this.collection = collection;
session = System.currentTimeMillis();
logManager = new LogEventManager(session, collection);
// this.start();
}
/**
* Return the current thread for a collection.
*
* @param c collection to fetch
*
* @return current running thread or null if nothing is running
*
*/
public static AuditTokens getThread( Collection c ) {
synchronized ( runningThreads ) {
if ( isRunning(c) ) {
public static AuditTokens getThread(Collection c) {
synchronized (runningThreads) {
if (isRunning(c)) {
KSFuture<AuditTokens> future = runningThreads.get(c);
return getAuditTokens(future);
}
......@@ -104,11 +98,11 @@ public final class AuditTokens extends Thread implements CancelCallback {
/**
* Return a new or existing thread. New threads will start replication
*
* @param c
* @return
*
* @param c the collection to create a thread for
* @return The AuditTokens thread
*/
public static AuditTokens createThread( Collection c ) {
public static AuditTokens createThread(Collection c) {
CollectionThreadPoolExecutor executor = CollectionThreadPoolExecutor.getExecutor();
AuditTokens at = new AuditTokens(c);
at.imsHost = AuditThreadFactory.getIMS();
......@@ -127,8 +121,8 @@ public final class AuditTokens extends Thread implements CancelCallback {
return at;
}
public static final boolean isRunning( Collection c ) {
synchronized ( runningThreads ) {
public static boolean isRunning(Collection c) {
synchronized (runningThreads) {
return runningThreads.containsKey(c);
}
}
......@@ -140,7 +134,7 @@ public final class AuditTokens extends Thread implements CancelCallback {
}
static void cancellAll() {
for ( KSFuture<AuditTokens> at : runningThreads.values() ) {
for (KSFuture<AuditTokens> at : runningThreads.values()) {
AuditTokens thread = getAuditTokens(at);
thread.cancel();
at.cancel(true);
......@@ -153,7 +147,7 @@ public final class AuditTokens extends Thread implements CancelCallback {
}
public long getTotalErrors() {
if ( callback == null ) {
if (callback == null) {
return 0;
}
return callback.getTotalErrors();
......@@ -164,7 +158,7 @@ public final class AuditTokens extends Thread implements CancelCallback {
}
public long getValidTokens() {
if ( callback == null ) {
if (callback == null) {
return 0;
}
return callback.getValidTokens();
......@@ -173,7 +167,7 @@ public final class AuditTokens extends Thread implements CancelCallback {
@Override
public void run() {
if ( hasRun ) {
if (hasRun) {
LOG.fatal("Cannot run thread, already executed");
throw new IllegalStateException("Thread has already run");
}
......@@ -184,12 +178,11 @@ public final class AuditTokens extends Thread implements CancelCallback {
doWork();
} catch ( Throwable e ) {
} catch (Throwable e) {
LOG.fatal("UNcaught exception in doWork()", e);
} finally {
itemMap.clear(); // free memory in case this gets stuck hanging around
}
}
private void markMissingTokensOffline() {
......@@ -207,20 +200,20 @@ public final class AuditTokens extends Thread implements CancelCallback {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
IMSService ims;
ims = IMSService.connect(imsHost,
imsPort,
AuditThreadFactory.useSSL(),
AuditThreadFactory.isBlocking(),
AuditThreadFactory.getMaxBlockTime());
imsPort,
AuditThreadFactory.useSSL(),
AuditThreadFactory.isBlocking(),
AuditThreadFactory.getMaxBlockTime());
callback = new TokenAuditCallback(itemMap, this, collection, session);
validator = ims.createTokenValidator(callback, 1000, 5000,
digest);