Skip to content

GitLab

Commit b2f5ecb3 authored by Michael Ritter's avatar Michael Ritter
Browse files

Merge branch 'release-1.10'

parents 06cc89a4 c66b09c5
Hide whitespace changes
Inline Side-by-side
Showing with 554 additions and 83 deletions
ace-am/pom.xml
View file @ b2f5ecb3
......@@ -4,7 +4,7 @@
<parent>
<artifactId>ace</artifactId>
<groupId>edu.umiacs.ace</groupId>
<version>1.10-SNAPSHOT</version>
<version>1.10</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>edu.umiacs.ace</groupId>
......@@ -252,7 +252,7 @@
<dependency>
<groupId>edu.umiacs.ace</groupId>
<artifactId>ace-ims-ws</artifactId>
<version>1.10-SNAPSHOT</version>
<version>1.10</version>
<type>jar</type>
</dependency>
<dependency>
......
ace-am/src/main/java/edu/umiacs/ace/monitor/access/CollectionCountContext.java
View file @ b2f5ecb3
......@@ -30,23 +30,25 @@
// $Id$
package edu.umiacs.ace.monitor.access;
import edu.umiacs.ace.util.PersistUtil;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.util.PersistUtil;
import edu.umiacs.sql.SQL;
import org.apache.log4j.Logger;
import org.apache.log4j.NDC;
import javax.persistence.EntityManager;
import javax.persistence.Query;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import javax.sql.DataSource;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import javax.persistence.EntityManager;
import javax.persistence.Query;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import javax.sql.DataSource;
import org.apache.log4j.Logger;
import org.apache.log4j.NDC;
/**
* Class to gather a count of all collections at startup.
......@@ -70,6 +72,7 @@ public class CollectionCountContext implements ServletContextListener {
private static Map<Collection, Long> totalSizeMap = new HashMap<Collection, Long>();
private static Map<Collection, Long> fileRemoteMissing = new HashMap<Collection, Long>();
private static Map<Collection, Long> fileRemoteCorrupt = new HashMap<Collection, Long>();
private static AtomicInteger totalCollections = new AtomicInteger(0);
private static Lock lock = new ReentrantLock();
private static boolean abort = false;
......@@ -99,6 +102,7 @@ public class CollectionCountContext implements ServletContextListener {
return;
}
queryCollection((Collection) o);
incrementTotalCollections();
}
em.close();
} catch (Exception e) {
......@@ -208,6 +212,18 @@ public class CollectionCountContext implements ServletContextListener {
return -1;
}
public static void incrementTotalCollections() {
totalCollections.incrementAndGet();
}
public static void decrementTotalCollections() {
totalCollections.decrementAndGet();
}
public static int getTotalCollections() {
return totalCollections.get();
}
/**
* Update statistics for a collection.
*
......
ace-am/src/main/java/edu/umiacs/ace/monitor/access/StatusServlet.java
View file @ b2f5ecb3
......@@ -30,88 +30,206 @@
// $Id$
package edu.umiacs.ace.monitor.access;
import edu.umiacs.ace.util.EntityManagerServlet;
import com.google.common.collect.ImmutableList;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.support.PageBean;
import edu.umiacs.ace.monitor.support.CStateBean;
import edu.umiacs.ace.util.EntityManagerServlet;
import edu.umiacs.util.Strings;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.log4j.Logger;
import javax.persistence.EntityManager;
import javax.persistence.Query;
import javax.servlet.RequestDispatcher;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
/**
*
* @author toaster
*/
public class StatusServlet extends EntityManagerServlet {
private static final Logger LOG = Logger.getLogger(StatusServlet.class);
private static final String PAGE_COLLECTIONS = "collections";
private static final String PAGE_STATES = "states";
private static final String PAGE_COUNT = "count";
private static final String PAGE_NUMBER = "page";
private static final long DEFAULT_PAGE = 0;
private static final int DEFAULT_COUNT = 100;
// TODO: Add session fields for pagination and search params
private static final String SESSION_WORKINGCOLLECTION = "workingCollection";
private static final String PARAM_CSV = "csv";
/**
// Pagination stuff
private static final String PARAM_COUNT = "count";
private static final String PARAM_PAGE = "page";
// Search Params
private static final String PARAM_GROUP = "group";
private static final String PARAM_STATE = "state";
private static final String PARAM_COLLECTION_LIKE = "collection";
private static final String PARAM_AUDIT_DATE = "audit";
// Filter params?
// ...group
// ...name (maybe some regex?)
/**
* Processes requests for both HTTP <code>GET</code> and <code>POST</code> methods.
* @param request servlet request
*
* @param request servlet request
* @param response servlet response
*/
@Override
protected void processRequest( HttpServletRequest request,
HttpServletResponse response, EntityManager em )
protected void processRequest(HttpServletRequest request,
HttpServletResponse response, EntityManager em)
throws ServletException, IOException {
RequestDispatcher dispatcher;
List<CollectionSummaryBean> collections;
List<Collection> items;
LOG.info("Processing /Status request");
LOG.info("Processing /Status additional info");
long page = getParameter(request, PARAM_PAGE, DEFAULT_PAGE);
int count = (int) getParameter(request, PARAM_COUNT, DEFAULT_COUNT);
String group = getParameter(request, PARAM_GROUP, null);
String collection = getParameter(request, PARAM_COLLECTION_LIKE, null);
String state = getParameter(request, PARAM_STATE, null);
// String date = getParameter(request, PARAM_GROUP, null);
PageBean pb = new PageBean((int) page, count, "Status");
long offset = page * count;
// our main query
StringBuilder queryString = new StringBuilder();
// for getting a count of items
StringBuilder countString = new StringBuilder();
// our parameters
StringBuilder params = new StringBuilder();
List<String> queries = new ArrayList<>();
if (!Strings.isEmpty(group)) {
queries.add("c.group LIKE :group");
pb.addParam(PARAM_GROUP, group);
}
if (!Strings.isEmpty(collection)) {
queries.add("c.name LIKE :collection");
pb.addParam(PARAM_COLLECTION_LIKE, collection);
}
// Enforce that the state is not empty, or larger than 1 character
if (!Strings.isEmpty(state) && state.length() == 1) {
queries.add("c.state = :state");
pb.addParam(PARAM_STATE, state);
}
queryString.append("SELECT c FROM Collection c");
countString.append("SELECT COUNT(c.id) FROM Collection c");
if (queries.size() > 0) {
params.append(" WHERE");
}
Iterator<String> it = queries.iterator();
while (it.hasNext()) {
String query = it.next();
params.append(" ")
.append(query);
if (it.hasNext()) {
params.append(" AND");
}
}
queryString.append(params);
// allows us to keep a consistent order when displaying collections
queryString.append(" ORDER BY c.group ASC, c.name ASC");
countString.append(params);
Query query =
em.createNamedQuery("Collection.listAllCollections");
em.createQuery(queryString.toString());
// em.createNamedQuery("Collection.listAllCollections");
query.setFirstResult((int) offset);
query.setMaxResults(count);
items = query.getResultList();
Query countQuery = em.createQuery(countString.toString());
collections = new ArrayList<CollectionSummaryBean>();
if (!Strings.isEmpty(group)) {
query.setParameter(PARAM_GROUP, "%" + group + "%");
countQuery.setParameter(PARAM_GROUP, "%" + group + "%");
}
if ( Strings.isValidLong(request.getParameter(PARAM_COLLECTION_ID)) && -1 == Long.parseLong(request.getParameter(
PARAM_COLLECTION_ID)) ) {
request.getSession().removeAttribute(SESSION_WORKINGCOLLECTION);
if (!Strings.isEmpty(collection)) {
query.setParameter(PARAM_COLLECTION_LIKE, "%" + collection + "%");
countQuery.setParameter(PARAM_COLLECTION_LIKE, "%" + collection + "%");
}
for ( Collection col : items ) {
CollectionSummaryBean csb = new CollectionSummaryBean();
csb.setCollection(col);
csb.setTotalFiles(CollectionCountContext.getFileCount(col));
csb.setActiveFiles(CollectionCountContext.getActiveCount(col));
csb.setCorruptFiles(CollectionCountContext.getCorruptCount(col));
csb.setInvalidDigests(CollectionCountContext.getTokenMismatchCount(
col));
csb.setMissingFiles(CollectionCountContext.getMissingCount(col));
csb.setMissingTokens(CollectionCountContext.getMissingTokenCount(col));
csb.setTotalSize(CollectionCountContext.getTotelSize(col));
csb.setTotalErrors(CollectionCountContext.getTotalErrors(col));
csb.setRemoteMissing(CollectionCountContext.getRemoteMissing(col));
csb.setRemoteCorrupt(CollectionCountContext.getRemoteCorrupt(col));
collections.add(csb);
// if param_collection was supplied as a parameter see if this bean
// should be set as working/details bean
if ( Strings.isValidLong(request.getParameter(PARAM_COLLECTION_ID)) && col.getId() == Long.parseLong(request.getParameter(
PARAM_COLLECTION_ID)) ) {
request.getSession().setAttribute(SESSION_WORKINGCOLLECTION, csb);
} else if ( request.getSession().getAttribute(
SESSION_WORKINGCOLLECTION) != null && ((CollectionSummaryBean) request.getSession().getAttribute(
SESSION_WORKINGCOLLECTION)).getCollection().getId().equals(col.getId()) ) {
request.getSession().setAttribute(SESSION_WORKINGCOLLECTION, csb);
if (!Strings.isEmpty(state) && state.length() == 1) {
query.setParameter(PARAM_STATE, state.charAt(0));
countQuery.setParameter(PARAM_STATE, state.charAt(0));
}
items = query.getResultList();
// We only need to execute this query if we have parameters
// and need to update the total count of collections
if (queries.size() > 0) {
long totalResults = (long) countQuery.getSingleResult();
LOG.info("Total results from query: " + totalResults);
pb.update(totalResults);
}
collections = new ArrayList<>();
long collectionId;
String idParam = request.getParameter(PARAM_COLLECTION_ID);
collectionId = Strings.isValidLong(idParam) ? Long.parseLong(idParam) : -1;
CollectionSummaryBean workingCollectionBean = (CollectionSummaryBean) request.getSession().getAttribute(SESSION_WORKINGCOLLECTION);
if (Strings.isValidLong(idParam) && -1 == collectionId) {
// clear the working collection
request.getSession().removeAttribute(SESSION_WORKINGCOLLECTION);
} else if (Strings.isValidLong(idParam) // valid param
&& (workingCollectionBean == null // no working collection
|| !workingCollectionBean.getCollection().getId().equals(collectionId))) { // or one which does not equal our requested collection
// Get the requested collection from the db and set it as our working collection
Collection workingCollection = em.find(Collection.class, collectionId);
if (workingCollection != null) {
workingCollectionBean = createCollectionSummary(workingCollection);
request.getSession().setAttribute(SESSION_WORKINGCOLLECTION, workingCollectionBean);
}
} else {
// Continue using the current working collection
request.getSession().setAttribute(SESSION_WORKINGCOLLECTION, workingCollectionBean);
}
for (Collection col : items) {
CollectionSummaryBean csb = createCollectionSummary(col);
collections.add(csb);
}
request.setAttribute(PAGE_COLLECTIONS, collections);
if ( hasJson(request) ) {
request.setAttribute(PAGE_STATES, ImmutableList.copyOf(CStateBean.values()));
request.setAttribute(PAGE_COUNT, count);
request.setAttribute(PAGE_NUMBER, pb);
if (hasJson(request)) {
dispatcher = request.getRequestDispatcher("status-json.jsp");
} else if ( hasCsv(request) ) {
} else if (hasCsv(request)) {
dispatcher = request.getRequestDispatcher("status-csv.jsp");
} else {
dispatcher = request.getRequestDispatcher("status.jsp");
......@@ -119,7 +237,24 @@ public class StatusServlet extends EntityManagerServlet {
dispatcher.forward(request, response);
}
private boolean hasCsv( HttpServletRequest request ) {
private CollectionSummaryBean createCollectionSummary(Collection col) {
CollectionSummaryBean csb = new CollectionSummaryBean();
csb.setCollection(col);
csb.setTotalFiles(CollectionCountContext.getFileCount(col));
csb.setActiveFiles(CollectionCountContext.getActiveCount(col));
csb.setCorruptFiles(CollectionCountContext.getCorruptCount(col));
csb.setInvalidDigests(CollectionCountContext.getTokenMismatchCount(
col));
csb.setMissingFiles(CollectionCountContext.getMissingCount(col));
csb.setMissingTokens(CollectionCountContext.getMissingTokenCount(col));
csb.setTotalSize(CollectionCountContext.getTotelSize(col));
csb.setTotalErrors(CollectionCountContext.getTotalErrors(col));
csb.setRemoteMissing(CollectionCountContext.getRemoteMissing(col));
csb.setRemoteCorrupt(CollectionCountContext.getRemoteCorrupt(col));
return csb;
}
private boolean hasCsv(HttpServletRequest request) {
String value = (String) request.getParameter(PARAM_CSV);
return !Strings.isEmpty(value);
}
......
ace-am/src/main/java/edu/umiacs/ace/monitor/access/browse/BrowseServlet.java
View file @ b2f5ecb3
......@@ -33,17 +33,16 @@ package edu.umiacs.ace.monitor.access.browse;
import edu.umiacs.ace.hashtree.Proof;
import edu.umiacs.ace.hashtree.ProofValidator;
import edu.umiacs.ace.monitor.core.MonitoredItem;
import edu.umiacs.ace.util.EntityManagerServlet;
import edu.umiacs.ace.monitor.access.browse.DirectoryTree.DirectoryNode;
import edu.umiacs.ace.monitor.audit.AuditThreadFactory;
import edu.umiacs.ace.monitor.audit.AuditTokens;
import edu.umiacs.ace.monitor.access.browse.DirectoryTree.DirectoryNode;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.core.MonitoredItem;
import edu.umiacs.ace.util.EntityManagerServlet;
import edu.umiacs.ace.util.HashValue;
import edu.umiacs.ace.util.TokenUtil;
import java.io.IOException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import org.apache.log4j.Logger;
import javax.persistence.EntityManager;
import javax.persistence.EntityNotFoundException;
import javax.servlet.RequestDispatcher;
......@@ -51,7 +50,9 @@ import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.HttpSession;
import org.apache.log4j.Logger;
import java.io.IOException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
/**
* Servlet to manage browsing a collection. This will store a directorytree in
......@@ -102,6 +103,7 @@ public class BrowseServlet extends EntityManagerServlet {
isRunning = true;
}
dt = new DirectoryTree(c);
request.setAttribute("collection", c);
session.setAttribute(SESSION_DIRECTORY_TREE, dt);
} else if ( itemId > 0 ) {
LOG.trace("Toggling item: " + itemId);
......@@ -114,6 +116,8 @@ public class BrowseServlet extends EntityManagerServlet {
AuditThreadFactory.isQueued(c)) {
isRunning = true;
}
request.setAttribute("collection", c);
session.setAttribute(SESSION_FILE,
loadFileBean(dt.getDirectoryNode(itemId), em,c));
if ( dt.getDirectoryNode(itemId).isDirectory() ) {
......@@ -131,7 +135,12 @@ public class BrowseServlet extends EntityManagerServlet {
dispatcher.forward(request, response);
}
private FileBean loadFileBean( DirectoryNode node, EntityManager em,Collection c ) {
private FileBean loadFileBean( DirectoryNode node, EntityManager em, Collection c ) {
// avoid possible null references below
if (node == null) {
return null;
}
FileBean retBean = new FileBean();
try {
......
ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java
View file @ b2f5ecb3
......@@ -191,6 +191,7 @@ public final class AuditThread extends Thread implements CancelCallback {
iterableItems.cancel();
}
/*
if (batch != null) {
batch.close();
}
......@@ -198,10 +199,12 @@ public final class AuditThread extends Thread implements CancelCallback {
if (validator != null) {
validator.close();
}
*/
if (AuditThreadFactory.isRunning(coll) || AuditThreadFactory.isQueued(coll)) {
AuditThreadFactory.finished(coll);
}
this.interrupt();
}
......
ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditTokens.java
View file @ b2f5ecb3
......@@ -63,7 +63,7 @@ public final class AuditTokens extends Thread implements CancelCallback {
new ConcurrentHashMap<>();
// private Map<TokenResponse, Token> tokenMap = new ConcurrentHashMap<TokenResponse, Token>();
private Map<AceToken, MonitoredItem> itemMap =
new ConcurrentHashMap<AceToken, MonitoredItem>();
new ConcurrentHashMap<>();
private static final Logger LOG = Logger.getLogger(AuditTokens.class);
private Collection collection;
private boolean cancel = false;
......@@ -186,7 +186,7 @@ public final class AuditTokens extends Thread implements CancelCallback {
} catch ( Throwable e ) {
LOG.fatal("UNcaught exception in doWork()", e);
} finally {
itemMap.clear(); // free memory in case this gets stuck hianging around
itemMap.clear(); // free memory in case this gets stuck hanging around
}
}
......@@ -235,6 +235,7 @@ public final class AuditTokens extends Thread implements CancelCallback {
TokenValidator validator = openIms();
if ( validator == null ) {
runningThreads.remove(collection);
return;
}
em = PersistUtil.getEntityManager();
......
ace-am/src/main/java/edu/umiacs/ace/monitor/core/ManageCollectionServlet.java
View file @ b2f5ecb3
......@@ -30,15 +30,14 @@
// $Id$
package edu.umiacs.ace.monitor.core;
import edu.umiacs.ace.util.EntityManagerServlet;
import edu.umiacs.ace.util.PersistUtil;
import edu.umiacs.ace.driver.StorageDriver;
import edu.umiacs.ace.driver.StorageDriverFactory;
import edu.umiacs.ace.monitor.access.CollectionCountContext;
import edu.umiacs.ace.util.EntityManagerServlet;
import edu.umiacs.ace.util.PersistUtil;
import edu.umiacs.util.Strings;
import java.io.IOException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.HashMap;
import org.apache.log4j.Logger;
import javax.persistence.EntityManager;
import javax.persistence.EntityTransaction;
import javax.persistence.Query;
......@@ -47,7 +46,10 @@ import javax.servlet.ServletException;
import javax.servlet.ServletRequest;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.log4j.Logger;
import java.io.IOException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.HashMap;
/**
* add,modify,remove the settings of a collection
......@@ -154,6 +156,7 @@ public class ManageCollectionServlet extends EntityManagerServlet {
PersistUtil.persist(collection);
storage = StorageDriverFactory.createStorageAccess(collection,
em);
CollectionCountContext.incrementTotalCollections();
}
dispatcher = request.getRequestDispatcher("collectionmodify.jsp");
......@@ -206,6 +209,7 @@ public class ManageCollectionServlet extends EntityManagerServlet {
storage.remove(em);
}
em.remove(collection);
CollectionCountContext.decrementTotalCollections();
trans.commit();
}
......
ace-am/src/main/java/edu/umiacs/ace/monitor/support/CStateBean.java 0 → 100644
View file @ b2f5ecb3
package edu.umiacs.ace.monitor.support;
/**
* A Bean to hold each collection status as well as its definition
*
* Created by shake on 4/5/16.
*/
public enum CStateBean {
NO_FILTER(""),
ACTIVE("A"),
ERROR("E"),
NEVER_SCANNED("N");
private final String state;
CStateBean(String state) {
this.state = state;
}
public String getState() {
return state;
}
}
ace-am/src/main/java/edu/umiacs/ace/monitor/support/PageBean.java 0 → 100644
View file @ b2f5ecb3
package edu.umiacs.ace.monitor.support;
import edu.umiacs.ace.monitor.access.CollectionCountContext;
import org.apache.log4j.Logger;
/**
*
* Created by shake on 1/28/16.
*/
public class PageBean {
private static Logger LOG = Logger.getLogger(PageBean.class);
// current page
int page;
// page number
int count;
// item offset (for db queries)
int offset;
int previous;
int next;
int end;
StringBuilder url;
private boolean firstParam;
public PageBean(int page, int count, String root) {
this.firstParam = true;
this.page = page;