Commit 123c6d17 authored by Michael Ritter's avatar Michael Ritter
Browse files

Merge branch 'release-1.13' into 'master'

Release 1.13

Closes #44

See merge request !9
parents e8d0f78a bbdacd08
......@@ -4,7 +4,7 @@
<parent>
<artifactId>ace</artifactId>
<groupId>edu.umiacs.ace</groupId>
<version>1.12-RELEASE</version>
<version>1.13-RELEASE</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>ace-am</artifactId>
......@@ -49,44 +49,6 @@
<target>1.8</target>
</configuration>
</plugin>
<!-- debian package made easy -->
<!--
<plugin>
<artifactId>jdeb</artifactId>
<groupId>org.vafer</groupId>
<version>0.11</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>jdeb</goal>
</goals>
<configuration>
<controlDir>${basedir}/src/main/deb/control</controlDir>
<dataSet>
<data>
<src>${project.build.directory}/${project.build.finalName}.war</src>
<type>file</type>
<dst>ace-am.war</dst>
<mapper>
<type>perm</type>
<prefix>/var/lib/tomcat6/webapps</prefix>
</mapper>
</data>
<data>
<src>${basedir}/src/main/sql/ace-am.sql</src>
<type>file</type>
<mapper>
<type>perm</type>
<prefix>/tmp</prefix>
</mapper>
</data>
</dataSet>
</configuration>
</execution>
</executions>
</plugin>
-->
</plugins>
</build>
......@@ -166,7 +128,6 @@
</dependency>
<!-- Various libs for helping with stuff -->
<dependency>
<groupId>commons-fileupload</groupId>
<artifactId>commons-fileupload</artifactId>
......@@ -194,7 +155,13 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>18.0</version>
<version>25.0-jre</version>
</dependency>
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
<version>2.6.2</version>
</dependency>
<dependency>
......@@ -276,7 +243,7 @@
<dependency>
<groupId>edu.umiacs.ace</groupId>
<artifactId>ace-ims-ws</artifactId>
<version>1.12-RELEASE</version>
<version>1.13-RELEASE</version>
<type>jar</type>
</dependency>
<dependency>
......
......@@ -31,15 +31,16 @@
package edu.umiacs.ace.driver;
import edu.umiacs.ace.monitor.settings.SettingsParameter;
import edu.umiacs.ace.monitor.settings.SettingsUtil;
import edu.umiacs.ace.util.PersistUtil;
import edu.umiacs.util.Strings;
import java.util.Timer;
import java.util.TimerTask;
import org.apache.log4j.Logger;
import javax.persistence.EntityManager;
import javax.persistence.Query;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import org.apache.log4j.Logger;
import java.util.Timer;
import java.util.TimerTask;
/**
*
......@@ -94,24 +95,22 @@ public class QueryThrottle implements ServletContextListener {
@Override
public void contextInitialized( ServletContextEvent sce ) {
EntityManager em = PersistUtil.getEntityManager();
Query q = em.createNamedQuery("SettingsParameter.getAttr");
q.setParameter("attr", PARAM_TIME);
SettingsParameter s = (SettingsParameter) q.getSingleResult();
String time = s.getValue();
if ( Strings.isValidInt(time) ) {
setMinWait(Integer.parseInt(time));
// Todo: migrate these out of here
SettingsParameter blockTime = SettingsUtil.getOrDefault(PARAM_TIME,
String.valueOf(minWait), em);
SettingsParameter bps = SettingsUtil.getOrDefault(PARAM_BPS,
String.valueOf(maxBps), em);
if (Strings.isValidInt(blockTime.getValue())) {
setMinWait(Integer.parseInt(blockTime.getValue()));
LOG.info("Setting query throttle minwait to " + minWait);
}
q.setParameter("attr", PARAM_BPS);
s = (SettingsParameter) q.getSingleResult();
String maxBpsString = s.getValue();
if ( Strings.isValidLong(maxBpsString) ) {
setMaxBps(Long.parseLong(maxBpsString));
if (Strings.isValidLong(bps.getValue())) {
setMaxBps(Long.parseLong(bps.getValue()));
}
if ( minWait > 0 ) {
checkTimer = new Timer("srb timer");
checkTimer.schedule(new TimerTask() {
......@@ -122,8 +121,6 @@ public class QueryThrottle implements ServletContextListener {
}
}, 0, minWait);
}
}
@Override
......
......@@ -18,6 +18,12 @@ public final class GroupSummary {
this.count = count;
}
public GroupSummary(String group, BigDecimal size, Long count) {
this.group = group;
this.size = size;
this.count = new BigDecimal(count);
}
public String getGroup() {
return group;
}
......
......@@ -15,9 +15,9 @@ import java.util.Map;
/**
* Servlet context listener which acts as a cache for Summary information about
* groups in ACE
*
* <p>
* If there are many groups, it's conceivable that this could be a strain on memory
*
* <p>
* Created by shake on 4/4/17.
*/
public class GroupSummaryContext implements ServletContextListener {
......@@ -28,33 +28,35 @@ public class GroupSummaryContext implements ServletContextListener {
/**
* Query to get ALL group summaries
*/
private static final String SUMMARY_QUERY_ALL = "SELECT c.colgroup, sum(m.size) AS size, sum(m.count) AS count " +
"FROM collection c " +
"JOIN ( " +
"SELECT sum(size) AS size, count(id) AS count, parentcollection_id " +
"FROM monitored_item " +
"WHERE directory = 0 " +
"GROUP BY parentcollection_id " +
") AS m ON c.id = m.parentcollection_id " +
"WHERE c.colgroup IS NOT NULL " +
"GROUP BY c.colgroup";
private static final String SUMMARY_QUERY_ALL =
"SELECT c.colgroup, sum(m.size) AS size, sum(m.count) AS count " +
"FROM collection c " +
"JOIN ( " +
"SELECT sum(size) AS size, count(id) AS count, parentcollection_id " +
"FROM monitored_item " +
"WHERE directory = 0 " +
"GROUP BY parentcollection_id " +
") AS m ON c.id = m.parentcollection_id " +
"WHERE c.colgroup IS NOT NULL " +
"GROUP BY c.colgroup";
/**
* Query to get the group summary for a single group
*/
private static final String SUMMARY_QUERY_GROUP = "select c.colgroup, sum(m.size) AS size, count(m.id) AS count " +
"FROM monitored_item m " +
"JOIN ( " +
" select colgroup, id " +
" FROM collection " +
" WHERE colgroup = ? " +
") c ON c.id = m.parentcollection_id WHERE directory = 0";
private static final String SUMMARY_QUERY_GROUP =
"select c.colgroup, sum(m.size) AS size, count(m.id) AS count " +
"FROM monitored_item m " +
"JOIN ( " +
" select colgroup, id " +
" FROM collection " +
" WHERE colgroup = ? " +
") c ON c.id = m.parentcollection_id WHERE directory = 0";
@Override
public void contextInitialized(ServletContextEvent servletContextEvent) {
summaries = new HashMap<>();
updateSummaries(ImmutableList.of(SUMMARY_QUERY_ALL),
ImmutableList.<String>of());
ImmutableList.of());
}
@Override
......@@ -80,7 +82,7 @@ public class GroupSummaryContext implements ServletContextListener {
/**
* Static method to allow us to update the summary of a group
*
* @param sql the sql query to build
* @param sql the sql query to build
* @param params the parameters to pass along to the query
*/
private static void updateSummaries(List<String> sql, List<String> params) {
......@@ -98,7 +100,7 @@ public class GroupSummaryContext implements ServletContextListener {
i++;
}
List<GroupSummary> results = (List<GroupSummary>)groupSummary.getResultList();
List<GroupSummary> results = (List<GroupSummary>) groupSummary.getResultList();
for (GroupSummary result : results) {
log.info("Result: group " + result.getGroup() + ", size: " + result.getSize() + ", count: " + result.getCount());
summaries.put(result.getGroup(), result);
......
......@@ -41,6 +41,7 @@ import org.apache.log4j.Logger;
import javax.persistence.EntityManager;
import javax.persistence.Query;
import javax.persistence.TypedQuery;
import javax.servlet.RequestDispatcher;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
......@@ -75,9 +76,9 @@ public class StatusServlet extends EntityManagerServlet {
private static final String PARAM_PAGE = "page";
// Search Params
private static final String PARAM_GROUP = "group";
private static final String PARAM_STATE = "state";
private static final String PARAM_COLLECTION_LIKE = "collection";
private static final String PARAM_GROUP = "status_group";
private static final String PARAM_STATE = "status_state";
private static final String PARAM_COLLECTION_LIKE = "status_collection";
private static final String PARAM_AUDIT_DATE = "audit";
// Filter params?
......@@ -103,8 +104,8 @@ public class StatusServlet extends EntityManagerServlet {
// local getParameter so that the session is checked as well
String group = getParameter(request, PARAM_GROUP, null);
String collection = getParameter(request, PARAM_COLLECTION_LIKE, null);
String state = getParameter(request, PARAM_STATE, null);
String collection = getParameter(request, PARAM_COLLECTION_LIKE, null);
// String date = getParameter(request, PARAM_GROUP, null);
PageBean pb = new PageBean((int) page, count, "");
......@@ -123,20 +124,20 @@ public class StatusServlet extends EntityManagerServlet {
// TODO: Can probably tidy this up a bit
if (!Strings.isEmpty(group)) {
queries.add("c.group LIKE :group");
queries.add("c.group LIKE :status_group");
pb.addParam(PARAM_GROUP, group);
request.setAttribute(PARAM_GROUP, group);
}
if (!Strings.isEmpty(collection)) {
queries.add("c.name LIKE :collection");
queries.add("c.name LIKE :status_collection");
pb.addParam(PARAM_COLLECTION_LIKE, collection);
request.setAttribute(PARAM_COLLECTION_LIKE, collection);
}
// Enforce that the state is not empty, or larger than 1 character
if (!Strings.isEmpty(state) && state.length() == 1) {
queries.add("c.state = :state");
queries.add("c.state = :status_state");
pb.addParam(PARAM_STATE, state);
request.setAttribute(PARAM_STATE, state);
}
......@@ -163,8 +164,8 @@ public class StatusServlet extends EntityManagerServlet {
queryString.append(" ORDER BY c.group ASC, c.name ASC");
countString.append(params);
Query query =
em.createQuery(queryString.toString());
TypedQuery<Collection> query =
em.createQuery(queryString.toString(), Collection.class);
// em.createNamedQuery("Collection.listAllCollections");
query.setFirstResult((int) offset);
query.setMaxResults(count);
......@@ -255,7 +256,7 @@ public class StatusServlet extends EntityManagerServlet {
col));
csb.setMissingFiles(CollectionCountContext.getMissingCount(col));
csb.setMissingTokens(CollectionCountContext.getMissingTokenCount(col));
csb.setTotalSize(CollectionCountContext.getTotelSize(col));
csb.setTotalSize(CollectionCountContext.getTotalSize(col));
csb.setTotalErrors(CollectionCountContext.getTotalErrors(col));
csb.setRemoteMissing(CollectionCountContext.getRemoteMissing(col));
csb.setRemoteCorrupt(CollectionCountContext.getRemoteCorrupt(col));
......
......@@ -120,22 +120,16 @@ public class BrowseServlet extends EntityManagerServlet {
request.setAttribute("collection", c);
session.setAttribute(SESSION_FILE,
loadFileBean(dt.getDirectoryNode(itemId), em,c));
if ( dt.getDirectoryNode(itemId).isDirectory() ) {
if (dt.getDirectoryNode(itemId) != null && dt.getDirectoryNode(itemId).isDirectory()) {
dt.toggleItem(itemId);
}
// else
// {
// session.setAttribute(SESSION_FILE,
// loadFileBean(dt.getDirectoryNode(itemId)));
// }
}
request.setAttribute(PAGE_ISAUDITING, isRunning);
RequestDispatcher dispatcher = request.getRequestDispatcher("browse.jsp");
dispatcher.forward(request, response);
}
private FileBean loadFileBean( DirectoryNode node, EntityManager em, Collection c ) {
private FileBean loadFileBean(DirectoryNode node, EntityManager em, Collection c) {
// avoid possible null references below
if (node == null) {
return null;
......@@ -144,20 +138,20 @@ public class BrowseServlet extends EntityManagerServlet {
FileBean retBean = new FileBean();
try {
MonitoredItem master = em.getReference(MonitoredItem.class,
node.getId());
MonitoredItem master = em.getReference(MonitoredItem.class, node.getId());
retBean.root = master;
retBean.name = node.getName();
if (master.getToken() != null) {
// TokenResponse resp = (TokenResponse)master.getToken().getToken();
MessageDigest digest = MessageDigest.getInstance(master.getToken().getProofAlgorithm());
MessageDigest digest = MessageDigest.getInstance(
master.getToken().getProofAlgorithm());
ProofValidator pv = new ProofValidator();
Proof proof = TokenUtil.extractProof(master.getToken());
String fileDigest = master.getFileDigest();
if ( fileDigest != null ){
byte[] root = pv.rootHash(digest, proof, HashValue.asBytes(master.getFileDigest()));
if (fileDigest != null){
byte[] root = pv.rootHash(digest, proof,
HashValue.asBytes(master.getFileDigest()));
retBean.itemProof = HashValue.asHexString(root);
}else {
retBean.itemProof = null;
......
......@@ -44,22 +44,23 @@ import org.apache.log4j.Logger;
import org.apache.log4j.NDC;
import javax.persistence.EntityManager;
import javax.persistence.EntityTransaction;
import javax.persistence.NoResultException;
import javax.persistence.Query;
import javax.persistence.TypedQuery;
import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import java.util.List;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import java.util.stream.Collectors;
/**
* Set the IMS for the AuditThread to use. Also, startup a background task
* to handle firing off monitor tasks.
*
* <p>
* TODO: Handle token validation
*
*
* @author toaster
*/
public final class AuditConfigurationContext implements ServletContextListener {
......@@ -77,174 +78,97 @@ public final class AuditConfigurationContext implements ServletContextListener {
private static final Logger LOG = Logger.getLogger(AuditConfigurationContext.class);
@Override
public void contextInitialized( ServletContextEvent arg0 ) {
public void contextInitialized(ServletContextEvent arg0) {
EntityManager em = PersistUtil.getEntityManager();
Query q = em.createNamedQuery("SettingsParameter.getAttr");
SettingsParameter s = null;
ServletContext ctx = arg0.getServletContext();
// set IMS for audit Thread from server parameter
q.setParameter("attr", SettingsConstants.PARAM_IMS);
s = (SettingsParameter) q.getSingleResult();
AuditThreadFactory.setIMS(s.getValue());
if ( Strings.isEmpty(AuditThreadFactory.getIMS()) ) {
throw new RuntimeException("IMS is empty");
}
q.setParameter("attr", SettingsConstants.PARAM_IMS_TOKEN_CLASS);
s = (SettingsParameter) q.getSingleResult();
if ( !Strings.isEmpty(s.getValue()) ) {
String tokenClass = s.getValue();
AuditThreadFactory.setTokenClass(tokenClass);
}
q.setParameter("attr", SettingsConstants.PARAM_IMS_PORT);
s = (SettingsParameter) q.getSingleResult();
if ( Strings.isValidInt(s.getValue()) ) {
int port = Integer.parseInt(s.getValue());
if ( port > 1 && port < 32768 ) {
AuditThreadFactory.setImsPort(port);
} else {
throw new RuntimeException("ims.port must be between 1 and 32768");
}
}
q.setParameter("attr", SettingsConstants.PARAM_AUDIT_ONLY);
try {
s = (SettingsParameter) q.getSingleResult();
AuditThreadFactory.setAuditOnly(Boolean.valueOf(s.getValue()));
}catch ( NoResultException ex ) {
EntityTransaction trans = em.getTransaction();
trans.begin();
em.persist(new SettingsParameter(SettingsConstants.PARAM_AUDIT_ONLY,
SettingsConstants.auditOnly, false));
trans.commit();
AuditThreadFactory.setAuditOnly(false);
}
q.setParameter("attr", SettingsConstants.PARAM_AUDIT_SAMPLE);
try {
s = (SettingsParameter) q.getSingleResult();
AuditThreadFactory.setAuditSampling(Boolean.valueOf(s.getValue()));
} catch (NoResultException ex) {
EntityTransaction trans = em.getTransaction();
trans.begin();
em.persist(new SettingsParameter(SettingsConstants.PARAM_AUDIT_SAMPLE,
SettingsConstants.auditSample, false));
trans.commit();
AuditThreadFactory.setAuditSampling(false);
}
q.setParameter("attr", SettingsConstants.PARAM_IMS_SSL);
try {
s = (SettingsParameter) q.getSingleResult();
AuditThreadFactory.setSSL(Boolean.valueOf(s.getValue()));
} catch (NoResultException ex) {
EntityTransaction trans = em.getTransaction();
trans.begin();
em.persist(new SettingsParameter(SettingsConstants.PARAM_IMS_SSL,
SettingsConstants.imsSSL, false));
trans.commit();
AuditThreadFactory.setSSL(false);
}
q.setParameter("attr", SettingsConstants.PARAM_AUDIT_BLOCKING);
try {
s = (SettingsParameter) q.getSingleResult();
AuditThreadFactory.setBlocking(Boolean.valueOf(s.getValue()));
} catch (NoResultException ex) {
EntityTransaction trans = em.getTransaction();
trans.begin();
em.persist(new SettingsParameter(SettingsConstants.PARAM_AUDIT_BLOCKING,
SettingsConstants.auditBlocking,
false));
trans.commit();
AuditThreadFactory.setBlocking(false);
}
q.setParameter("attr", SettingsConstants.PARAM_AUDIT_MAX_BLOCK_TIME);
try {
s = (SettingsParameter) q.getSingleResult();
int blockTime = 0;
String val = s.getValue();
if (Strings.isValidInt(val)) {
blockTime = Integer.parseInt(val);
}
// Just in case...
if ( blockTime < 0 ) {
blockTime = 0;
}
AuditThreadFactory.setMaxBlockTime(blockTime);
} catch (NoResultException ex) {
EntityTransaction trans = em.getTransaction();
trans.begin();
em.persist(new SettingsParameter(SettingsConstants.PARAM_AUDIT_MAX_BLOCK_TIME,
SettingsConstants.auditMaxBlockTime,
false));
trans.commit();
AuditThreadFactory.setBlocking(false);
TypedQuery<SettingsParameter> nq = em.createNamedQuery(
"SettingsParameter.getCurrentSettings",
SettingsParameter.class);
List<SettingsParameter> resultList = nq.getResultList();
Map<String, String> resultMap = resultList.stream()
.collect(Collectors.toMap((SettingsParameter::getName),
(SettingsParameter::getValue)));
em.close();
if (resultList.isEmpty()) {
SettingsUtil.updateSettings(SettingsUtil.getDefaultSettings());
}
// PauseBean first because of dependencies
ServletContext ctx = arg0.getServletContext();
PauseBean pb = new PauseBean();
ctx.setAttribute(ATTRIBUTE_PAUSE, pb);
q.setParameter("attr", SettingsConstants.PARAM_AUTO_AUDIT_ENABLE);
s = (SettingsParameter) q.getSingleResult();
// Invert the boolean because the PB checks if we're paused, not enabled
String enableAudits = s.getValue();
String enableAudits = resultMap.getOrDefault(
SettingsConstants.PARAM_AUTO_AUDIT_ENABLE,
SettingsConstants.autoAudit);
pb.setPaused(!Boolean.valueOf(enableAudits));
q.setParameter("attr", SettingsConstants.PARAM_THROTTLE_MAXAUDIT);
s = (SettingsParameter) q.getSingleResult();
String maxRun = s.getValue();
if ( Strings.isValidInt(maxRun) ) {
int audit = Integer.parseInt(maxRun);
if ( audit > 0 ) {
AuditThreadFactory.setMaxAudits(audit);
}
}
// q.setParam(attr, continuous audit)
// s = q.getSingleResult
// seed = s.getValue
//bgAudit = new BackgroundAuditorFactory();
//BackgroundAuditorFactory.start();
em.close();
checkTimer = new Timer("Audit Check Timer");
checkTimer.schedule(new MyTimerTask(pb), 0, HOUR);
}
// set IMS for audit Thread from server parameter
AuditThreadFactory.setIMS(resultMap.getOrDefault(
SettingsConstants.PARAM_IMS,
SettingsConstants.ims));
String tokenClass = resultMap.getOrDefault(
SettingsConstants.PARAM_IMS_TOKEN_CLASS,
SettingsConstants.imsTokenClass);
AuditThreadFactory.setTokenClass(tokenClass);
String port = resultMap.getOrDefault(
SettingsConstants.PARAM_IMS_PORT,
SettingsConstants.imsPort);
if (Strings.isValidInt(port)) {
AuditThreadFactory.setImsPort(Integer.parseInt(port));
}
/**