Skip to content

GitLab

Commit e8d0f78a authored by Michael Ritter's avatar Michael Ritter
Browse files

Merge branch 'release-1.12'

parents 429b42ec fb81788f
Hide whitespace changes
Inline Side-by-side
Showing with 539 additions and 121 deletions
.gitlab-ci.yml 0 → 100644
View file @ e8d0f78a
---
# This file is a template, and might need editing before it works on your project.
# Build JAVA applications using Apache Maven (http://maven.apache.org)
# For docker image tags see https://hub.docker.com/_/maven/
#
# For general lifecycle information see https://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html
#
# This template will build and test your projects as well as create the documentation.
#
# * Caches downloaded dependencies and plugins between invocation.
# * Does only verify merge requests but deploy built artifacts of the
# master branch.
# * Shows how to use multiple jobs in test stage for verifying functionality
# with multiple JDKs.
# * Uses site:stage to collect the documentation for multi-module projects.
# * Publishes the documentation for `master` branch.
variables:
# This will supress any download for dependencies and plugins or upload messages which would clutter the console log.
# `showDateTime` will show the passed time in milliseconds. You need to specify `--batch-mode` to make this work.
MAVEN_OPTS: "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Dorg.slf4j.simpleLogger.showDateTime=true -Djava.awt.headless=true"
# As of Maven 3.3.0 instead of this you may define these options in `.mvn/maven.config` so the same config is used
# when running from the command line.
# `installAtEnd` and `deployAtEnd`are only effective with recent version of the corresponding plugins.
MAVEN_CLI_OPTS: "--batch-mode --errors --fail-at-end --show-version -DinstallAtEnd=true -DdeployAtEnd=true"
# Cache downloaded dependencies and plugins between builds.
cache:
paths:
- /root/.m2/repository/
# This will only validate and compile stuff and run e.g. maven-enforcer-plugin.
# Because some enforcer rules might check dependency convergence and class duplications
# we use `test-compile` here instead of `validate`, so the correct classpath is picked up.
.validate: &validate
stage: build
script:
- 'mvn $MAVEN_CLI_OPTS test-compile'
# For merge requests do not `deploy` but only run `verify`.
# See https://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html
.verify: &verify
stage: test
script:
- 'mvn $MAVEN_CLI_OPTS verify'
except:
- master
# Validate merge requests using JDK8
validate:jdk8:
<<: *validate
image: maven:3.3.9-jdk-8
# Verify merge requests using JDK8
verify:jdk8:
<<: *verify
image: maven:3.3.9-jdk-8
# For `master` branch run `mvn deploy` automatically.
# Here you need to decide whether you want to use JDK7 or 8.
# To get this working you need to define a volume while configuring your gitlab-ci-multi-runner.
# Mount your `settings.xml` as `/root/.m2/settings.xml` which holds your secrets.
# See https://maven.apache.org/settings.html
deploy:jdk8:
# Use stage test here, so the pages job may later pickup the created site.
stage: test
script:
- 'mvn $MAVEN_CLI_OPTS deploy'
only:
- master
# Archive up the built documentation site.
image: maven:3.3.9-jdk-8
upload:
image: maven:3.3.9-jdk-8
stage: deploy
only:
- master
- develop
script:
- 'mvn package'
- 'for f in `find ace-am/src/main/sql -type f`; do curl -X POST -H "Authorization: token $BUILD_TOKEN" --data-binary @$f $SERVER/artifacts/ace/$CI_BUILD_REF_NAME/$(echo $CI_BUILD_REF | cut -c -7)/$(basename $f); done'
- 'for f in `find ace-am/target -type f -name "*.war"`; do curl -X POST -H "Authorization: token $BUILD_TOKEN" --data-binary @$f $SERVER/artifacts/ace/$CI_BUILD_REF_NAME/$(echo $CI_BUILD_REF | cut -c -7)/$(basename $f); done'
- 'for f in `find ace-ims-ear/target -type f -name "*.ear"`; do curl -X POST -H "Authorization: token $BUILD_TOKEN" --data-binary @$f $SERVER/artifacts/ace/$CI_BUILD_REF_NAME/$(echo $CI_BUILD_REF | cut -c -7)/$(basename $f); done'
# release:
# only:
# - master
ace-am/pom.xml
View file @ e8d0f78a
......@@ -4,7 +4,7 @@
<parent>
<artifactId>ace</artifactId>
<groupId>edu.umiacs.ace</groupId>
<version>1.11-RELEASE</version>
<version>1.12-RELEASE</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>ace-am</artifactId>
......@@ -41,6 +41,14 @@
</webResources>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<!-- debian package made easy -->
<!--
<plugin>
......@@ -268,7 +276,7 @@
<dependency>
<groupId>edu.umiacs.ace</groupId>
<artifactId>ace-ims-ws</artifactId>
<version>1.11-RELEASE</version>
<version>1.12-RELEASE</version>
<type>jar</type>
</dependency>
<dependency>
......
ace-am/src/main/java/edu/umiacs/ace/monitor/access/CollectionCountContext.java
View file @ e8d0f78a
......@@ -44,8 +44,9 @@ import javax.sql.DataSource;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
......@@ -62,16 +63,16 @@ public class CollectionCountContext implements ServletContextListener {
public static final String CTX_STARTUP = "startup_complete";
private static final Logger LOG = Logger.getLogger(
CollectionCountContext.class);
private static Map<Collection, Long> fileCountMap = new HashMap<Collection, Long>();
private static Map<Collection, Long> fileActiveMap = new HashMap<Collection, Long>();
private static Map<Collection, Long> fileCorruptMap = new HashMap<Collection, Long>();
private static Map<Collection, Long> fileMissingMap = new HashMap<Collection, Long>();
private static Map<Collection, Long> fileMissingTokenMap = new HashMap<Collection, Long>();
private static Map<Collection, Long> fileTokenMismatchMap = new HashMap<Collection, Long>();
private static Map<Collection, Long> totalErrorMap = new HashMap<Collection, Long>();
private static Map<Collection, Long> totalSizeMap = new HashMap<Collection, Long>();
private static Map<Collection, Long> fileRemoteMissing = new HashMap<Collection, Long>();
private static Map<Collection, Long> fileRemoteCorrupt = new HashMap<Collection, Long>();
private static Map<Collection, Long> fileCountMap = new ConcurrentHashMap<>();
private static Map<Collection, Long> fileActiveMap = new ConcurrentHashMap<>();
private static Map<Collection, Long> fileCorruptMap = new ConcurrentHashMap<>();
private static Map<Collection, Long> fileMissingMap = new ConcurrentHashMap<>();
private static Map<Collection, Long> fileMissingTokenMap = new ConcurrentHashMap<>();
private static Map<Collection, Long> fileTokenMismatchMap = new ConcurrentHashMap<>();
private static Map<Collection, Long> totalErrorMap = new ConcurrentHashMap<>();
private static Map<Collection, Long> totalSizeMap = new ConcurrentHashMap<>();
private static Map<Collection, Long> fileRemoteMissing = new ConcurrentHashMap<>();
private static Map<Collection, Long> fileRemoteCorrupt = new ConcurrentHashMap<>();
private static AtomicInteger totalCollections = new AtomicInteger(0);
private static Lock lock = new ReentrantLock();
private static boolean abort = false;
......@@ -216,8 +217,20 @@ public class CollectionCountContext implements ServletContextListener {
totalCollections.incrementAndGet();
}
public static void decrementTotalCollections() {
public static void decrementTotalCollections(Collection collection) {
totalCollections.decrementAndGet();
fileCountMap.remove(collection);
fileActiveMap.remove(collection);
fileCorruptMap.remove(collection);
fileMissingMap.remove(collection);
fileMissingTokenMap.remove(collection);
fileTokenMismatchMap.remove(collection);
totalErrorMap.remove(collection);
totalSizeMap.remove(collection);
fileRemoteMissing.remove(collection);
fileRemoteCorrupt.remove(collection);
GroupSummaryContext.updateGroup(collection.getGroup());
}
public static int getTotalCollections() {
......@@ -226,10 +239,11 @@ public class CollectionCountContext implements ServletContextListener {
/**
* Update statistics for a collection.
*
*
* @param c
*/
private static void queryCollection(Collection c) {
private static boolean queryCollection(Collection c) {
boolean update = false;
Connection connection = null;
PreparedStatement ps = null;
ResultSet rs = null;
......@@ -249,14 +263,13 @@ public class CollectionCountContext implements ServletContextListener {
while (rs.next()) {
if (abort) {
return;
return update;
}
char state = rs.getString(1).charAt(0);
long count = rs.getLong(2);
total += count;
switch (state) {
case 'A':
fileActiveMap.put(c, count);
......@@ -288,7 +301,7 @@ public class CollectionCountContext implements ServletContextListener {
}
}
fileCountMap.put(c, total);
update = !Objects.equals(fileCountMap.put(c, total), total);
totalErrorMap.put(c, totalErrors);
SQL.release(rs);
SQL.release(ps);
......@@ -303,8 +316,7 @@ public class CollectionCountContext implements ServletContextListener {
rs = ps.executeQuery();
rs.next();
long totalSize = rs.getLong(1);
totalSizeMap.put(c, totalSize);
update = update | !Objects.equals(totalSizeMap.put(c, totalSize), totalSize);
} catch (Exception e) {
LOG.error("Error starting up, collection count", e);
} finally {
......@@ -312,8 +324,9 @@ public class CollectionCountContext implements ServletContextListener {
SQL.release(ps);
SQL.release(connection);
LOG.trace("Finished count on " + c.getName());
}
return update;
}
public static void updateCollection(final Collection c) {
......@@ -325,7 +338,10 @@ public class CollectionCountContext implements ServletContextListener {
lock.lock();
try {
queryCollection(c);
boolean update = queryCollection(c);
if (update) {
GroupSummaryContext.updateGroup(c.getGroup());
}
} finally {
lock.unlock();
}
......
ace-am/src/main/java/edu/umiacs/ace/monitor/access/CollectionSummaryServlet.java
View file @ e8d0f78a
......@@ -30,19 +30,20 @@
// $Id$
package edu.umiacs.ace.monitor.access;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.core.MonitoredItem;
import edu.umiacs.ace.util.EntityManagerServlet;
import edu.umiacs.ace.monitor.core.Collection;
import java.io.IOException;
import java.io.Writer;
import java.util.List;
import org.apache.log4j.Logger;
import javax.persistence.EntityManager;
import javax.persistence.Query;
import javax.servlet.RequestDispatcher;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.log4j.Logger;
import java.io.IOException;
import java.io.Writer;
import java.util.List;
/**
* Show the current status of a collection, listing any non-active files
......@@ -60,6 +61,7 @@ public class CollectionSummaryServlet extends EntityManagerServlet {
public static final String PAGE_COLLECTION = "collection";
public static final String PAGE_ITEMS = "items";
public static final String PAGE_COUNT = "count";
public static final String PAGE_SESSION = "session";
/**
* Processes requests for both HTTP <code>GET</code> and <code>POST</code> methods.
......@@ -122,6 +124,9 @@ public class CollectionSummaryServlet extends EntityManagerServlet {
return;
}
// get latest session
Long session = getRecentSession(c, em);
CollectionSummaryBean csb = new CollectionSummaryBean();
csb.setCollection(c);
csb.setTotalFiles(CollectionCountContext.getFileCount(c));
......@@ -135,6 +140,7 @@ public class CollectionSummaryServlet extends EntityManagerServlet {
csb.setRemoteMissing(CollectionCountContext.getRemoteMissing(c));
csb.setRemoteCorrupt(CollectionCountContext.getRemoteCorrupt(c));
request.setAttribute(PAGE_SESSION, session);
request.setAttribute(PAGE_ITEMS, miList);
request.setAttribute(PAGE_COLLECTION, csb);
request.setAttribute(PAGE_COUNT, count);
......@@ -148,4 +154,12 @@ public class CollectionSummaryServlet extends EntityManagerServlet {
dispatch.forward(request, response);
}
private Long getRecentSession(Collection c, EntityManager em) {
String query = "SELECT e.session FROM LogEvent e WHERE e.collection = :c ORDER BY e.id DESC";
Query q = em.createQuery(query);
q.setParameter("c", c);
q.setMaxResults(1);
return (Long) q.getSingleResult();
}
}
ace-am/src/main/java/edu/umiacs/ace/monitor/access/GroupSummary.java 0 → 100644
View file @ e8d0f78a
package edu.umiacs.ace.monitor.access;
import java.math.BigDecimal;
/**
*
* Created by shake on 4/4/17.
*/
public final class GroupSummary {
private final String group;
private final BigDecimal size;
private final BigDecimal count;
public GroupSummary(String group, BigDecimal size, BigDecimal count) {
this.group = group;
this.size = size;
this.count = count;
}
public String getGroup() {
return group;
}
public BigDecimal getSize() {
return size;
}
public BigDecimal getCount() {
return count;
}
}
ace-am/src/main/java/edu/umiacs/ace/monitor/access/GroupSummaryContext.java 0 → 100644
View file @ e8d0f78a
package edu.umiacs.ace.monitor.access;
import com.google.common.collect.ImmutableList;
import edu.umiacs.ace.util.PersistUtil;
import org.apache.log4j.Logger;
import javax.persistence.EntityManager;
import javax.persistence.Query;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* Servlet context listener which acts as a cache for Summary information about
* groups in ACE
*
* If there are many groups, it's conceivable that this could be a strain on memory
*
* Created by shake on 4/4/17.
*/
public class GroupSummaryContext implements ServletContextListener {
private static final Logger log = Logger.getLogger(GroupSummaryContext.class);
public static Map<String, GroupSummary> summaries;
/**
* Query to get ALL group summaries
*/
private static final String SUMMARY_QUERY_ALL = "SELECT c.colgroup, sum(m.size) AS size, sum(m.count) AS count " +
"FROM collection c " +
"JOIN ( " +
"SELECT sum(size) AS size, count(id) AS count, parentcollection_id " +
"FROM monitored_item " +
"WHERE directory = 0 " +
"GROUP BY parentcollection_id " +
") AS m ON c.id = m.parentcollection_id " +
"WHERE c.colgroup IS NOT NULL " +
"GROUP BY c.colgroup";
/**
* Query to get the group summary for a single group
*/
private static final String SUMMARY_QUERY_GROUP = "select c.colgroup, sum(m.size) AS size, count(m.id) AS count " +
"FROM monitored_item m " +
"JOIN ( " +
" select colgroup, id " +
" FROM collection " +
" WHERE colgroup = ? " +
") c ON c.id = m.parentcollection_id WHERE directory = 0";
@Override
public void contextInitialized(ServletContextEvent servletContextEvent) {
summaries = new HashMap<>();
updateSummaries(ImmutableList.of(SUMMARY_QUERY_ALL),
ImmutableList.<String>of());
}
@Override
public void contextDestroyed(ServletContextEvent servletContextEvent) {
summaries.clear();
}
/**
* Public api method to update a single group
*
* @param group the group to update
*/
@SuppressWarnings("WeakerAccess")
public static void updateGroup(String group) {
if (group != null) {
log.debug("Updating group summary for " + group);
updateSummaries(
ImmutableList.of(SUMMARY_QUERY_GROUP),
ImmutableList.of(group));
}
}
/**
* Static method to allow us to update the summary of a group
*
* @param sql the sql query to build
* @param params the parameters to pass along to the query
*/
private static void updateSummaries(List<String> sql, List<String> params) {
EntityManager em = PersistUtil.getEntityManager();
StringBuilder query = new StringBuilder();
for (String s : sql) {
query.append(s);
}
Query groupSummary = em.createNativeQuery(query.toString(), "GroupSummaryMapping");
int i = 1;
for (String param : params) {
groupSummary.setParameter(i, param);
i++;
}
List<GroupSummary> results = (List<GroupSummary>)groupSummary.getResultList();
for (GroupSummary result : results) {
log.info("Result: group " + result.getGroup() + ", size: " + result.getSize() + ", count: " + result.getCount());
summaries.put(result.getGroup(), result);
}
}
}
ace-am/src/main/java/edu/umiacs/ace/monitor/access/StatusServlet.java
View file @ e8d0f78a
......@@ -32,6 +32,7 @@ package edu.umiacs.ace.monitor.access;
import com.google.common.collect.ImmutableList;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.core.CollectionState;
import edu.umiacs.ace.monitor.support.CStateBean;
import edu.umiacs.ace.monitor.support.PageBean;
import edu.umiacs.ace.util.EntityManagerServlet;
......@@ -96,8 +97,6 @@ public class StatusServlet extends EntityManagerServlet {
RequestDispatcher dispatcher;
List<CollectionSummaryBean> collections;
List<Collection> items;
LOG.info("Processing /Status request");
LOG.info("Processing /Status additional info");
long page = getParameter(request, PARAM_PAGE, DEFAULT_PAGE);
int count = (int) getParameter(request, PARAM_COUNT, DEFAULT_COUNT);
......@@ -184,8 +183,8 @@ public class StatusServlet extends EntityManagerServlet {
}
if (!Strings.isEmpty(state) && state.length() == 1) {
query.setParameter(PARAM_STATE, state.charAt(0));
countQuery.setParameter(PARAM_STATE, state.charAt(0));
query.setParameter(PARAM_STATE, CollectionState.fromChar(state.charAt(0)));
countQuery.setParameter(PARAM_STATE, CollectionState.fromChar(state.charAt(0)));
}
items = query.getResultList();
......@@ -210,6 +209,7 @@ public class StatusServlet extends EntityManagerServlet {
request.setAttribute(PAGE_STATES, ImmutableList.copyOf(CStateBean.values()));
request.setAttribute(PAGE_COUNT, count);
request.setAttribute(PAGE_NUMBER, pb);
request.setAttribute("groups", GroupSummaryContext.summaries);
if (hasJson(request)) {
dispatcher = request.getRequestDispatcher("status-json.jsp");
} else if (hasCsv(request)) {
......
ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditConfigurationContext.java
View file @ e8d0f78a
......@@ -190,10 +190,11 @@ public final class AuditConfigurationContext implements ServletContextListener {
PauseBean pb = new PauseBean();
ctx.setAttribute(ATTRIBUTE_PAUSE, pb);
q.setParameter("attr", SettingsConstants.PARAM_DISABLE_AUTO_AUDIT);
q.setParameter("attr", SettingsConstants.PARAM_AUTO_AUDIT_ENABLE);
s = (SettingsParameter) q.getSingleResult();
String startPaused = s.getValue();
pb.setPaused(Boolean.valueOf(startPaused));
// Invert the boolean because the PB checks if we're paused, not enabled
String enableAudits = s.getValue();
pb.setPaused(!Boolean.valueOf(enableAudits));
q.setParameter("attr", SettingsConstants.PARAM_THROTTLE_MAXAUDIT);
s = (SettingsParameter) q.getSingleResult();
......
ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java
View file @ e8d0f78a
......@@ -45,6 +45,7 @@ import edu.umiacs.ace.monitor.access.CollectionCountContext;
import edu.umiacs.ace.monitor.compare.CollectionCompare2;
import edu.umiacs.ace.monitor.compare.CompareResults;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.core.CollectionState;
import edu.umiacs.ace.monitor.core.ConfigConstants;
import edu.umiacs.ace.monitor.core.MonitoredItem;
import edu.umiacs.ace.monitor.core.MonitoredItemManager;
......@@ -347,7 +348,6 @@ public final class AuditThread extends Thread implements CancelCallback {
private void performAudit() {
// 1. Setup audit
//TODO Clean up filtering creation
PathFilter filter = new SimpleFilter(coll);
Date startDate = new Date();
......@@ -399,6 +399,10 @@ public final class AuditThread extends Thread implements CancelCallback {
iterableItems.cancel();
}
// 4. Clean up, set local inactive
lastFileSeen = "looking for missed items";
setInactiveBefore(startDate);
if (cancel || abortException != null) {
return;
}
......@@ -411,14 +415,9 @@ public final class AuditThread extends Thread implements CancelCallback {
batch = null;
}
// 4. Clean up, set local inactive
lastFileSeen = "looking for missed items";
setInactiveBefore(startDate);
// harvest remote collections
lastFileSeen = "comparing to peer sites";
compareToPeers();
}
private void generateAuditReport() {
......@@ -450,7 +449,7 @@ public final class AuditThread extends Thread implements CancelCallback {
if (abortException instanceof InterruptedException
|| abortException.getCause() instanceof InterruptedException) {
LOG.trace("Audit ending with Interrupt");
logManager.persistCollectionEvent(LogEnum.FILE_AUDIT_FINISH,
logManager.persistCollectionEvent(LogEnum.FILE_AUDIT_ABORT,
"Audit Interrupted", em);
} else {
LOG.error("Uncaught exception in audit thread", abortException);
......@@ -458,12 +457,12 @@ public final class AuditThread extends Thread implements CancelCallback {
String message = Strings.exceptionAsString(abortException);
logManager.persistCollectionEvent(
LogEnum.SYSTEM_ERROR, message, em);
logManager.persistCollectionEvent(LogEnum.FILE_AUDIT_FINISH,
logManager.persistCollectionEvent(LogEnum.FILE_AUDIT_ABORT,
"Uncaught audit thread exception ", em);
}
} else if (cancel) {
logManager.persistCollectionEvent(LogEnum.FILE_AUDIT_FINISH,
logManager.persistCollectionEvent(LogEnum.FILE_AUDIT_CANCEL,
"Audit interrupted by user or token registration", em);
LOG.trace("Audit ending on cancel request");
} else {
......@@ -720,7 +719,8 @@ public final class AuditThread extends Thread implements CancelCallback {
EntityManager em = PersistUtil.getEntityManager();
if (baseItemPathList != null) {
if (coll.getState() == '\0' || coll.getState() == 'N') {
CollectionState state = coll.getStateEnum();
if (state == null || state == CollectionState.NEVER) {
return;
}
} else {
......@@ -729,10 +729,12 @@ public final class AuditThread extends Thread implements CancelCallback {
MonitoredItemManager mim = new MonitoredItemManager(em);