Skip to content

GitLab

Commit 86061733 authored by Michael Ritter's avatar Michael Ritter
Browse files

Merge branch 'release-1.14'

parents 1858eb18 1748b110
Hide whitespace changes
Inline Side-by-side
Showing with 515 additions and 624 deletions
.gitlab-ci.yml
View file @ 86061733
......@@ -18,7 +18,7 @@
variables:
# This will supress any download for dependencies and plugins or upload messages which would clutter the console log.
# `showDateTime` will show the passed time in milliseconds. You need to specify `--batch-mode` to make this work.
MAVEN_OPTS: "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Dorg.slf4j.simpleLogger.showDateTime=true -Djava.awt.headless=true"
MAVEN_OPTS: "-Dmaven.repo.local=$CI_PROJECT_DIR/.m2/repository -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Dorg.slf4j.simpleLogger.showDateTime=true -Djava.awt.headless=true"
# As of Maven 3.3.0 instead of this you may define these options in `.mvn/maven.config` so the same config is used
# when running from the command line.
# `installAtEnd` and `deployAtEnd`are only effective with recent version of the corresponding plugins.
......@@ -27,63 +27,42 @@ variables:
# Cache downloaded dependencies and plugins between builds.
cache:
paths:
- /root/.m2/repository/
- .m2/repository/
# This will only validate and compile stuff and run e.g. maven-enforcer-plugin.
# Because some enforcer rules might check dependency convergence and class duplications
# we use `test-compile` here instead of `validate`, so the correct classpath is picked up.
.validate: &validate
stage: build
script:
- 'mvn $MAVEN_CLI_OPTS test-compile'
# For merge requests do not `deploy` but only run `verify`.
# See https://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html
.verify: &verify
stage: test
script:
- 'mvn $MAVEN_CLI_OPTS verify'
except:
- master
image: maven:3-jdk-8
# Validate merge requests using JDK8
validate:jdk8:
<<: *validate
image: maven:3.3.9-jdk-8
stage: build
script:
- mvn $MAVEN_CLI_OPTS test-compile
# Verify merge requests using JDK8
verify:jdk8:
<<: *verify
image: maven:3.3.9-jdk-8
stage: test
script:
- mvn $MAVEN_CLI_OPTS verify
# For `master` branch run `mvn deploy` automatically.
# Here you need to decide whether you want to use JDK7 or 8.
# To get this working you need to define a volume while configuring your gitlab-ci-multi-runner.
# Mount your `settings.xml` as `/root/.m2/settings.xml` which holds your secrets.
# See https://maven.apache.org/settings.html
# Call mvn deploy to upload artifacts for devlop and releases
# Requires settings.xml to be mounted in /root/.m2/settings.xml
deploy:jdk8:
# Use stage test here, so the pages job may later pickup the created site.
stage: test
stage: deploy
script:
- 'mvn $MAVEN_CLI_OPTS deploy'
- mvn $MAVEN_CLI_OPTS deploy
only:
- master
# Archive up the built documentation site.
image: maven:3.3.9-jdk-8
- develop
- tags
- /^release-.*$/
upload:
image: maven:3.3.9-jdk-8
stage: deploy
only:
- master
- develop
- tags
- /^release-.*$/
script:
- 'mvn package'
- 'for f in `find ace-am/src/main/sql -type f`; do curl -X POST -H "Authorization: token $BUILD_TOKEN" --data-binary @$f $SERVER/artifacts/ace/$CI_BUILD_REF_NAME/$(echo $CI_BUILD_REF | cut -c -7)/$(basename $f); done'
- 'for f in `find ace-am/target -type f -name "*.war"`; do curl -X POST -H "Authorization: token $BUILD_TOKEN" --data-binary @$f $SERVER/artifacts/ace/$CI_BUILD_REF_NAME/$(echo $CI_BUILD_REF | cut -c -7)/$(basename $f); done'
- 'for f in `find ace-ims-ear/target -type f -name "*.ear"`; do curl -X POST -H "Authorization: token $BUILD_TOKEN" --data-binary @$f $SERVER/artifacts/ace/$CI_BUILD_REF_NAME/$(echo $CI_BUILD_REF | cut -c -7)/$(basename $f); done'
- 'for f in `find ace-am/src/main/sql -type f`; do curl -X POST -H "Content-Type: application/octet-stream" -H "Authorization: $BUILD_TOKEN" --data-binary @$f $SERVER/resource/ace/$CI_BUILD_REF_NAME/$(echo $CI_BUILD_REF | cut -c -7)/$(basename $f); done'
- 'for f in `find ace-am/target -type f -name "*.war"`; do curl -X POST -H "Content-Type: application/octet-stream" -H "Authorization: $BUILD_TOKEN" --data-binary @$f $SERVER/resource/ace/$CI_BUILD_REF_NAME/$(echo $CI_BUILD_REF | cut -c -7)/$(basename $f); done'
- 'for f in `find ace-ims-ear/target -type f -name "*.ear"`; do curl -X POST -H "Content-Type: application/octet-stream" -H "Authorization: $BUILD_TOKEN" --data-binary @$f $SERVER/resource/ace/$CI_BUILD_REF_NAME/$(echo $CI_BUILD_REF | cut -c -7)/$(basename $f); done'
# release:
# only:
# - master
ace-am/pom.xml
View file @ 86061733
......@@ -4,8 +4,7 @@
<parent>
<artifactId>ace</artifactId>
<groupId>edu.umiacs.ace</groupId>
<version>1.13-RELEASE</version>
<relativePath>../pom.xml</relativePath>
<version>1.14-RELEASE</version>
</parent>
<artifactId>ace-am</artifactId>
<name>ace-am</name>
......@@ -21,6 +20,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-war-plugin</artifactId>
<version>3.2.2</version>
<configuration>
<warSourceExcludes>**/footer.jsp</warSourceExcludes>
<webResources>
......@@ -31,13 +31,6 @@
<include>**/footer.jsp</include>
</includes>
</resource>
<!--<resource>
<directory>src/main/webapp</directory>
<filtering>false</filtering>
<excludes>
<exclude>**/footer.jsp</exclude>
</excludes>
</resource>-->
</webResources>
</configuration>
</plugin>
......@@ -111,7 +104,7 @@
<scope>provided</scope>
</dependency>
<dependency>
<groupId>jstl</groupId>
<groupId>javax.servlet</groupId>
<artifactId>jstl</artifactId>
<version>1.1.2</version>
</dependency>
......@@ -202,34 +195,8 @@
<artifactId>irods-api</artifactId>
<version>1.6</version>
</dependency>
<!--<dependency>
<groupId>org.apache.jackrabbit</groupId>
<artifactId>jackrabbit-core</artifactId>
<version>2.2.7</version>
<exclusions>
<exclusion>
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>jcl-over-slf4j</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
</exclusions>
</dependency> -->
<!-- web services
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-server</artifactId>
<version>1.6</version>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
<version>1.6</version>
</dependency>
-->
<!-- web services -->
<dependency>
<groupId>org.glassfish.jersey.containers</groupId>
<artifactId>jersey-container-servlet</artifactId>
......@@ -240,12 +207,6 @@
<artifactId>jersey-media-json-jackson</artifactId>
<version>2.24</version>
</dependency>
<dependency>
<groupId>edu.umiacs.ace</groupId>
<artifactId>ace-ims-ws</artifactId>
<version>1.13-RELEASE</version>
<type>jar</type>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
......
ace-am/src/main/java/edu/umiacs/ace/driver/localfile/LocalFileAccess.java
View file @ 86061733
......@@ -34,22 +34,23 @@ import edu.umiacs.ace.driver.AuditIterable;
import edu.umiacs.ace.driver.DriverStateBean;
import edu.umiacs.ace.driver.DriverStateBean.State;
import edu.umiacs.ace.driver.FileBean;
import edu.umiacs.ace.driver.filter.PathFilter;
import edu.umiacs.ace.driver.StorageDriver;
import edu.umiacs.ace.monitor.core.MonitoredItem;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.driver.QueryThrottle;
import edu.umiacs.ace.driver.StateBeanDigestListener;
import edu.umiacs.ace.driver.StorageDriver;
import edu.umiacs.ace.driver.filter.PathFilter;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.core.MonitoredItem;
import edu.umiacs.ace.util.HashValue;
import edu.umiacs.ace.util.ThreadedDigestStream;
import edu.umiacs.ace.util.ThrottledInputStream;
import edu.umiacs.io.IO;
import edu.umiacs.util.Strings;
import org.apache.log4j.Logger;
import javax.persistence.EntityManager;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
......@@ -58,12 +59,10 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import javax.persistence.EntityManager;
import org.apache.log4j.Logger;
/**
* Storage driver for accessing files stored on a local file system (ie, java.io.File)
*
*
* @author toaster
*/
public class LocalFileAccess extends StorageDriver {
......@@ -138,18 +137,19 @@ public class LocalFileAccess extends StorageDriver {
class MyIterator implements Iterator<FileBean> {
private FileBean next;
private Queue<File> dirsToProcess = new LinkedList<File>();
private Queue<File> filesToProcess = new LinkedList<File>();
private Queue<File> dirsToProcess = new LinkedList<>();
private Queue<File> filesToProcess = new LinkedList<>();
private MessageDigest digest;
// private byte[] buffer = new byte[BLOCK_SIZE];
private File rootFile;
private PathFilter filter;
private DriverStateBean statebean;
private ThreadedDigestStream reader;
private boolean cancel = false;
public MyIterator(MonitoredItem[] startPath, PathFilter filter,
String digestAlgorithm, DriverStateBean statebean) {
public MyIterator(MonitoredItem[] startPath,
PathFilter filter,
String digestAlgorithm,
DriverStateBean statebean) {
this.statebean = statebean;
this.filter = filter;
try {
......@@ -163,8 +163,7 @@ public class LocalFileAccess extends StorageDriver {
if (startPath != null) {
for (MonitoredItem mi : startPath) {
File startFile;
startFile = new File(
getCollection().getDirectory() + mi.getPath());
startFile = new File(getCollection().getDirectory() + mi.getPath());
if (startFile.isDirectory()) {
dirsToProcess.add(startFile);
......@@ -224,13 +223,13 @@ public class LocalFileAccess extends StorageDriver {
if (fileList == null) {
LOG.info("Could not read directory, skipping: " + directory);
} else {
for (File f : directory.listFiles()) {
for (File f : fileList) {
LOG.trace("Found item " + f);
if ( f.isDirectory() &&
if ( f.isDirectory() &&
filter.process(extractPathList(f), true)) {
LOG.trace("Adding matching directory: " + f);
dirsToProcess.add(f);
} else if ( f.isFile() &&
} else if ( f.isFile() &&
filter.process(extractPathList(f), false)) {
LOG.trace("Adding matching file: " + f);
filesToProcess.add(f);
......@@ -254,11 +253,9 @@ public class LocalFileAccess extends StorageDriver {
int substrLength = rootFile.getPath().length();
// build directory path
List<String> dirPathList = new ArrayList<String>();
List<String> dirPathList = new ArrayList<>();
File currFile = file;
while (!currFile.equals(rootFile)) {
// LOG.trace("Adding dir to path: " + currFile.getPath().substring(
// substrLength));
String pathToAdd = currFile.getPath().substring(substrLength);
pathToAdd = pathToAdd.replace(File.separatorChar, '/');
dirPathList.add(pathToAdd);
......@@ -269,11 +266,7 @@ public class LocalFileAccess extends StorageDriver {
@SuppressWarnings("empty-statement")
private FileBean processFile(File file) {
DigestInputStream dis = null;
FileBean fb = new FileBean();
fb.setPathList(extractPathList(file));
LOG.trace("Processing file: " + file);
......@@ -307,7 +300,6 @@ public class LocalFileAccess extends StorageDriver {
fb.setError(true);
fb.setErrorMessage(Strings.exceptionAsString(ie));
} finally {
IO.release(dis);
statebean.setStateAndReset(State.IDLE);
if (cancel) {
return null;
......
ace-am/src/main/java/edu/umiacs/ace/monitor/access/CollectionSummaryServlet.java
View file @ 86061733
......@@ -36,7 +36,9 @@ import edu.umiacs.ace.util.EntityManagerServlet;
import org.apache.log4j.Logger;
import javax.persistence.EntityManager;
import javax.persistence.NoResultException;
import javax.persistence.Query;
import javax.persistence.TypedQuery;
import javax.servlet.RequestDispatcher;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
......@@ -47,7 +49,7 @@ import java.util.List;
/**
* Show the current status of a collection, listing any non-active files
*
*
* @author toaster
*/
public class CollectionSummaryServlet extends EntityManagerServlet {
......@@ -63,15 +65,16 @@ public class CollectionSummaryServlet extends EntityManagerServlet {
public static final String PAGE_COUNT = "count";
public static final String PAGE_SESSION = "session";
/**
/**
* Processes requests for both HTTP <code>GET</code> and <code>POST</code> methods.
* @param request servlet request
*
* @param request servlet request
* @param response servlet response
*/
@Override
protected void processRequest( HttpServletRequest request,
HttpServletResponse response, EntityManager em )
throws ServletException, IOException {
protected void processRequest(HttpServletRequest request,
HttpServletResponse response,
EntityManager em) throws ServletException, IOException {
RequestDispatcher dispatch;
long startid = getParameter(request, PARAM_START, 0);
long topid = getParameter(request, PARAM_TOP, 0);
......@@ -84,41 +87,47 @@ public class CollectionSummaryServlet extends EntityManagerServlet {
// determine start
String startquery = "";
if ( startid > 0 ) {
if (startid > 0) {
startquery = "AND m.id > :id ";
} else if ( topid > 0 ) {
} else if (topid > 0) {
startquery = "AND m.id < :id ";
}
// build query
Query q;
TypedQuery<MonitoredItem> q;
Collection c = getCollection(request, em);
// todo: if null should we return all error'd items from the db?
// probably a bad idea for this servlet
if (c == null) {
response.sendError(400, "collectionid must be present!");
return;
}
query += "m.parentCollection = :coll " + startquery;
q = em.createQuery(query);
q = em.createQuery(query, MonitoredItem.class);
q.setParameter("coll", c);
LOG.debug("query: " + query);
// fill in start ids if necessary
if ( startid > 0 ) {
if (startid > 0) {
q.setParameter("id", startid);
} else if ( topid > 0 ) {
} else if (topid > 0) {
q.setParameter("id", topid);
}
if ( count != -1 ) {
if (count != -1) {
q.setMaxResults(count);
}
List<MonitoredItem> miList = q.getResultList();
if ( miList != null && miList.size() > 0 ) {
request.setAttribute(PAGE_NEXT,
miList.get(miList.size() - 1).getId() + 1);
if (miList != null && miList.size() > 0) {
request.setAttribute(PAGE_NEXT, miList.get(miList.size() - 1).getId() + 1);
}
if ( listOnly ) {
if (listOnly) {
Writer writer = response.getWriter();
response.setContentType("text/plain");
for ( MonitoredItem mi : miList ) {
for (MonitoredItem mi : miList) {
writer.write(mi.getState() + ":" + mi.getPath() + "\r\n");
}
return;
......@@ -132,8 +141,7 @@ public class CollectionSummaryServlet extends EntityManagerServlet {
csb.setTotalFiles(CollectionCountContext.getFileCount(c));
csb.setActiveFiles(CollectionCountContext.getActiveCount(c));
csb.setCorruptFiles(CollectionCountContext.getCorruptCount(c));
csb.setInvalidDigests(CollectionCountContext.getTokenMismatchCount(
c));
csb.setInvalidDigests(CollectionCountContext.getTokenMismatchCount(c));
csb.setMissingFiles(CollectionCountContext.getMissingCount(c));
csb.setMissingTokens(CollectionCountContext.getMissingTokenCount(c));
csb.setTotalErrors(CollectionCountContext.getTotalErrors(c));
......@@ -145,7 +153,7 @@ public class CollectionSummaryServlet extends EntityManagerServlet {
request.setAttribute(PAGE_COLLECTION, csb);
request.setAttribute(PAGE_COUNT, count);
if ( hasJson(request) ) {
if (hasJson(request)) {
dispatch = request.getRequestDispatcher("report-json.jsp");
} else {
dispatch = request.getRequestDispatcher("report.jsp");
......@@ -160,6 +168,11 @@ public class CollectionSummaryServlet extends EntityManagerServlet {
Query q = em.createQuery(query);
q.setParameter("c", c);
q.setMaxResults(1);
return (Long) q.getSingleResult();
try {
return (Long) q.getSingleResult();
} catch (NoResultException e) {
LOG.warn("No sessions available for collection", e);
return null;
}
}
}
ace-am/src/main/java/edu/umiacs/ace/monitor/access/ListItemDetailsServlet.java
View file @ 86061733
......@@ -30,23 +30,24 @@
// $Id$
package edu.umiacs.ace.monitor.access;
import edu.umiacs.ace.util.EntityManagerServlet;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.core.MonitoredItem;
import edu.umiacs.ace.monitor.core.MonitoredItemManager;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.util.EntityManagerServlet;
import edu.umiacs.util.Strings;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import javax.persistence.EntityManager;
import javax.servlet.RequestDispatcher;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
/**
* List details for a file or directory
*
*
* @author toaster
*/
public final class ListItemDetailsServlet extends EntityManagerServlet {
......@@ -56,30 +57,30 @@ public final class ListItemDetailsServlet extends EntityManagerServlet {
private static final String PARAM_PATH = "itempath";
@Override
protected void processRequest( HttpServletRequest request,
HttpServletResponse response, EntityManager em ) throws ServletException, IOException {
protected void processRequest(HttpServletRequest request,
HttpServletResponse response,
EntityManager em) throws ServletException, IOException {
Collection coll = getCollection(request, em);
String path = getParameter(request, PARAM_PATH, "");
MonitoredItemManager mim = new MonitoredItemManager(em);
if ( coll == null ) {
throw new ServletException("No collection found");
if (coll == null) {
response.sendError(400, "collectionid must be present");
return;
}
if ( Strings.isEmpty(path) ) {
if (Strings.isEmpty(path)) {
request.setAttribute(PAGE_ITEMS, mim.getCollectionRoots(coll));
} else {
MonitoredItem item = mim.getItemByPath(path, coll);
if ( item == null ) {
throw new ServletException("No item for path " + path);
if (item == null) {
response.sendError(400, "No item for path " + path);
return;
}
List<MonitoredItem> childItems;
if ( item.isDirectory() ) {
childItems = mim.listChildren(item.getParentCollection(),
item.getPath());
if (item.isDirectory()) {
childItems = mim.listChildren(item.getParentCollection(), item.getPath());
} else {
childItems = Collections.emptyList();
}
......@@ -88,13 +89,12 @@ public final class ListItemDetailsServlet extends EntityManagerServlet {
}
RequestDispatcher dispatcher;
if ( hasJson(request) ) {
if (hasJson(request)) {
dispatcher = request.getRequestDispatcher("listitem-json.jsp");
} else {
dispatcher = request.getRequestDispatcher("listitem.jsp");
}
dispatcher.forward(request, response);
dispatcher.forward(request, response);
}
}
ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditConfigurationContext.java
View file @ 86061733
......@@ -55,6 +55,8 @@ import java.util.Timer;
import java.util.TimerTask;
import java.util.stream.Collectors;
import static edu.umiacs.ace.monitor.settings.SettingsConstants.*;
/**
* Set the IMS for the AuditThread to use. Also, startup a background task
* to handle firing off monitor tasks.
......@@ -65,13 +67,6 @@ import java.util.stream.Collectors;
*/
public final class AuditConfigurationContext implements ServletContextListener {
//private static final String PARAM_IMS = "ims";
//private static final String PARAM_IMS_PORT = "ims.port";
//private static final String PARAM_IMS_TOKEN_CLASS = "ims.tokenclass";
//private static final String PARAM_IMS_SSL = "ims.ssl";
//private static final String PARAM_DISABLE_AUTO_AUDIT = "auto.audit.disable";
//private static final String PARAM_THROTTLE_MAXAUDIT = "throttle.maxaudit";
//private static final String PARAM_AUDIT_ONLY = "audit.only";
public static final String ATTRIBUTE_PAUSE = "pause";
private static final long HOUR = 1000 * 60 * 60;
private Timer checkTimer;
......@@ -98,68 +93,51 @@ public final class AuditConfigurationContext implements ServletContextListener {
ctx.setAttribute(ATTRIBUTE_PAUSE, pb);
// Invert the boolean because the PB checks if we're paused, not enabled
String enableAudits = resultMap.getOrDefault(
SettingsConstants.PARAM_AUTO_AUDIT_ENABLE,
SettingsConstants.autoAudit);
String enableAudits = resultMap.getOrDefault(PARAM_AUTO_AUDIT_ENABLE, autoAudit);
pb.setPaused(!Boolean.valueOf(enableAudits));
checkTimer = new Timer("Audit Check Timer");
checkTimer.schedule(new MyTimerTask(pb), 0, HOUR);
// set IMS for audit Thread from server parameter
AuditThreadFactory.setIMS(resultMap.getOrDefault(
SettingsConstants.PARAM_IMS,
SettingsConstants.ims));
AuditThreadFactory.setIMS(resultMap.getOrDefault(PARAM_IMS, ims));
String tokenClass = resultMap.getOrDefault(
SettingsConstants.PARAM_IMS_TOKEN_CLASS,
SettingsConstants.imsTokenClass);
String tokenClass = resultMap.getOrDefault(PARAM_IMS_TOKEN_CLASS, imsTokenClass);
AuditThreadFactory.setTokenClass(tokenClass);
String port = resultMap.getOrDefault(
SettingsConstants.PARAM_IMS_PORT,
SettingsConstants.imsPort);
String port = resultMap.getOrDefault(PARAM_IMS_PORT, imsPort);
if (Strings.isValidInt(port)) {
AuditThreadFactory.setImsPort(Integer.parseInt(port));
}
String auditOnly = resultMap.getOrDefault(
SettingsConstants.PARAM_AUDIT_ONLY,
SettingsConstants.auditOnly);
String auditOnly = resultMap.getOrDefault(PARAM_AUDIT_ONLY,