Commit 86061733 authored by Michael Ritter's avatar Michael Ritter

Merge branch 'release-1.14'

parents 1858eb18 1748b110
...@@ -18,7 +18,7 @@ ...@@ -18,7 +18,7 @@
variables: variables:
# This will supress any download for dependencies and plugins or upload messages which would clutter the console log. # This will supress any download for dependencies and plugins or upload messages which would clutter the console log.
# `showDateTime` will show the passed time in milliseconds. You need to specify `--batch-mode` to make this work. # `showDateTime` will show the passed time in milliseconds. You need to specify `--batch-mode` to make this work.
MAVEN_OPTS: "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Dorg.slf4j.simpleLogger.showDateTime=true -Djava.awt.headless=true" MAVEN_OPTS: "-Dmaven.repo.local=$CI_PROJECT_DIR/.m2/repository -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Dorg.slf4j.simpleLogger.showDateTime=true -Djava.awt.headless=true"
# As of Maven 3.3.0 instead of this you may define these options in `.mvn/maven.config` so the same config is used # As of Maven 3.3.0 instead of this you may define these options in `.mvn/maven.config` so the same config is used
# when running from the command line. # when running from the command line.
# `installAtEnd` and `deployAtEnd`are only effective with recent version of the corresponding plugins. # `installAtEnd` and `deployAtEnd`are only effective with recent version of the corresponding plugins.
...@@ -27,63 +27,42 @@ variables: ...@@ -27,63 +27,42 @@ variables:
# Cache downloaded dependencies and plugins between builds. # Cache downloaded dependencies and plugins between builds.
cache: cache:
paths: paths:
- /root/.m2/repository/ - .m2/repository/
# This will only validate and compile stuff and run e.g. maven-enforcer-plugin. image: maven:3-jdk-8
# Because some enforcer rules might check dependency convergence and class duplications
# we use `test-compile` here instead of `validate`, so the correct classpath is picked up.
.validate: &validate
stage: build
script:
- 'mvn $MAVEN_CLI_OPTS test-compile'
# For merge requests do not `deploy` but only run `verify`.
# See https://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html
.verify: &verify
stage: test
script:
- 'mvn $MAVEN_CLI_OPTS verify'
except:
- master
# Validate merge requests using JDK8 # Validate merge requests using JDK8
validate:jdk8: validate:jdk8:
<<: *validate stage: build
image: maven:3.3.9-jdk-8 script:
- mvn $MAVEN_CLI_OPTS test-compile
# Verify merge requests using JDK8 # Verify merge requests using JDK8
verify:jdk8: verify:jdk8:
<<: *verify stage: test
image: maven:3.3.9-jdk-8 script:
- mvn $MAVEN_CLI_OPTS verify
# For `master` branch run `mvn deploy` automatically. # Call mvn deploy to upload artifacts for devlop and releases
# Here you need to decide whether you want to use JDK7 or 8. # Requires settings.xml to be mounted in /root/.m2/settings.xml
# To get this working you need to define a volume while configuring your gitlab-ci-multi-runner.
# Mount your `settings.xml` as `/root/.m2/settings.xml` which holds your secrets.
# See https://maven.apache.org/settings.html
deploy:jdk8: deploy:jdk8:
# Use stage test here, so the pages job may later pickup the created site. stage: deploy
stage: test
script: script:
- 'mvn $MAVEN_CLI_OPTS deploy' - mvn $MAVEN_CLI_OPTS deploy
only: only:
- master - develop
# Archive up the built documentation site. - tags
image: maven:3.3.9-jdk-8 - /^release-.*$/
upload: upload:
image: maven:3.3.9-jdk-8
stage: deploy stage: deploy
only: only:
- master
- develop - develop
- tags
- /^release-.*$/
script: script:
- 'mvn package' - 'mvn package'
- 'for f in `find ace-am/src/main/sql -type f`; do curl -X POST -H "Authorization: token $BUILD_TOKEN" --data-binary @$f $SERVER/artifacts/ace/$CI_BUILD_REF_NAME/$(echo $CI_BUILD_REF | cut -c -7)/$(basename $f); done' - 'for f in `find ace-am/src/main/sql -type f`; do curl -X POST -H "Content-Type: application/octet-stream" -H "Authorization: $BUILD_TOKEN" --data-binary @$f $SERVER/resource/ace/$CI_BUILD_REF_NAME/$(echo $CI_BUILD_REF | cut -c -7)/$(basename $f); done'
- 'for f in `find ace-am/target -type f -name "*.war"`; do curl -X POST -H "Authorization: token $BUILD_TOKEN" --data-binary @$f $SERVER/artifacts/ace/$CI_BUILD_REF_NAME/$(echo $CI_BUILD_REF | cut -c -7)/$(basename $f); done' - 'for f in `find ace-am/target -type f -name "*.war"`; do curl -X POST -H "Content-Type: application/octet-stream" -H "Authorization: $BUILD_TOKEN" --data-binary @$f $SERVER/resource/ace/$CI_BUILD_REF_NAME/$(echo $CI_BUILD_REF | cut -c -7)/$(basename $f); done'
- 'for f in `find ace-ims-ear/target -type f -name "*.ear"`; do curl -X POST -H "Authorization: token $BUILD_TOKEN" --data-binary @$f $SERVER/artifacts/ace/$CI_BUILD_REF_NAME/$(echo $CI_BUILD_REF | cut -c -7)/$(basename $f); done' - 'for f in `find ace-ims-ear/target -type f -name "*.ear"`; do curl -X POST -H "Content-Type: application/octet-stream" -H "Authorization: $BUILD_TOKEN" --data-binary @$f $SERVER/resource/ace/$CI_BUILD_REF_NAME/$(echo $CI_BUILD_REF | cut -c -7)/$(basename $f); done'
# release:
# only:
# - master
...@@ -4,8 +4,7 @@ ...@@ -4,8 +4,7 @@
<parent> <parent>
<artifactId>ace</artifactId> <artifactId>ace</artifactId>
<groupId>edu.umiacs.ace</groupId> <groupId>edu.umiacs.ace</groupId>
<version>1.13-RELEASE</version> <version>1.14-RELEASE</version>
<relativePath>../pom.xml</relativePath>
</parent> </parent>
<artifactId>ace-am</artifactId> <artifactId>ace-am</artifactId>
<name>ace-am</name> <name>ace-am</name>
...@@ -21,6 +20,7 @@ ...@@ -21,6 +20,7 @@
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-war-plugin</artifactId> <artifactId>maven-war-plugin</artifactId>
<version>3.2.2</version>
<configuration> <configuration>
<warSourceExcludes>**/footer.jsp</warSourceExcludes> <warSourceExcludes>**/footer.jsp</warSourceExcludes>
<webResources> <webResources>
...@@ -31,13 +31,6 @@ ...@@ -31,13 +31,6 @@
<include>**/footer.jsp</include> <include>**/footer.jsp</include>
</includes> </includes>
</resource> </resource>
<!--<resource>
<directory>src/main/webapp</directory>
<filtering>false</filtering>
<excludes>
<exclude>**/footer.jsp</exclude>
</excludes>
</resource>-->
</webResources> </webResources>
</configuration> </configuration>
</plugin> </plugin>
...@@ -111,7 +104,7 @@ ...@@ -111,7 +104,7 @@
<scope>provided</scope> <scope>provided</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>jstl</groupId> <groupId>javax.servlet</groupId>
<artifactId>jstl</artifactId> <artifactId>jstl</artifactId>
<version>1.1.2</version> <version>1.1.2</version>
</dependency> </dependency>
...@@ -202,34 +195,8 @@ ...@@ -202,34 +195,8 @@
<artifactId>irods-api</artifactId> <artifactId>irods-api</artifactId>
<version>1.6</version> <version>1.6</version>
</dependency> </dependency>
<!--<dependency>
<groupId>org.apache.jackrabbit</groupId> <!-- web services -->
<artifactId>jackrabbit-core</artifactId>
<version>2.2.7</version>
<exclusions>
<exclusion>
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>jcl-over-slf4j</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
</exclusions>
</dependency> -->
<!-- web services
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-server</artifactId>
<version>1.6</version>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
<version>1.6</version>
</dependency>
-->
<dependency> <dependency>
<groupId>org.glassfish.jersey.containers</groupId> <groupId>org.glassfish.jersey.containers</groupId>
<artifactId>jersey-container-servlet</artifactId> <artifactId>jersey-container-servlet</artifactId>
...@@ -240,12 +207,6 @@ ...@@ -240,12 +207,6 @@
<artifactId>jersey-media-json-jackson</artifactId> <artifactId>jersey-media-json-jackson</artifactId>
<version>2.24</version> <version>2.24</version>
</dependency> </dependency>
<dependency>
<groupId>edu.umiacs.ace</groupId>
<artifactId>ace-ims-ws</artifactId>
<version>1.13-RELEASE</version>
<type>jar</type>
</dependency>
<dependency> <dependency>
<groupId>org.apache.httpcomponents</groupId> <groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId> <artifactId>httpclient</artifactId>
......
...@@ -34,22 +34,23 @@ import edu.umiacs.ace.driver.AuditIterable; ...@@ -34,22 +34,23 @@ import edu.umiacs.ace.driver.AuditIterable;
import edu.umiacs.ace.driver.DriverStateBean; import edu.umiacs.ace.driver.DriverStateBean;
import edu.umiacs.ace.driver.DriverStateBean.State; import edu.umiacs.ace.driver.DriverStateBean.State;
import edu.umiacs.ace.driver.FileBean; import edu.umiacs.ace.driver.FileBean;
import edu.umiacs.ace.driver.filter.PathFilter;
import edu.umiacs.ace.driver.StorageDriver;
import edu.umiacs.ace.monitor.core.MonitoredItem;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.driver.QueryThrottle; import edu.umiacs.ace.driver.QueryThrottle;
import edu.umiacs.ace.driver.StateBeanDigestListener; import edu.umiacs.ace.driver.StateBeanDigestListener;
import edu.umiacs.ace.driver.StorageDriver;
import edu.umiacs.ace.driver.filter.PathFilter;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.core.MonitoredItem;
import edu.umiacs.ace.util.HashValue; import edu.umiacs.ace.util.HashValue;
import edu.umiacs.ace.util.ThreadedDigestStream; import edu.umiacs.ace.util.ThreadedDigestStream;
import edu.umiacs.ace.util.ThrottledInputStream; import edu.umiacs.ace.util.ThrottledInputStream;
import edu.umiacs.io.IO;
import edu.umiacs.util.Strings; import edu.umiacs.util.Strings;
import org.apache.log4j.Logger;
import javax.persistence.EntityManager;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.security.DigestInputStream;
import java.security.MessageDigest; import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException; import java.security.NoSuchAlgorithmException;
import java.util.ArrayList; import java.util.ArrayList;
...@@ -58,12 +59,10 @@ import java.util.LinkedList; ...@@ -58,12 +59,10 @@ import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Queue; import java.util.Queue;
import javax.persistence.EntityManager;
import org.apache.log4j.Logger;
/** /**
* Storage driver for accessing files stored on a local file system (ie, java.io.File) * Storage driver for accessing files stored on a local file system (ie, java.io.File)
* *
* @author toaster * @author toaster
*/ */
public class LocalFileAccess extends StorageDriver { public class LocalFileAccess extends StorageDriver {
...@@ -138,18 +137,19 @@ public class LocalFileAccess extends StorageDriver { ...@@ -138,18 +137,19 @@ public class LocalFileAccess extends StorageDriver {
class MyIterator implements Iterator<FileBean> { class MyIterator implements Iterator<FileBean> {
private FileBean next; private FileBean next;
private Queue<File> dirsToProcess = new LinkedList<File>(); private Queue<File> dirsToProcess = new LinkedList<>();
private Queue<File> filesToProcess = new LinkedList<File>(); private Queue<File> filesToProcess = new LinkedList<>();
private MessageDigest digest; private MessageDigest digest;
// private byte[] buffer = new byte[BLOCK_SIZE];
private File rootFile; private File rootFile;
private PathFilter filter; private PathFilter filter;
private DriverStateBean statebean; private DriverStateBean statebean;
private ThreadedDigestStream reader; private ThreadedDigestStream reader;
private boolean cancel = false; private boolean cancel = false;
public MyIterator(MonitoredItem[] startPath, PathFilter filter, public MyIterator(MonitoredItem[] startPath,
String digestAlgorithm, DriverStateBean statebean) { PathFilter filter,
String digestAlgorithm,
DriverStateBean statebean) {
this.statebean = statebean; this.statebean = statebean;
this.filter = filter; this.filter = filter;
try { try {
...@@ -163,8 +163,7 @@ public class LocalFileAccess extends StorageDriver { ...@@ -163,8 +163,7 @@ public class LocalFileAccess extends StorageDriver {
if (startPath != null) { if (startPath != null) {
for (MonitoredItem mi : startPath) { for (MonitoredItem mi : startPath) {
File startFile; File startFile;
startFile = new File( startFile = new File(getCollection().getDirectory() + mi.getPath());
getCollection().getDirectory() + mi.getPath());
if (startFile.isDirectory()) { if (startFile.isDirectory()) {
dirsToProcess.add(startFile); dirsToProcess.add(startFile);
...@@ -224,13 +223,13 @@ public class LocalFileAccess extends StorageDriver { ...@@ -224,13 +223,13 @@ public class LocalFileAccess extends StorageDriver {
if (fileList == null) { if (fileList == null) {
LOG.info("Could not read directory, skipping: " + directory); LOG.info("Could not read directory, skipping: " + directory);
} else { } else {
for (File f : directory.listFiles()) { for (File f : fileList) {
LOG.trace("Found item " + f); LOG.trace("Found item " + f);
if ( f.isDirectory() && if ( f.isDirectory() &&
filter.process(extractPathList(f), true)) { filter.process(extractPathList(f), true)) {
LOG.trace("Adding matching directory: " + f); LOG.trace("Adding matching directory: " + f);
dirsToProcess.add(f); dirsToProcess.add(f);
} else if ( f.isFile() && } else if ( f.isFile() &&
filter.process(extractPathList(f), false)) { filter.process(extractPathList(f), false)) {
LOG.trace("Adding matching file: " + f); LOG.trace("Adding matching file: " + f);
filesToProcess.add(f); filesToProcess.add(f);
...@@ -254,11 +253,9 @@ public class LocalFileAccess extends StorageDriver { ...@@ -254,11 +253,9 @@ public class LocalFileAccess extends StorageDriver {
int substrLength = rootFile.getPath().length(); int substrLength = rootFile.getPath().length();
// build directory path // build directory path
List<String> dirPathList = new ArrayList<String>(); List<String> dirPathList = new ArrayList<>();
File currFile = file; File currFile = file;
while (!currFile.equals(rootFile)) { while (!currFile.equals(rootFile)) {
// LOG.trace("Adding dir to path: " + currFile.getPath().substring(
// substrLength));
String pathToAdd = currFile.getPath().substring(substrLength); String pathToAdd = currFile.getPath().substring(substrLength);
pathToAdd = pathToAdd.replace(File.separatorChar, '/'); pathToAdd = pathToAdd.replace(File.separatorChar, '/');
dirPathList.add(pathToAdd); dirPathList.add(pathToAdd);
...@@ -269,11 +266,7 @@ public class LocalFileAccess extends StorageDriver { ...@@ -269,11 +266,7 @@ public class LocalFileAccess extends StorageDriver {
@SuppressWarnings("empty-statement") @SuppressWarnings("empty-statement")
private FileBean processFile(File file) { private FileBean processFile(File file) {
DigestInputStream dis = null;
FileBean fb = new FileBean(); FileBean fb = new FileBean();
fb.setPathList(extractPathList(file)); fb.setPathList(extractPathList(file));
LOG.trace("Processing file: " + file); LOG.trace("Processing file: " + file);
...@@ -307,7 +300,6 @@ public class LocalFileAccess extends StorageDriver { ...@@ -307,7 +300,6 @@ public class LocalFileAccess extends StorageDriver {
fb.setError(true); fb.setError(true);
fb.setErrorMessage(Strings.exceptionAsString(ie)); fb.setErrorMessage(Strings.exceptionAsString(ie));
} finally { } finally {
IO.release(dis);
statebean.setStateAndReset(State.IDLE); statebean.setStateAndReset(State.IDLE);
if (cancel) { if (cancel) {
return null; return null;
......
...@@ -36,7 +36,9 @@ import edu.umiacs.ace.util.EntityManagerServlet; ...@@ -36,7 +36,9 @@ import edu.umiacs.ace.util.EntityManagerServlet;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import javax.persistence.EntityManager; import javax.persistence.EntityManager;
import javax.persistence.NoResultException;
import javax.persistence.Query; import javax.persistence.Query;
import javax.persistence.TypedQuery;
import javax.servlet.RequestDispatcher; import javax.servlet.RequestDispatcher;
import javax.servlet.ServletException; import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
...@@ -47,7 +49,7 @@ import java.util.List; ...@@ -47,7 +49,7 @@ import java.util.List;
/** /**
* Show the current status of a collection, listing any non-active files * Show the current status of a collection, listing any non-active files
* *
* @author toaster * @author toaster
*/ */
public class CollectionSummaryServlet extends EntityManagerServlet { public class CollectionSummaryServlet extends EntityManagerServlet {
...@@ -63,15 +65,16 @@ public class CollectionSummaryServlet extends EntityManagerServlet { ...@@ -63,15 +65,16 @@ public class CollectionSummaryServlet extends EntityManagerServlet {
public static final String PAGE_COUNT = "count"; public static final String PAGE_COUNT = "count";
public static final String PAGE_SESSION = "session"; public static final String PAGE_SESSION = "session";
/** /**
* Processes requests for both HTTP <code>GET</code> and <code>POST</code> methods. * Processes requests for both HTTP <code>GET</code> and <code>POST</code> methods.
* @param request servlet request *
* @param request servlet request
* @param response servlet response * @param response servlet response
*/ */
@Override @Override
protected void processRequest( HttpServletRequest request, protected void processRequest(HttpServletRequest request,
HttpServletResponse response, EntityManager em ) HttpServletResponse response,
throws ServletException, IOException { EntityManager em) throws ServletException, IOException {
RequestDispatcher dispatch; RequestDispatcher dispatch;
long startid = getParameter(request, PARAM_START, 0); long startid = getParameter(request, PARAM_START, 0);
long topid = getParameter(request, PARAM_TOP, 0); long topid = getParameter(request, PARAM_TOP, 0);
...@@ -84,41 +87,47 @@ public class CollectionSummaryServlet extends EntityManagerServlet { ...@@ -84,41 +87,47 @@ public class CollectionSummaryServlet extends EntityManagerServlet {
// determine start // determine start
String startquery = ""; String startquery = "";
if ( startid > 0 ) { if (startid > 0) {
startquery = "AND m.id > :id "; startquery = "AND m.id > :id ";
} else if ( topid > 0 ) { } else if (topid > 0) {
startquery = "AND m.id < :id "; startquery = "AND m.id < :id ";
} }
// build query // build query
Query q; TypedQuery<MonitoredItem> q;
Collection c = getCollection(request, em); Collection c = getCollection(request, em);
// todo: if null should we return all error'd items from the db?
// probably a bad idea for this servlet
if (c == null) {
response.sendError(400, "collectionid must be present!");
return;
}
query += "m.parentCollection = :coll " + startquery; query += "m.parentCollection = :coll " + startquery;
q = em.createQuery(query); q = em.createQuery(query, MonitoredItem.class);
q.setParameter("coll", c); q.setParameter("coll", c);
LOG.debug("query: " + query); LOG.debug("query: " + query);
// fill in start ids if necessary // fill in start ids if necessary
if ( startid > 0 ) { if (startid > 0) {
q.setParameter("id", startid); q.setParameter("id", startid);
} else if ( topid > 0 ) { } else if (topid > 0) {
q.setParameter("id", topid); q.setParameter("id", topid);
} }
if ( count != -1 ) { if (count != -1) {
q.setMaxResults(count); q.setMaxResults(count);
} }
List<MonitoredItem> miList = q.getResultList(); List<MonitoredItem> miList = q.getResultList();
if ( miList != null && miList.size() > 0 ) { if (miList != null && miList.size() > 0) {
request.setAttribute(PAGE_NEXT, request.setAttribute(PAGE_NEXT, miList.get(miList.size() - 1).getId() + 1);
miList.get(miList.size() - 1).getId() + 1);
} }
if ( listOnly ) { if (listOnly) {
Writer writer = response.getWriter(); Writer writer = response.getWriter();
response.setContentType("text/plain"); response.setContentType("text/plain");
for ( MonitoredItem mi : miList ) { for (MonitoredItem mi : miList) {
writer.write(mi.getState() + ":" + mi.getPath() + "\r\n"); writer.write(mi.getState() + ":" + mi.getPath() + "\r\n");
} }
return; return;
...@@ -132,8 +141,7 @@ public class CollectionSummaryServlet extends EntityManagerServlet { ...@@ -132,8 +141,7 @@ public class CollectionSummaryServlet extends EntityManagerServlet {
csb.setTotalFiles(CollectionCountContext.getFileCount(c)); csb.setTotalFiles(CollectionCountContext.getFileCount(c));
csb.setActiveFiles(CollectionCountContext.getActiveCount(c)); csb.setActiveFiles(CollectionCountContext.getActiveCount(c));
csb.setCorruptFiles(CollectionCountContext.getCorruptCount(c)); csb.setCorruptFiles(CollectionCountContext.getCorruptCount(c));
csb.setInvalidDigests(CollectionCountContext.getTokenMismatchCount( csb.setInvalidDigests(CollectionCountContext.getTokenMismatchCount(c));
c));
csb.setMissingFiles(CollectionCountContext.getMissingCount(c)); csb.setMissingFiles(CollectionCountContext.getMissingCount(c));
csb.setMissingTokens(CollectionCountContext.getMissingTokenCount(c)); csb.setMissingTokens(CollectionCountContext.getMissingTokenCount(c));
csb.setTotalErrors(CollectionCountContext.getTotalErrors(c)); csb.setTotalErrors(CollectionCountContext.getTotalErrors(c));
...@@ -145,7 +153,7 @@ public class CollectionSummaryServlet extends EntityManagerServlet { ...@@ -145,7 +153,7 @@ public class CollectionSummaryServlet extends EntityManagerServlet {
request.setAttribute(PAGE_COLLECTION, csb); request.setAttribute(PAGE_COLLECTION, csb);
request.setAttribute(PAGE_COUNT, count); request.setAttribute(PAGE_COUNT, count);