Commit e8d0f78a authored by Michael Ritter's avatar Michael Ritter
Browse files

Merge branch 'release-1.12'

parents 429b42ec fb81788f
---
# This file is a template, and might need editing before it works on your project.
# Build JAVA applications using Apache Maven (http://maven.apache.org)
# For docker image tags see https://hub.docker.com/_/maven/
#
# For general lifecycle information see https://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html
#
# This template will build and test your projects as well as create the documentation.
#
# * Caches downloaded dependencies and plugins between invocation.
# * Does only verify merge requests but deploy built artifacts of the
# master branch.
# * Shows how to use multiple jobs in test stage for verifying functionality
# with multiple JDKs.
# * Uses site:stage to collect the documentation for multi-module projects.
# * Publishes the documentation for `master` branch.
variables:
# This will supress any download for dependencies and plugins or upload messages which would clutter the console log.
# `showDateTime` will show the passed time in milliseconds. You need to specify `--batch-mode` to make this work.
MAVEN_OPTS: "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Dorg.slf4j.simpleLogger.showDateTime=true -Djava.awt.headless=true"
# As of Maven 3.3.0 instead of this you may define these options in `.mvn/maven.config` so the same config is used
# when running from the command line.
# `installAtEnd` and `deployAtEnd`are only effective with recent version of the corresponding plugins.
MAVEN_CLI_OPTS: "--batch-mode --errors --fail-at-end --show-version -DinstallAtEnd=true -DdeployAtEnd=true"
# Cache downloaded dependencies and plugins between builds.
cache:
paths:
- /root/.m2/repository/
# This will only validate and compile stuff and run e.g. maven-enforcer-plugin.
# Because some enforcer rules might check dependency convergence and class duplications
# we use `test-compile` here instead of `validate`, so the correct classpath is picked up.
.validate: &validate
stage: build
script:
- 'mvn $MAVEN_CLI_OPTS test-compile'
# For merge requests do not `deploy` but only run `verify`.
# See https://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html
.verify: &verify
stage: test
script:
- 'mvn $MAVEN_CLI_OPTS verify'
except:
- master
# Validate merge requests using JDK8
validate:jdk8:
<<: *validate
image: maven:3.3.9-jdk-8
# Verify merge requests using JDK8
verify:jdk8:
<<: *verify
image: maven:3.3.9-jdk-8
# For `master` branch run `mvn deploy` automatically.
# Here you need to decide whether you want to use JDK7 or 8.
# To get this working you need to define a volume while configuring your gitlab-ci-multi-runner.
# Mount your `settings.xml` as `/root/.m2/settings.xml` which holds your secrets.
# See https://maven.apache.org/settings.html
deploy:jdk8:
# Use stage test here, so the pages job may later pickup the created site.
stage: test
script:
- 'mvn $MAVEN_CLI_OPTS deploy'
only:
- master
# Archive up the built documentation site.
image: maven:3.3.9-jdk-8
upload:
image: maven:3.3.9-jdk-8
stage: deploy
only:
- master
- develop
script:
- 'mvn package'
- 'for f in `find ace-am/src/main/sql -type f`; do curl -X POST -H "Authorization: token $BUILD_TOKEN" --data-binary @$f $SERVER/artifacts/ace/$CI_BUILD_REF_NAME/$(echo $CI_BUILD_REF | cut -c -7)/$(basename $f); done'
- 'for f in `find ace-am/target -type f -name "*.war"`; do curl -X POST -H "Authorization: token $BUILD_TOKEN" --data-binary @$f $SERVER/artifacts/ace/$CI_BUILD_REF_NAME/$(echo $CI_BUILD_REF | cut -c -7)/$(basename $f); done'
- 'for f in `find ace-ims-ear/target -type f -name "*.ear"`; do curl -X POST -H "Authorization: token $BUILD_TOKEN" --data-binary @$f $SERVER/artifacts/ace/$CI_BUILD_REF_NAME/$(echo $CI_BUILD_REF | cut -c -7)/$(basename $f); done'
# release:
# only:
# - master
...@@ -4,7 +4,7 @@ ...@@ -4,7 +4,7 @@
<parent> <parent>
<artifactId>ace</artifactId> <artifactId>ace</artifactId>
<groupId>edu.umiacs.ace</groupId> <groupId>edu.umiacs.ace</groupId>
<version>1.11-RELEASE</version> <version>1.12-RELEASE</version>
<relativePath>../pom.xml</relativePath> <relativePath>../pom.xml</relativePath>
</parent> </parent>
<artifactId>ace-am</artifactId> <artifactId>ace-am</artifactId>
...@@ -41,6 +41,14 @@ ...@@ -41,6 +41,14 @@
</webResources> </webResources>
</configuration> </configuration>
</plugin> </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<!-- debian package made easy --> <!-- debian package made easy -->
<!-- <!--
<plugin> <plugin>
...@@ -268,7 +276,7 @@ ...@@ -268,7 +276,7 @@
<dependency> <dependency>
<groupId>edu.umiacs.ace</groupId> <groupId>edu.umiacs.ace</groupId>
<artifactId>ace-ims-ws</artifactId> <artifactId>ace-ims-ws</artifactId>
<version>1.11-RELEASE</version> <version>1.12-RELEASE</version>
<type>jar</type> <type>jar</type>
</dependency> </dependency>
<dependency> <dependency>
......
...@@ -44,8 +44,9 @@ import javax.sql.DataSource; ...@@ -44,8 +44,9 @@ import javax.sql.DataSource;
import java.sql.Connection; import java.sql.Connection;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantLock;
...@@ -62,16 +63,16 @@ public class CollectionCountContext implements ServletContextListener { ...@@ -62,16 +63,16 @@ public class CollectionCountContext implements ServletContextListener {
public static final String CTX_STARTUP = "startup_complete"; public static final String CTX_STARTUP = "startup_complete";
private static final Logger LOG = Logger.getLogger( private static final Logger LOG = Logger.getLogger(
CollectionCountContext.class); CollectionCountContext.class);
private static Map<Collection, Long> fileCountMap = new HashMap<Collection, Long>(); private static Map<Collection, Long> fileCountMap = new ConcurrentHashMap<>();
private static Map<Collection, Long> fileActiveMap = new HashMap<Collection, Long>(); private static Map<Collection, Long> fileActiveMap = new ConcurrentHashMap<>();
private static Map<Collection, Long> fileCorruptMap = new HashMap<Collection, Long>(); private static Map<Collection, Long> fileCorruptMap = new ConcurrentHashMap<>();
private static Map<Collection, Long> fileMissingMap = new HashMap<Collection, Long>(); private static Map<Collection, Long> fileMissingMap = new ConcurrentHashMap<>();
private static Map<Collection, Long> fileMissingTokenMap = new HashMap<Collection, Long>(); private static Map<Collection, Long> fileMissingTokenMap = new ConcurrentHashMap<>();
private static Map<Collection, Long> fileTokenMismatchMap = new HashMap<Collection, Long>(); private static Map<Collection, Long> fileTokenMismatchMap = new ConcurrentHashMap<>();
private static Map<Collection, Long> totalErrorMap = new HashMap<Collection, Long>(); private static Map<Collection, Long> totalErrorMap = new ConcurrentHashMap<>();
private static Map<Collection, Long> totalSizeMap = new HashMap<Collection, Long>(); private static Map<Collection, Long> totalSizeMap = new ConcurrentHashMap<>();
private static Map<Collection, Long> fileRemoteMissing = new HashMap<Collection, Long>(); private static Map<Collection, Long> fileRemoteMissing = new ConcurrentHashMap<>();
private static Map<Collection, Long> fileRemoteCorrupt = new HashMap<Collection, Long>(); private static Map<Collection, Long> fileRemoteCorrupt = new ConcurrentHashMap<>();
private static AtomicInteger totalCollections = new AtomicInteger(0); private static AtomicInteger totalCollections = new AtomicInteger(0);
private static Lock lock = new ReentrantLock(); private static Lock lock = new ReentrantLock();
private static boolean abort = false; private static boolean abort = false;
...@@ -216,8 +217,20 @@ public class CollectionCountContext implements ServletContextListener { ...@@ -216,8 +217,20 @@ public class CollectionCountContext implements ServletContextListener {
totalCollections.incrementAndGet(); totalCollections.incrementAndGet();
} }
public static void decrementTotalCollections() { public static void decrementTotalCollections(Collection collection) {
totalCollections.decrementAndGet(); totalCollections.decrementAndGet();
fileCountMap.remove(collection);
fileActiveMap.remove(collection);
fileCorruptMap.remove(collection);
fileMissingMap.remove(collection);
fileMissingTokenMap.remove(collection);
fileTokenMismatchMap.remove(collection);
totalErrorMap.remove(collection);
totalSizeMap.remove(collection);
fileRemoteMissing.remove(collection);
fileRemoteCorrupt.remove(collection);
GroupSummaryContext.updateGroup(collection.getGroup());
} }
public static int getTotalCollections() { public static int getTotalCollections() {
...@@ -226,10 +239,11 @@ public class CollectionCountContext implements ServletContextListener { ...@@ -226,10 +239,11 @@ public class CollectionCountContext implements ServletContextListener {
/** /**
* Update statistics for a collection. * Update statistics for a collection.
* *
* @param c * @param c
*/ */
private static void queryCollection(Collection c) { private static boolean queryCollection(Collection c) {
boolean update = false;
Connection connection = null; Connection connection = null;
PreparedStatement ps = null; PreparedStatement ps = null;
ResultSet rs = null; ResultSet rs = null;
...@@ -249,14 +263,13 @@ public class CollectionCountContext implements ServletContextListener { ...@@ -249,14 +263,13 @@ public class CollectionCountContext implements ServletContextListener {
while (rs.next()) { while (rs.next()) {
if (abort) { if (abort) {
return; return update;
} }
char state = rs.getString(1).charAt(0); char state = rs.getString(1).charAt(0);
long count = rs.getLong(2); long count = rs.getLong(2);
total += count; total += count;
switch (state) { switch (state) {
case 'A': case 'A':
fileActiveMap.put(c, count); fileActiveMap.put(c, count);
...@@ -288,7 +301,7 @@ public class CollectionCountContext implements ServletContextListener { ...@@ -288,7 +301,7 @@ public class CollectionCountContext implements ServletContextListener {
} }
} }
fileCountMap.put(c, total); update = !Objects.equals(fileCountMap.put(c, total), total);
totalErrorMap.put(c, totalErrors); totalErrorMap.put(c, totalErrors);
SQL.release(rs); SQL.release(rs);
SQL.release(ps); SQL.release(ps);
...@@ -303,8 +316,7 @@ public class CollectionCountContext implements ServletContextListener { ...@@ -303,8 +316,7 @@ public class CollectionCountContext implements ServletContextListener {
rs = ps.executeQuery(); rs = ps.executeQuery();
rs.next(); rs.next();
long totalSize = rs.getLong(1); long totalSize = rs.getLong(1);
totalSizeMap.put(c, totalSize); update = update | !Objects.equals(totalSizeMap.put(c, totalSize), totalSize);
} catch (Exception e) { } catch (Exception e) {
LOG.error("Error starting up, collection count", e); LOG.error("Error starting up, collection count", e);
} finally { } finally {
...@@ -312,8 +324,9 @@ public class CollectionCountContext implements ServletContextListener { ...@@ -312,8 +324,9 @@ public class CollectionCountContext implements ServletContextListener {
SQL.release(ps); SQL.release(ps);
SQL.release(connection); SQL.release(connection);
LOG.trace("Finished count on " + c.getName()); LOG.trace("Finished count on " + c.getName());
} }
return update;
} }
public static void updateCollection(final Collection c) { public static void updateCollection(final Collection c) {
...@@ -325,7 +338,10 @@ public class CollectionCountContext implements ServletContextListener { ...@@ -325,7 +338,10 @@ public class CollectionCountContext implements ServletContextListener {
lock.lock(); lock.lock();
try { try {
queryCollection(c); boolean update = queryCollection(c);
if (update) {
GroupSummaryContext.updateGroup(c.getGroup());
}
} finally { } finally {
lock.unlock(); lock.unlock();
} }
......
...@@ -30,19 +30,20 @@ ...@@ -30,19 +30,20 @@
// $Id$ // $Id$
package edu.umiacs.ace.monitor.access; package edu.umiacs.ace.monitor.access;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.core.MonitoredItem; import edu.umiacs.ace.monitor.core.MonitoredItem;
import edu.umiacs.ace.util.EntityManagerServlet; import edu.umiacs.ace.util.EntityManagerServlet;
import edu.umiacs.ace.monitor.core.Collection; import org.apache.log4j.Logger;
import java.io.IOException;
import java.io.Writer;
import java.util.List;
import javax.persistence.EntityManager; import javax.persistence.EntityManager;
import javax.persistence.Query; import javax.persistence.Query;
import javax.servlet.RequestDispatcher; import javax.servlet.RequestDispatcher;
import javax.servlet.ServletException; import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import org.apache.log4j.Logger; import java.io.IOException;
import java.io.Writer;
import java.util.List;
/** /**
* Show the current status of a collection, listing any non-active files * Show the current status of a collection, listing any non-active files
...@@ -60,6 +61,7 @@ public class CollectionSummaryServlet extends EntityManagerServlet { ...@@ -60,6 +61,7 @@ public class CollectionSummaryServlet extends EntityManagerServlet {
public static final String PAGE_COLLECTION = "collection"; public static final String PAGE_COLLECTION = "collection";
public static final String PAGE_ITEMS = "items"; public static final String PAGE_ITEMS = "items";
public static final String PAGE_COUNT = "count"; public static final String PAGE_COUNT = "count";
public static final String PAGE_SESSION = "session";
/** /**
* Processes requests for both HTTP <code>GET</code> and <code>POST</code> methods. * Processes requests for both HTTP <code>GET</code> and <code>POST</code> methods.
...@@ -122,6 +124,9 @@ public class CollectionSummaryServlet extends EntityManagerServlet { ...@@ -122,6 +124,9 @@ public class CollectionSummaryServlet extends EntityManagerServlet {
return; return;
} }
// get latest session
Long session = getRecentSession(c, em);
CollectionSummaryBean csb = new CollectionSummaryBean(); CollectionSummaryBean csb = new CollectionSummaryBean();
csb.setCollection(c); csb.setCollection(c);
csb.setTotalFiles(CollectionCountContext.getFileCount(c)); csb.setTotalFiles(CollectionCountContext.getFileCount(c));
...@@ -135,6 +140,7 @@ public class CollectionSummaryServlet extends EntityManagerServlet { ...@@ -135,6 +140,7 @@ public class CollectionSummaryServlet extends EntityManagerServlet {
csb.setRemoteMissing(CollectionCountContext.getRemoteMissing(c)); csb.setRemoteMissing(CollectionCountContext.getRemoteMissing(c));
csb.setRemoteCorrupt(CollectionCountContext.getRemoteCorrupt(c)); csb.setRemoteCorrupt(CollectionCountContext.getRemoteCorrupt(c));
request.setAttribute(PAGE_SESSION, session);
request.setAttribute(PAGE_ITEMS, miList); request.setAttribute(PAGE_ITEMS, miList);
request.setAttribute(PAGE_COLLECTION, csb); request.setAttribute(PAGE_COLLECTION, csb);
request.setAttribute(PAGE_COUNT, count); request.setAttribute(PAGE_COUNT, count);
...@@ -148,4 +154,12 @@ public class CollectionSummaryServlet extends EntityManagerServlet { ...@@ -148,4 +154,12 @@ public class CollectionSummaryServlet extends EntityManagerServlet {
dispatch.forward(request, response); dispatch.forward(request, response);
} }
private Long getRecentSession(Collection c, EntityManager em) {
String query = "SELECT e.session FROM LogEvent e WHERE e.collection = :c ORDER BY e.id DESC";
Query q = em.createQuery(query);
q.setParameter("c", c);
q.setMaxResults(1);
return (Long) q.getSingleResult();
}
} }
package edu.umiacs.ace.monitor.access;
import java.math.BigDecimal;
/**
*
* Created by shake on 4/4/17.
*/
public final class GroupSummary {
private final String group;
private final BigDecimal size;
private final BigDecimal count;
public GroupSummary(String group, BigDecimal size, BigDecimal count) {
this.group = group;
this.size = size;
this.count = count;
}
public String getGroup() {
return group;
}
public BigDecimal getSize() {
return size;
}
public BigDecimal getCount() {
return count;
}
}
package edu.umiacs.ace.monitor.access;
import com.google.common.collect.ImmutableList;
import edu.umiacs.ace.util.PersistUtil;
import org.apache.log4j.Logger;
import javax.persistence.EntityManager;
import javax.persistence.Query;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* Servlet context listener which acts as a cache for Summary information about
* groups in ACE
*
* If there are many groups, it's conceivable that this could be a strain on memory
*
* Created by shake on 4/4/17.
*/
public class GroupSummaryContext implements ServletContextListener {
private static final Logger log = Logger.getLogger(GroupSummaryContext.class);
public static Map<String, GroupSummary> summaries;
/**
* Query to get ALL group summaries
*/
private static final String SUMMARY_QUERY_ALL = "SELECT c.colgroup, sum(m.size) AS size, sum(m.count) AS count " +
"FROM collection c " +
"JOIN ( " +
"SELECT sum(size) AS size, count(id) AS count, parentcollection_id " +
"FROM monitored_item " +
"WHERE directory = 0 " +
"GROUP BY parentcollection_id " +
") AS m ON c.id = m.parentcollection_id " +
"WHERE c.colgroup IS NOT NULL " +
"GROUP BY c.colgroup";
/**
* Query to get the group summary for a single group
*/
private static final String SUMMARY_QUERY_GROUP = "select c.colgroup, sum(m.size) AS size, count(m.id) AS count " +
"FROM monitored_item m " +
"JOIN ( " +
" select colgroup, id " +
" FROM collection " +
" WHERE colgroup = ? " +
") c ON c.id = m.parentcollection_id WHERE directory = 0";
@Override
public void contextInitialized(ServletContextEvent servletContextEvent) {
summaries = new HashMap<>();
updateSummaries(ImmutableList.of(SUMMARY_QUERY_ALL),
ImmutableList.<String>of());
}
@Override
public void contextDestroyed(ServletContextEvent servletContextEvent) {
summaries.clear();
}
/**
* Public api method to update a single group
*
* @param group the group to update
*/
@SuppressWarnings("WeakerAccess")
public static void updateGroup(String group) {
if (group != null) {
log.debug("Updating group summary for " + group);
updateSummaries(
ImmutableList.of(SUMMARY_QUERY_GROUP),
ImmutableList.of(group));
}
}
/**
* Static method to allow us to update the summary of a group
*
* @param sql the sql query to build
* @param params the parameters to pass along to the query
*/
private static void updateSummaries(List<String> sql, List<String> params) {
EntityManager em = PersistUtil.getEntityManager();
StringBuilder query = new StringBuilder();
for (String s : sql) {
query.append(s);
}
Query groupSummary = em.createNativeQuery(query.toString(), "GroupSummaryMapping");
int i = 1;
for (String param : params) {
groupSummary.setParameter(i, param);
i++;
}
List<GroupSummary> results = (List<GroupSummary>)groupSummary.getResultList();
for (GroupSummary result : results) {
log.info("Result: group " + result.getGroup() + ", size: " + result.getSize() + ", count: " + result.getCount());
summaries.put(result.getGroup(), result);
}
}
}
...@@ -32,6 +32,7 @@ package edu.umiacs.ace.monitor.access; ...@@ -32,6 +32,7 @@ package edu.umiacs.ace.monitor.access;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import edu.umiacs.ace.monitor.core.Collection; import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.core.CollectionState;
import edu.umiacs.ace.monitor.support.CStateBean; import edu.umiacs.ace.monitor.support.CStateBean;
import edu.umiacs.ace.monitor.support.PageBean; import edu.umiacs.ace.monitor.support.PageBean;
import edu.umiacs.ace.util.EntityManagerServlet; import edu.umiacs.ace.util.EntityManagerServlet;
...@@ -96,8 +97,6 @@ public class StatusServlet extends EntityManagerServlet { ...@@ -96,8 +97,6 @@ public class StatusServlet extends EntityManagerServlet {
RequestDispatcher dispatcher; RequestDispatcher dispatcher;
List<CollectionSummaryBean> collections; List<CollectionSummaryBean> collections;
List<Collection> items; List<Collection> items;
LOG.info("Processing /Status request");
LOG.info("Processing /Status additional info");
long page = getParameter(request, PARAM_PAGE, DEFAULT_PAGE); long page = getParameter(request, PARAM_PAGE, DEFAULT_PAGE);
int count = (int) getParameter(request, PARAM_COUNT, DEFAULT_COUNT); int count = (int) getParameter(request, PARAM_COUNT, DEFAULT_COUNT);
...@@ -184,8 +183,8 @@ public class StatusServlet extends EntityManagerServlet { ...@@ -184,8 +183,8 @@ public class StatusServlet extends EntityManagerServlet {
} }
if (!Strings.isEmpty(state) && state.length() == 1) { if (!Strings.isEmpty(state) && state.length() == 1) {
query.setParameter(PARAM_STATE, state.charAt(0)); query.setParameter(PARAM_STATE, CollectionState.fromChar(state.charAt(0)));
countQuery.setParameter(PARAM_STATE, state.charAt(0)); countQuery.setParameter(PARAM_STATE, CollectionState.fromChar(state.charAt(0)));