Commit 526c6bc5 authored by Michael Ritter's avatar Michael Ritter

Merge branch 'docker-resolutions'

See merge request !14
parents 7d1aa430 fc251500
......@@ -69,13 +69,11 @@ import javax.mail.MessagingException;
import javax.persistence.EntityManager;
import javax.persistence.EntityTransaction;
import javax.persistence.Query;
import javax.sql.DataSource;
import java.io.InputStream;
import java.security.MessageDigest;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
......@@ -732,44 +730,45 @@ public final class AuditThread extends Thread implements CancelCallback {
em.close();
}
private void setInactiveBefore(Date d) {
private void setInactiveBefore(Date startDate) {
// ignore partial audits
if (baseItemPathList != null) {
return;
}
DataSource dataSource = PersistUtil.getDataSource();
EntityManager em = PersistUtil.getEntityManager();
EntityTransaction trans = em.getTransaction();
trans.begin();
try {
// Truncate the start date of the audit to seconds because we don't have fractional
// seconds in our database
Instant startInstant = startDate.toInstant().truncatedTo(ChronoUnit.SECONDS);
Timestamp start = Timestamp.from(startInstant);
trans.begin();
// Update the monitored item table
Query query = em.createNamedQuery("MonitoredItem.updateMissing")
.setParameter("coll", coll)
.setParameter("date", d);
.setParameter("date", start);
int i = query.executeUpdate();
if (i > 0) {
LOG.info("Set " + i + " new missing items");
// Add log entries for the new missing items
Connection connection = dataSource.getConnection();
PreparedStatement ps = connection.prepareStatement("INSERT INTO logevent(session, path, date, logtype, collection_id) " +
"SELECT ?, path, NOW(), ?, parentcollection_id FROM monitored_item m WHERE m.parentcollection_id = ? AND m.state = ? AND m.statechange = ?");
ps.setLong(1, session);
ps.setInt(2, LogEnum.FILE_MISSING.getType());
ps.setLong(3, coll.getId());
ps.setString(4, String.valueOf('M'));
ps.setTimestamp(5, new Timestamp(d.getTime()));
ps.executeUpdate();
ps.close();
connection.close();
final String sql =
"INSERT INTO logevent(session, path, date, logtype, collection_id) " +
"SELECT ?, path, NOW(), ?, parentcollection_id FROM monitored_item m " +
"WHERE m.parentcollection_id = ? AND m.state = ? AND m.statechange = ?";
Query insert = em.createNativeQuery(sql);
insert.setParameter(1, session);
insert.setParameter(2, LogEnum.FILE_MISSING.getType());
insert.setParameter(3, coll.getId());
insert.setParameter(4, String.valueOf('M'));
insert.setParameter(5, start);
insert.executeUpdate();
}
trans.commit();
} catch (SQLException e) {
trans.rollback();
LOG.error("SQL error, rolling back missing items and log entries", e);
} finally {
trans.commit();
em.close();
}
}
......
......@@ -34,11 +34,11 @@ package edu.umiacs.ace.monitor.core;
import edu.umiacs.ace.monitor.log.LogEnum;
import edu.umiacs.ace.monitor.log.LogEventManager;
import edu.umiacs.ace.util.PersistUtil;
import org.apache.log4j.Logger;
import javax.persistence.EntityManager;
import javax.persistence.EntityTransaction;
import javax.persistence.Query;
import javax.persistence.TypedQuery;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
......@@ -49,41 +49,25 @@ import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
/**
*
* @author toaster
*/
public class MonitoredItemManager {
private static Lock lock = new ReentrantLock();
private static final Logger LOG = Logger.getLogger(
MonitoredItemManager.class);
EntityManager em;
private EntityManager em;
// TODO: This may leak...
private Set<String> existingParents = new HashSet<>();
public MonitoredItemManager( EntityManager em ) {
public MonitoredItemManager(EntityManager em) {
this.em = em;
}
public Iterable<MonitoredItem> listItemsBefore( Collection c, Date d ) {
lock.lock();
try {
Query q = em.createNamedQuery("MonitoredItem.listItemsBefore");
q.setParameter("date", d);
q.setParameter("coll", c);
return q.getResultList();
} finally {
lock.unlock();
}
}
public void createDirectory( String directory, String root, Collection c ) {
public void createDirectory(String directory, String root, Collection c) {
MonitoredItem mi;
if ( existingParents.contains(directory) || directory == null ) {
if (existingParents.contains(directory) || directory == null) {
return;
}
if ( (mi = getItemByPath(directory, c)) != null ) {
if ((mi = getItemByPath(directory, c)) != null) {
EntityTransaction trans = em.getTransaction();
trans.begin();
Date d = new Date();
......@@ -100,26 +84,25 @@ public class MonitoredItemManager {
}
/**
*
*
* @param path path to look for
* @return MonitoredItem if exist, null otherwise
*/
public MonitoredItem getItemByPath( String path, Collection c ) {
public MonitoredItem getItemByPath(String path, Collection c) {
lock.lock();
try {
MonitoredItem tmp = new MonitoredItem();
tmp.setPath(path);
Query q = em.createNamedQuery("MonitoredItem.getItemByPath");
TypedQuery<MonitoredItem> q =
em.createNamedQuery("MonitoredItem.getItemByPath", MonitoredItem.class);
q.setParameter("path", path);
q.setParameter("coll", c);
// Make sure we have the correct item
List<MonitoredItem> li = q.getResultList();
Collections.sort(li, new PathComparator());
li.sort(new PathComparator());
int idx = Collections.binarySearch(li, tmp, new PathComparator());
return (idx >= 0 ? li.get(idx): null);
return (idx >= 0 ? li.get(idx) : null);
} finally {
lock.unlock();
}
......@@ -130,23 +113,23 @@ public class MonitoredItemManager {
* @param id to get
* @return the monitored item
*/
public MonitoredItem getItemById( String id, Collection c ) {
public MonitoredItem getItemById(String id, Collection c) {
EntityManager pem = PersistUtil.getEntityManager();
return pem.find(MonitoredItem.class, Long.parseLong(id));
}
/**
* Set the state on all items that contain no tokens to false
*
*
* @param c
*/
public void setMissingTokensInvalid( Collection c, long session ) {
public void setMissingTokensInvalid(Collection c, long session) {
Query q = em.createNamedQuery("MonitoredItem.listNullTokens");
q.setParameter("coll", c);
LogEventManager lem = new LogEventManager(session,c);
LogEventManager lem = new LogEventManager(session, c);
for ( Object o : q.getResultList() ) {
for (Object o : q.getResultList()) {
MonitoredItem mi = (MonitoredItem) o;
mi.setState('T');
EntityTransaction et = em.getTransaction();
......@@ -159,16 +142,21 @@ public class MonitoredItemManager {
/**
* Create new master item, master item has no replicas.
*
* @param path relative path to this item (w/o collection prefix)
* @param parentDir relative parent path to this item, null if parent is collection
* @param directory true if this is a directory
*
* @param path relative path to this item (w/o collection prefix)
* @param parentDir relative parent path to this item, null if parent is collection
* @param directory true if this is a directory
* @param parentCollection collection this item belongs to
* @param initialState the initial state to set
* @param size the size to set
* @return
*/
public MonitoredItem addItem( String path, String parentDir,
boolean directory,
Collection parentCollection, char initialState, long size ) {
public MonitoredItem addItem(String path,
String parentDir,
boolean directory,
Collection parentCollection,
char initialState,
long size) {
lock.lock();
try {
MonitoredItem mi = new MonitoredItem();
......@@ -193,11 +181,11 @@ public class MonitoredItemManager {
}
}
public List<MonitoredItem> getCollectionRoots( Collection parent ) {
public List<MonitoredItem> getCollectionRoots(Collection parent) {
lock.lock();
try {
Query q = em.createNamedQuery("MonitoredItem.listRoots");
TypedQuery<MonitoredItem> q =
em.createNamedQuery("MonitoredItem.listRoots", MonitoredItem.class);
q.setParameter("coll", parent);
return q.getResultList();
} finally {
......@@ -207,14 +195,15 @@ public class MonitoredItemManager {
/**
* List all files that are in state P or D
*
*
* @param coll
* @return
*/
public List<MonitoredItem> listRemoteErrors( Collection coll ) {
public List<MonitoredItem> listRemoteErrors(Collection coll) {
lock.lock();
try {
Query q = em.createNamedQuery("MonitoredItem.listRemoteErrors");
TypedQuery<MonitoredItem> q =
em.createNamedQuery("MonitoredItem.listRemoteErrors", MonitoredItem.class);
q.setParameter("coll", coll);
return q.getResultList();
} finally {
......@@ -222,24 +211,22 @@ public class MonitoredItemManager {
}
}
public List<MonitoredItem> listChildren( Collection coll, String parentPath ) {
public List<MonitoredItem> listChildren(Collection coll, String parentPath) {
lock.lock();
try {
Query q = em.createNamedQuery("MonitoredItem.listChildren");
TypedQuery<MonitoredItem> q =
em.createNamedQuery("MonitoredItem.listChildren", MonitoredItem.class);
q.setParameter("coll", coll);
q.setParameter("parent", parentPath);
return q.getResultList();
} finally {
lock.unlock();
}
}
public Long countErrorsInCollection( Collection c ) {
public Long countErrorsInCollection(Collection c) {
Query q = em.createNamedQuery("MonitoredItem.countErrorsInCollection");
q.setParameter("coll", c);
return (Long) q.getSingleResult();
}
......
......@@ -32,10 +32,10 @@ package edu.umiacs.ace.monitor.core;
import edu.umiacs.ace.hashtree.ProofHash;
import edu.umiacs.ace.hashtree.ProofNode;
import edu.umiacs.util.Strings;
import edu.umiacs.ace.monitor.core.Token;
import edu.umiacs.ace.util.HashValue;
import edu.umiacs.util.Check;
import edu.umiacs.util.Strings;
import java.util.Date;
/**
......@@ -81,6 +81,10 @@ public class TokenBuilder {
}
public void addHashLevel(int inheritIdx, ProofNode hashes) {
if ( hashes.getHashes() == null || hashes.getHashes().size() < 1 ) {
throw new IllegalArgumentException("Hash list must have at least 1 hash");
}
int pos = 0;
int size = hashes.getHashes().size();
......@@ -89,10 +93,6 @@ public class TokenBuilder {
+ inheritIdx + "/" + hashes.getHashes().size());
}
if ( hashes.getHashes() == null || hashes.getHashes().size() < 1 ) {
throw new IllegalArgumentException("Hash list must have at least 1 hash");
}
for ( ProofHash hash : hashes.getHashes() ) {
if ( pos == inheritIdx ) {
proof.append("X:");
......
......@@ -51,35 +51,32 @@ import java.util.GregorianCalendar;
/**
* Report Generation job for quartz
*
* @author toaster
*/
public class ReportJob implements Job {
public static final String ATTR_POLICY = "policy";
private static final Logger LOG = Logger.getLogger(ReportJob.class);
private LogEventManager logManager;
public ReportJob() {
}
@Override
public void execute( JobExecutionContext arg0 ) throws JobExecutionException {
public void execute(JobExecutionContext arg0) throws JobExecutionException {
NDC.push("[report]");
ReportPolicy reportPolicy = null;
reportPolicy = (ReportPolicy) arg0.getJobDetail().getJobDataMap().get(
ATTR_POLICY);
ReportPolicy reportPolicy;
reportPolicy = (ReportPolicy) arg0.getJobDetail().getJobDataMap().get(ATTR_POLICY);
long session = System.currentTimeMillis();
logManager = new LogEventManager(session, reportPolicy.getCollection());
LogEventManager logManager = new LogEventManager(session, reportPolicy.getCollection());
LOG.debug("Starting job " + reportPolicy.getName());
try {
CronTrigger cron = (CronTrigger) arg0.getTrigger();
CronExpression expression = new CronExpression(
cron.getCronExpression());
CronExpression expression = new CronExpression(cron.getCronExpression());
Date startDate = extractDuration(expression,
arg0.getScheduledFireTime());
Date startDate = extractDuration(expression, arg0.getScheduledFireTime());
SummaryGenerator sg = new SummaryGenerator(reportPolicy.getName(),
reportPolicy.getCollection(),
......@@ -88,17 +85,16 @@ public class ReportJob implements Job {
ReportSummary report = sg.generateReport();
SchedulerContextListener.mailReport(report, createMailList(reportPolicy));
} catch ( ParseException ex ) {
} catch (ParseException ex) {
LOG.error("Unknown parse exception", ex);
throw new JobExecutionException(ex);
} catch (MessagingException ex) {
EntityManager em = PersistUtil.getEntityManager();
logManager.persistCollectionEvent(LogEnum.SMTP_ERROR,
ex.getMessage(), em);
logManager.persistCollectionEvent(LogEnum.SMTP_ERROR, ex.getMessage(), em);
em.close();
LOG.error("Could not send report summary", ex);
throw new JobExecutionException(ex);
} catch ( Exception e ) {
} catch (Exception e) {
LOG.error("Unknown exception", e);
throw new JobExecutionException(e);
} finally {
......@@ -107,27 +103,28 @@ public class ReportJob implements Job {
}
}
private String[] createMailList( ReportPolicy rp ) {
private String[] createMailList(ReportPolicy rp) {
return rp.getEmailList().split("\\s*,\\s*");
}
private Date extractDuration( CronExpression expression, Date triggerDate )
private Date extractDuration(CronExpression expression, Date triggerDate)
throws JobExecutionException {
// todo: is it possible to replace this with java.time?
Calendar prevCal = new GregorianCalendar();
int count = 0;
prevCal.setTime(triggerDate);
prevCal.add(Calendar.MONTH, -1);
while ( !expression.isSatisfiedBy(prevCal.getTime()) && count < 12 ) {
while (!expression.isSatisfiedBy(prevCal.getTime()) && count < 12) {
count++;
prevCal.add(Calendar.MONTH, -1);
}
if ( !expression.isSatisfiedBy(prevCal.getTime()) ) {
if (!expression.isSatisfiedBy(prevCal.getTime())) {
throw new JobExecutionException(
"Count not find previous execution time, curr " + triggerDate + " expression "
+ expression.getCronExpression());
+ expression.getCronExpression());
}
return prevCal.getTime();
......
......@@ -54,9 +54,9 @@ import java.util.List;
/**
* Log summary generator, this will generate a report summarizing all log entries
* made between the specified dates. If the end date is the current time, then
* additional information will be contained showing the current state of the
* additional information will be contained showing the current state of the
* collection.
*
*
* @author toaster
*/
public class SummaryGenerator extends ReportItemTypes {
......@@ -65,29 +65,27 @@ public class SummaryGenerator extends ReportItemTypes {
private Date startDate;
private long sessionID = 0;
private String reportName;
private List<ReportItem> itemList = new ArrayList<ReportItem>();
private List<ReportItem> itemList = new ArrayList<>();
private static final Logger LOG = Logger.getLogger(SummaryGenerator.class);
public SummaryGenerator( String reportName, Collection collection,
Date startDate ) {
public SummaryGenerator(String reportName, Collection collection, Date startDate) {
this.collection = collection;
this.startDate = Argument.dateClone(startDate);
if ( Strings.isEmpty(reportName) ) {
reportName = "Report Starting " + startDate;
}
{
if (Strings.isEmpty(reportName)) {
this.reportName = "Report Starting " + startDate;
} else {
this.reportName = reportName;
}
}
public SummaryGenerator( Collection collection, long session ) {
public SummaryGenerator(Collection collection, long session) {
this.collection = collection;
this.sessionID = session;
reportName = "Session Report: " + session;
}
public SummaryGenerator( Collection collection ) {
public SummaryGenerator(Collection collection) {
this.collection = collection;
reportName = "Entire Collection Report";
}
......@@ -99,7 +97,7 @@ public class SummaryGenerator extends ReportItemTypes {
try {
queryCurrentState();
queryLogHistory();
} catch ( SQLException e ) {
} catch (SQLException e) {
LOG.error("Error creating report", e);
}
......@@ -112,7 +110,7 @@ public class SummaryGenerator extends ReportItemTypes {
summary.setEndDate(reportEndDate);
summary.setReportName(reportName);
for ( ReportItem ri : itemList ) {
for (ReportItem ri : itemList) {
ri.setReport(summary);
}
......@@ -124,12 +122,12 @@ public class SummaryGenerator extends ReportItemTypes {
try {
em.persist(summary);
et.commit();
} catch ( Exception e ) {
} catch (Exception e) {
LOG.error("Error commiting summary ", e);
et.rollback();
}
} finally {
if ( em != null ) {
if (em != null) {
em.close();
}
LOG.debug("Finished creating report for " + collection.getName());
......@@ -147,7 +145,7 @@ public class SummaryGenerator extends ReportItemTypes {
try {
// query selects lowest date, id, count for groupings of log types
connection = ds.getConnection();
if ( sessionID > 0 ) {
if (sessionID > 0) {
ps = connection.prepareStatement("SELECT MIN(logevent.DATE), "
+ "MIN(logevent.ID), logevent.LOGTYPE, count(logevent.LOGTYPE)"
+ "FROM logevent WHERE logevent.SESSION = ? AND "
......@@ -155,8 +153,8 @@ public class SummaryGenerator extends ReportItemTypes {
ps.setLong(1, sessionID);
ps.setLong(2, collection.getId());
} else if ( startDate != null ) {
} else if (startDate != null) {
// startDate is truncated from the argument.dateClone above
ps = connection.prepareStatement("SELECT MIN(logevent.DATE), "
+ "MIN(logevent.ID), logevent.LOGTYPE, count(logevent.LOGTYPE)"
+ "FROM logevent WHERE logevent.DATE >= ? AND "
......@@ -175,11 +173,11 @@ public class SummaryGenerator extends ReportItemTypes {
// create entries for each result, for start date and session id
// use lowest returned
rs = ps.executeQuery();
while ( rs.next() ) {
if ( startDate == null || startDate.after(rs.getTimestamp(1)) ) {
while (rs.next()) {
if (startDate == null || startDate.after(rs.getTimestamp(1))) {
startDate = rs.getTimestamp(1);
}
if ( sessionID == 0 || sessionID > rs.getLong(2) ) {
if (sessionID == 0 || sessionID > rs.getLong(2)) {
sessionID = rs.getLong(2);
}
String logName = LogEnum.valueOf(rs.getInt(3)).getShortName();
......@@ -204,21 +202,21 @@ public class SummaryGenerator extends ReportItemTypes {
connection = ds.getConnection();
ps = connection.prepareStatement(
"SELECT monitored_item.STATE, count(monitored_item.STATE) "
+ "FROM monitored_item " + "WHERE monitored_item.PARENTCOLLECTION_ID = ? AND "
+ "monitored_item.DIRECTORY = 0 " + "GROUP BY monitored_item.STATE");
+ "FROM monitored_item " + "WHERE monitored_item.PARENTCOLLECTION_ID = ? AND "
+ "monitored_item.DIRECTORY = 0 " + "GROUP BY monitored_item.STATE");
ps.setLong(1, collection.getId());
rs = ps.executeQuery();
long total = 0;
long totalErrors = 0;
while ( rs.next() ) {
while (rs.next()) {
char state = rs.getString(1).charAt(0);
long count = rs.getLong(2);
total += count;
switch ( state ) {
switch (state) {
case 'A':
itemList.add(new ReportItem(ACTIVE, count, false));
break;
......
package edu.umiacs.ace.rest;
import java.io.IOException;
import javax.servlet.Filter;
import javax.servlet.FilterChain;
import javax.servlet.FilterConfig;
import javax.servlet.ServletException;
import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import java.io.IOException;
/**
* Set the encoding for requests/responses to be UTF-8
* <p>
* From http://stackoverflow.com/questions/138948/how-to-get-utf-8-working-in-java-webapps
*
* @author shake
*/
public class CharsetFilter implements Filter {
private String encoding;
@Override
public void init(FilterConfig config) throws ServletException {
public void init(FilterConfig config) {
encoding = config.getInitParameter("requestEncoding");
if (encoding == null) {
......@@ -23,8 +30,8 @@ public class CharsetFilter implements Filter {
@Override
public void doFilter(ServletRequest request,
ServletResponse response,
FilterChain next)
ServletResponse response,
FilterChain next)
throws IOException, ServletException {
// Respect the client-specified character encoding
// (see HTTP specification section 3.4.1)
......@@ -32,16 +39,12 @@ public class CharsetFilter implements Filter {
request.setCharacterEncoding(encoding);
}
/**
* Set the default response content type and encoding
*/
response.setContentType("text/html; charset=UTF-8");
// Set the default response encoding
response.setCharacterEncoding("UTF-8");
next.doFilter(request, response);
}
@Override
public void destroy() {
}
}
\ No newline at end of file
}
package edu.umiacs.ace.rest;
import edu.umiacs.util.Argument;
import java.util.Date;
/**
......@@ -22,15 +24,22 @@ public class MonitoredItemBean {
public MonitoredItemBean() {
}
public MonitoredItemBean(Long id, String path, char state, String fileDigest, Long size , Date lastSeen, Date stateChange, Date lastVisited) {
public MonitoredItemBean(Long id,
String path,