diff --git a/ace-am-api/pom.xml b/ace-am-api/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..8df322f9d3a27355f8874ff750d3d07a423ffd34 --- /dev/null +++ b/ace-am-api/pom.xml @@ -0,0 +1,31 @@ + + + 4.0.0 + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + + + + + + + edu.umiacs.ace + ace + 1.11-SNAPSHOT + + + ace-am-api + ace-am-api + 1.11-SNAPSHOT + jar + + + diff --git a/ace-am-api/src/main/java/edu/umiacs/ace/am/Collection.java b/ace-am-api/src/main/java/edu/umiacs/ace/am/Collection.java new file mode 100644 index 0000000000000000000000000000000000000000..86789df0979c00d775b89bf0318c2bfe5209d9dd --- /dev/null +++ b/ace-am-api/src/main/java/edu/umiacs/ace/am/Collection.java @@ -0,0 +1,92 @@ +package edu.umiacs.ace.am; + +import java.util.Date; + +/** + * Simplified view of a collection + * + * Created by shake on 6/13/16. + */ +public class Collection { + + private static final long serialVersionUID = 1L; + + private String name; + private String group; + + private Date lastSync; + private State state; + + private String digestAlgorithm; + + public String getName() { + return name; + } + + public Collection setName(String name) { + this.name = name; + return this; + } + + public String getGroup() { + return group; + } + + public Collection setGroup(String group) { + this.group = group; + return this; + } + + public Date getLastSync() { + return lastSync; + } + + public Collection setLastSync(Date lastSync) { + this.lastSync = lastSync; + return this; + } + + public State getState() { + return state; + } + + public Collection setState(char state) { + this.state = State.fromIdentifier(state); + return this; + } + + public String getDigestAlgorithm() { + return digestAlgorithm; + } + + public Collection setDigestAlgorithm(String digestAlgorithm) { + this.digestAlgorithm = digestAlgorithm; + return this; + } + + enum State { + + ACTIVE('A', "Active"), + NEW('N', "New"), + ERROR('E', "Error"); + + final char identifier; + final String description; + + State(char identifier, String description) { + this.identifier = identifier; + this.description = description; + } + + static State fromIdentifier(char identifier) { + switch (identifier) { + case 'A': return ACTIVE; + case 'N': return NEW; + case 'E': return ERROR; + default: + throw new IllegalArgumentException("Identifier " + identifier + " does not match any known type"); + } + } + + } +} diff --git a/ace-am-api/src/main/java/edu/umiacs/ace/am/Item.java b/ace-am-api/src/main/java/edu/umiacs/ace/am/Item.java new file mode 100644 index 0000000000000000000000000000000000000000..b7c5c0d988a2ec33bc884ac1abafb74bc05200a1 --- /dev/null +++ b/ace-am-api/src/main/java/edu/umiacs/ace/am/Item.java @@ -0,0 +1,111 @@ +package edu.umiacs.ace.am; + +import java.util.Date; + +/** + * Simplified view of a MonitoredItem from aceam + * + * + * Created by shake on 6/13/16. + */ +public class Item { + + // path relative to base directory + private String path; + + private Date lastSeen; + private Date stateChange; + private Date lastVisited; + + private Collection parentCollection; + + // registered digest + private String fileDigest; + + // current digest + private String lastDigest; + + private char state; + private long size; + + public String getPath() { + return path; + } + + public Item setPath(String path) { + this.path = path; + return this; + } + + public Date getLastSeen() { + return lastSeen; + } + + public Item setLastSeen(Date lastSeen) { + this.lastSeen = lastSeen; + return this; + } + + public Date getStateChange() { + return stateChange; + } + + public Item setStateChange(Date stateChange) { + this.stateChange = stateChange; + return this; + } + + public Date getLastVisited() { + return lastVisited; + } + + public Item setLastVisited(Date lastVisited) { + this.lastVisited = lastVisited; + return this; + } + + public Collection getParentCollection() { + return parentCollection; + } + + public Item setParentCollection(Collection parentCollection) { + this.parentCollection = parentCollection; + return this; + } + + public String getFileDigest() { + return fileDigest; + } + + public Item setFileDigest(String fileDigest) { + this.fileDigest = fileDigest; + return this; + } + + public String getLastDigest() { + return lastDigest; + } + + public Item setLastDigest(String lastDigest) { + this.lastDigest = lastDigest; + return this; + } + + public char getState() { + return state; + } + + public Item setState(char state) { + this.state = state; + return this; + } + + public long getSize() { + return size; + } + + public Item setSize(long size) { + this.size = size; + return this; + } +} diff --git a/ace-am-api/src/main/java/edu/umiacs/ace/am/LogEnum.java b/ace-am-api/src/main/java/edu/umiacs/ace/am/LogEnum.java new file mode 100644 index 0000000000000000000000000000000000000000..da7c74a58f25bdf5b28322084976e0c7ffdb34cc --- /dev/null +++ b/ace-am-api/src/main/java/edu/umiacs/ace/am/LogEnum.java @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2007-2010, University of Maryland + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided + * that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions + * and the following disclaimer in the documentation and/or other materials provided with the + * distribution. + * + * Neither the name of the University of Maryland nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR + * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ACE Components were written in the ADAPT Project at the University of + * Maryland Institute for Advanced Computer Study. + */ +// $Id$ + +package edu.umiacs.ace.am; + +/** + * Enumeration of all possible log types used in LogEvent + * + * @author toaster + */ +public enum LogEnum { + + LOG_TYPE_UNKNOWN(0, "Unknown", "Unknown Log Entry Type"), + /** + * New file registered into the system + */ + FILE_NEW(1, "New File", "New file added for auditing"), + /** + * Previously registered file no longer exists + */ + FILE_MISSING(2, "File Missing", "File could not be accessed for auditing"), + /** + * file checksums do not match + */ + FILE_CORRUPT(3, "Corrupt File", "File's checksum does not match stored checksum"), + /** + * Found a previously missing file and marked it intact + */ + FILE_ONLINE(4, "File Online", "Previously offline file is available"), + ADD_TOKEN(5, "Add Token", "New token from IMS added for this file"), + CREATE_TOKEN_ERROR(6, "Create Token Error", "Error creating token for this file"), + MISSING_TOKEN(7, "Missing Token", "File is registered, but has no token"), + REMOVE_ITEM(8, "Item Removed", "File or directory and tokens removed from monitoring"), + TOKEN_INVALID(9, "Digest Token Mismatch", "Possibly corrupt checksum or proof, local hash with proof does not match IMS CSI"), + SITE_UNACCESSIBLE(10, "Collection Unaccessible", "Unable to connect to collection for auditing"), //TODO + /** + * Error reading file from local storage + */ + ERROR_READING(11, "Read Error", "Error reading the file locally, check local file"), + /** + * unknown error returned from ims + */ + UNKNOWN_IMS_COMMUNICATION_ERROR(12, "IMS Error", "Unkown error trying to communicate with the IMS, check IMS logs"), + /** + * token state changed from invalid token to valid, this should be VERY VERY rare + */ + TOKEN_VALID(13, "Token Digest Revalidated", "hash and proof match CSI"), + REMOTE_FILE_CORRUPT(14, "Remote File Corrupt", "remote file digest differs from local file"), + REMOTE_FILE_MISSING(15, "Remote File Missing", "not corresponding file on remote site"), + REMOTE_FILE_ONLINE(16, "Remote File Online", "Remote file marked as active"), + REMOVE_STORAGE_DRIVER(17, "Remove Storage Driver", "Configured storage driver was removed"), + //STORAGE_DRIVER_ADDED(18, "Add Storage Driver", "New Storage Driver configured"), + COLLECTION_REGISTERED(19,"Collection Registered", "Collection registered for monitoring"), + /////////// audit start/stops + /** + * Start a synchronization run on a master site + */ + FILE_AUDIT_START(20, "File Audit Start", "Auditing of this collection's files started"), + /** + * Finish a sync run on a master site + */ + FILE_AUDIT_FINISH(21, "File Audit Finish", "Auditing of this collection's files finished"), + /** + * Start a synchronization run on a master site + */ + TOKEN_AUDIT_START(22, "Token Audit Start", "Auditing of this collection's tokens started"), + /** + * Finish a sync run on a master site + */ + TOKEN_AUDIT_FINISH(23, "Token Audit Finish", "Auditing of this collection's tokens finished"), + TOKEN_INGEST_UPDATE(24, "Token Ingest Update", "Token was out of date and has been updated"), + FILE_REGISTER(25, "File Registered", "New file registered but is not ready for auditing"), + FILE_AUDIT_FALLBACK(26, "File Audit Fallback", "File Audit could not connect to the IMS, falling back to audit-only mode"), + SMTP_ERROR(27, "SMTP Communication Error", "Could not connect to designated SMTP host"), + SYSTEM_ERROR(99, "System Error", "Unknown system error occurred, check server logs"); + private int type; + private String shortName; + private String details; + + LogEnum(int i, String shortName, String details ) { + this.type = i; + this.shortName = shortName; + this.details = details; + } + + public int getType() { + return type; + } + + public String getDetails() { + return details; + } + + public String getShortName() { + return shortName; + } + + public static LogEnum valueOf( int i ) { + switch ( i ) { + case 0: + return LOG_TYPE_UNKNOWN; + case 1: + return FILE_NEW; + case 2: + return FILE_MISSING; + case 3: + return FILE_CORRUPT; + case 4: + return FILE_ONLINE; + case 5: + return ADD_TOKEN; + case 6: + return CREATE_TOKEN_ERROR; + case 7: + return MISSING_TOKEN; + case 8: + return REMOVE_ITEM; + case 9: + return TOKEN_INVALID; + case 10: + return SITE_UNACCESSIBLE; + case 11: + return ERROR_READING; + case 12: + return UNKNOWN_IMS_COMMUNICATION_ERROR; + case 13: + return TOKEN_VALID; + case 14: + return REMOTE_FILE_CORRUPT; + case 15: + return REMOTE_FILE_MISSING; + case 16: + return REMOTE_FILE_ONLINE; + case 17: + return REMOVE_STORAGE_DRIVER; +// case 18: +// return STORAGE_DRIVER_ADDED; + case 19: + return COLLECTION_REGISTERED; + case 20: + return FILE_AUDIT_START; + case 21: + return FILE_AUDIT_FINISH; + case 22: + return TOKEN_AUDIT_START; + case 23: + return TOKEN_AUDIT_FINISH; + case 24: + return TOKEN_INGEST_UPDATE; + case 25: + return FILE_REGISTER; + case 26: + return FILE_AUDIT_FALLBACK; + + case 99: + return SYSTEM_ERROR; + } + + throw new IllegalArgumentException("Int " + i + " does not match any known type"); + } +} diff --git a/ace-am-api/src/main/java/edu/umiacs/ace/am/LogItem.java b/ace-am-api/src/main/java/edu/umiacs/ace/am/LogItem.java new file mode 100644 index 0000000000000000000000000000000000000000..68cd55dd2be7a19dd5c5cd4e17b1d2cf155090a3 --- /dev/null +++ b/ace-am-api/src/main/java/edu/umiacs/ace/am/LogItem.java @@ -0,0 +1,81 @@ +package edu.umiacs.ace.am; + +import java.util.Date; + +/** + * + * Created by shake on 6/14/16. + */ +public class LogItem { + + private LogEnum type; + private String description; + private long session; + private String path; + private Collection collection; + private Date date; + + public LogEnum getType() { + return type; + } + + public LogItem setType(LogEnum type) { + this.type = type; + return this; + } + + public String getDescription() { + return description; + } + + public LogItem setDescription(String description) { + this.description = description; + return this; + } + + public long getSession() { + return session; + } + + public LogItem setSession(long session) { + this.session = session; + return this; + } + + public String getPath() { + return path; + } + + public LogItem setPath(String path) { + this.path = path; + return this; + } + + public Collection getCollection() { + return collection; + } + + public LogItem setCollection(Collection collection) { + this.collection = collection; + return this; + } + + public Date getDate() { + return date; + } + + public LogItem setDate(Date date) { + this.date = date; + return this; + } + + /** + * What we want: + * Map ??? + * -> Want to map from a log type to the list of log entries for that type + * -> can keep the LogItem (hold all the columns) in its own class + * -> Have a LogGroup class which holds the grouping of log items + * -> LogGroup should keep a Cursor for iterating over the entries + */ + +} diff --git a/ace-am-api/src/main/java/edu/umiacs/ace/am/Report.java b/ace-am-api/src/main/java/edu/umiacs/ace/am/Report.java new file mode 100644 index 0000000000000000000000000000000000000000..d3eeba336a3f2b2158bc35715fffe1ece24af715 --- /dev/null +++ b/ace-am-api/src/main/java/edu/umiacs/ace/am/Report.java @@ -0,0 +1,75 @@ +package edu.umiacs.ace.am; + +import java.util.Date; + +/** + * Class which makes up a Report to send out + * Should contain all the necessary information from an audit + * + * Created by shake on 6/13/16. + */ +public class Report { + + private Date generated; + private Date start; + private Date end; + + private long session; + + private Collection collection; + private ReportInfo info; + + public Collection getCollection() { + return collection; + } + + public Report setCollection(Collection collection) { + this.collection = collection; + return this; + } + + public long getSession() { + return session; + } + + public Report setSession(long session) { + this.session = session; + return this; + } + + public Date getEnd() { + return end; + } + + public Report setEnd(Date end) { + this.end = end; + return this; + } + + public Date getStart() { + return start; + } + + public Report setStart(Date start) { + this.start = start; + return this; + } + + public Date getGenerated() { + return generated; + } + + public Report setGenerated(Date generated) { + this.generated = generated; + return this; + } + + public ReportInfo getInfo() { + return info; + } + + public Report setInfo(ReportInfo info) { + this.info = info; + return this; + } +} diff --git a/ace-am-api/src/main/java/edu/umiacs/ace/am/ReportInfo.java b/ace-am-api/src/main/java/edu/umiacs/ace/am/ReportInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..4db21d209350054d1ef451a4f2fcb983a145cd55 --- /dev/null +++ b/ace-am-api/src/main/java/edu/umiacs/ace/am/ReportInfo.java @@ -0,0 +1,16 @@ +package edu.umiacs.ace.am; + +import java.util.List; + +/** + * Interface from which we get information about our audit; implemented in am to allow db access + * + * + * Created by shake on 6/15/16. + */ +public interface ReportInfo { + + List getErrors(); + List getLogEntries(); + +} diff --git a/ace-am-api/src/main/java/edu/umiacs/ace/am/Reporter.java b/ace-am-api/src/main/java/edu/umiacs/ace/am/Reporter.java new file mode 100644 index 0000000000000000000000000000000000000000..d3dcf6cbdab146c5d1f01a9fd57420001194da8f --- /dev/null +++ b/ace-am-api/src/main/java/edu/umiacs/ace/am/Reporter.java @@ -0,0 +1,34 @@ +package edu.umiacs.ace.am; + +import java.util.List; +import java.util.Map; + +/** + * Interface from which we send Reports out + * + * Created by shake on 6/13/16. + */ +public interface Reporter { + + /** + * Get the settings for the reporter + * + * @return a list of settings the reporter uses for configuration + */ + List settings(); + + /** + * Process a report + * + * @param report the report to use + */ + void report(Report report); + + /** + * Initialize a reporter + * + * @param settings the settings for the reporter + */ + void init(Map settings); + +} diff --git a/ace-am/pom.xml b/ace-am/pom.xml index 869422c64dee7260f888445ecc925110d36072b3..e8dd6d0fe58ac617daa5145766f046793d23cf5f 100644 --- a/ace-am/pom.xml +++ b/ace-am/pom.xml @@ -7,7 +7,6 @@ 1.11-SNAPSHOT ../pom.xml - edu.umiacs.ace ace-am ace-am war @@ -42,6 +41,14 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + + + + edu.umiacs.ace + ace-am-api + ${project.version} + + edu.umiacs.ace ace-ims-api @@ -94,7 +107,7 @@ org.apache.tomcat catalina - 6.0.32 + 6.0.41 provided @@ -104,6 +117,7 @@ eclipselink 2.2.0 + org.eclipse.persistence javax.persistence @@ -138,7 +152,7 @@ javax.servlet.jsp jsp-api - 2.1 + 2.2 provided @@ -148,8 +162,8 @@ javax.servlet - servlet-api - 2.5 + javax.servlet-api + 3.0.1 provided @@ -207,6 +221,7 @@ swap-lib 1.0 + org.apache.mina mina-core @@ -244,22 +259,31 @@ jersey-server 1.6 + com.sun.jersey jersey-json 1.6 + edu.umiacs.ace ace-ims-ws 1.11-SNAPSHOT jar + org.apache.httpcomponents httpclient 4.3.6 + + + io.github.lukehutch + fast-classpath-scanner + LATEST + diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/access/StatusServlet.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/access/StatusServlet.java index ddd340ab94f99cd9eb2a76d35308885cf9d0740c..260bbadbbe01bc2a307c88bdb7133f3d023c1283 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/access/StatusServlet.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/access/StatusServlet.java @@ -108,7 +108,7 @@ public class StatusServlet extends EntityManagerServlet { String collection = getParameter(request, PARAM_COLLECTION_LIKE, null); String state = getParameter(request, PARAM_STATE, null); // String date = getParameter(request, PARAM_GROUP, null); - PageBean pb = new PageBean((int) page, count, "Status"); + PageBean pb = new PageBean((int) page, count, "/"); long offset = page * count; diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java index 904f4f8dfbb3f073fe16c9d5db48d2e12a10f512..b1aa4654a83afb37828a4424f3f8cf80fb3acb67 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java @@ -30,50 +30,57 @@ // $Id$ package edu.umiacs.ace.monitor.audit; +import edu.umiacs.ace.am.Report; +import edu.umiacs.ace.am.ReportInfo; +import edu.umiacs.ace.am.Reporter; import edu.umiacs.ace.driver.AuditIterable; import edu.umiacs.ace.driver.DriverStateBean; -import edu.umiacs.ace.driver.filter.PathFilter; import edu.umiacs.ace.driver.FileBean; import edu.umiacs.ace.driver.StorageDriver; +import edu.umiacs.ace.driver.filter.PathFilter; +import edu.umiacs.ace.driver.filter.SimpleFilter; import edu.umiacs.ace.ims.api.IMSException; import edu.umiacs.ace.ims.api.IMSService; import edu.umiacs.ace.ims.api.TokenRequestBatch; import edu.umiacs.ace.ims.api.TokenValidator; import edu.umiacs.ace.ims.ws.TokenRequest; import edu.umiacs.ace.monitor.access.CollectionCountContext; -import edu.umiacs.ace.monitor.register.IngestThreadPool; -import edu.umiacs.ace.util.PersistUtil; -import edu.umiacs.ace.driver.filter.SimpleFilter; import edu.umiacs.ace.monitor.compare.CollectionCompare2; import edu.umiacs.ace.monitor.compare.CompareResults; -import edu.umiacs.ace.remote.JsonGateway; +import edu.umiacs.ace.monitor.core.Collection; +import edu.umiacs.ace.monitor.core.ConfigConstants; import edu.umiacs.ace.monitor.core.MonitoredItem; import edu.umiacs.ace.monitor.core.MonitoredItemManager; +import edu.umiacs.ace.monitor.log.LogEnum; +import edu.umiacs.ace.monitor.log.LogEvent; import edu.umiacs.ace.monitor.log.LogEventManager; +import edu.umiacs.ace.monitor.peers.PeerCollection; +import edu.umiacs.ace.monitor.reporting.DbReportInfo; +import edu.umiacs.ace.monitor.reporting.ReportPlugin; import edu.umiacs.ace.monitor.reporting.ReportSummary; +import edu.umiacs.ace.monitor.reporting.ReporterContextListener; import edu.umiacs.ace.monitor.reporting.SchedulerContextListener; import edu.umiacs.ace.monitor.reporting.SummaryGenerator; -import edu.umiacs.ace.monitor.core.Collection; -import edu.umiacs.ace.monitor.core.ConfigConstants; import edu.umiacs.ace.monitor.settings.SettingsUtil; -import edu.umiacs.ace.monitor.log.LogEnum; -import edu.umiacs.ace.monitor.log.LogEvent; -import edu.umiacs.ace.monitor.peers.PeerCollection; +import edu.umiacs.ace.remote.JsonGateway; import edu.umiacs.ace.token.AceToken; +import edu.umiacs.ace.util.PersistUtil; import edu.umiacs.ace.util.TokenUtil; import edu.umiacs.util.Strings; +import org.apache.log4j.Logger; +import org.apache.log4j.NDC; + +import javax.mail.MessagingException; +import javax.persistence.EntityManager; +import javax.persistence.EntityTransaction; import java.io.InputStream; import java.security.MessageDigest; +import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import javax.mail.MessagingException; -import javax.persistence.EntityManager; -import javax.persistence.EntityTransaction; -import org.apache.log4j.Logger; -import org.apache.log4j.NDC; /** * @@ -280,6 +287,11 @@ public final class AuditThread extends Thread implements CancelCallback { if ( verbose ) { logAuditFinish(); generateAuditReport(); + try { + handleReporters(); + } catch (Exception e) { + LOG.error("Caught exception during extra reporting", e); + } } AuditThreadFactory.finished(coll); @@ -289,6 +301,49 @@ public final class AuditThread extends Thread implements CancelCallback { LOG.info("Exiting audit thread"); } + private void handleReporters() { + LOG.trace("Generating extra reports on " + session + " coll " + + coll.getName()); + Report report = new Report(); + ReportInfo info = new DbReportInfo(coll, session); + + Map settings = coll.getSettings(); + edu.umiacs.ace.am.Collection c = new edu.umiacs.ace.am.Collection(); + + c.setDigestAlgorithm(coll.getDigestAlgorithm()); + c.setGroup(coll.getGroup()); + c.setLastSync(coll.getLastSync()); + c.setName(coll.getName()); + c.setState(coll.getState()); + + report.setEnd(new Date()); + report.setStart(new Date(session)); + report.setGenerated(new Date()); + + report.setInfo(info); + report.setCollection(c); + report.setSession(session); + + // Get the plugins for our collection + // Then attempt to instantiate them based on the class we captured at startup + // Finally, init and run our reporter + List plugins = coll.getReportPlugins(); + Map map = ReporterContextListener.map; + for (ReportPlugin plugin : plugins) { + ReporterContextListener.Plugin p = map.get(plugin.getNamedClass()); + try { + Reporter reporter = Reporter.class.cast(p.heldClass().newInstance()); + reporter.init(settings); + reporter.report(report); + } catch (InstantiationException | IllegalAccessException e) { + LOG.error("Unable to create reporter for " + plugin, e); + } + } + + LOG.trace("Finished extra reports on " + session + " coll " + + coll.getName()); + } + private boolean openIms() { try { IMSService ims; @@ -723,10 +778,10 @@ public final class AuditThread extends Thread implements CancelCallback { return parentName; } - private String[] createMailList() { + // package local so we can test it + String[] createMailList() { String addrs = SettingsUtil.getString(coll, ConfigConstants.ATTR_EMAIL_RECIPIENTS); - String[] maillist = (addrs == null ? null : addrs.split("\\s*,\\s*")); - return maillist; + return (addrs == null ? null : addrs.split("\\s*,\\s*")); } private void setCollectionState() { diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThreadFactory.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThreadFactory.java index bea5a44d480ab09fcc067e007b457c8957e562f2..89d0519f0de76697229874640fe7e81b08188782 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThreadFactory.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThreadFactory.java @@ -30,6 +30,8 @@ // $Id$ package edu.umiacs.ace.monitor.audit; +import com.google.common.collect.ImmutableList; +import edu.umiacs.ace.am.Reporter; import edu.umiacs.ace.driver.StorageDriver; import edu.umiacs.ace.monitor.core.Collection; import edu.umiacs.ace.monitor.core.MonitoredItem; @@ -50,7 +52,7 @@ import java.util.concurrent.ConcurrentHashMap; import static edu.umiacs.ace.util.Submittable.RunState.QUEUED; import static edu.umiacs.ace.util.Submittable.RunState.RUNNING; -/**w +/** * * @author toaster */ @@ -69,7 +71,9 @@ public class AuditThreadFactory { public static boolean blocking = false; public static int maxBlockTime = 0; - public static void setIMS( String ims ) { + private static List reporters; + + public static void setIMS(String ims) { if (Strings.isEmpty(ims)) { LOG.error("Empty ims string, ignoring"); return; @@ -93,8 +97,8 @@ public class AuditThreadFactory { return imsPort; } - public static void setImsPort( int imsPort ) { - if ( imsPort < 1 && imsPort > 32768 ) { + public static void setImsPort(int imsPort) { + if (imsPort < 1 && imsPort > 32768) { LOG.error("ims port must be between 1 and 32768"); return; } @@ -105,7 +109,7 @@ public class AuditThreadFactory { AuditThreadFactory.auditOnly = auditOnlyMode; } - public static void setAuditSampling(boolean auditSampling ) { + public static void setAuditSampling(boolean auditSampling) { AuditThreadFactory.auditSample = auditSampling; } @@ -118,14 +122,14 @@ public class AuditThreadFactory { } public static void setMaxBlockTime(int maxBlockTime) { - if ( maxBlockTime < 0 ) { + if (maxBlockTime < 0) { maxBlockTime = 0; } AuditThreadFactory.maxBlockTime = maxBlockTime; } public static int getMaxBlockTime() { - return maxBlockTime; + return maxBlockTime; } public static boolean isAuditing() { @@ -134,7 +138,7 @@ public class AuditThreadFactory { /** * Return a new or existing thread if any room is available New threads will start replication - * + * * @param c * @param tri * @return @@ -206,7 +210,7 @@ public class AuditThreadFactory { // Why does max_audits have an underscore? Oh well... public static void setMaxAudits(int max_audits) { - if ( max_audits <= 0 ) { + if (max_audits <= 0) { return; } AuditThreadFactory.max_audits = max_audits; @@ -219,9 +223,9 @@ public class AuditThreadFactory { /** * Return the current thread for a collection. * @param c collection to fetch - * + * * @return current running thread or null if nothing is running - * + * public static AuditThread getThread( Collection c ) { synchronized ( runningThreads ) { if ( isRunning(c) ) { @@ -270,7 +274,7 @@ public class AuditThreadFactory { SecureRandom rand = new SecureRandom(); List items = new LinkedList<>(); - for ( int i=0;i < size; i++) { + for (int i = 0; i < size; i++) { int idxToGet = rand.nextInt(itemIds.size()); MonitoredItem item = em.find(MonitoredItem.class, itemIds.remove(idxToGet)); items.add(item); @@ -291,4 +295,17 @@ public class AuditThreadFactory { future.cancel(true); } } + + public static List getReporters() { + try { + Class c = Class.forName("edu.umiacs.ace.monitor.reporting.SMTPReporter"); + Object instance = c.newInstance(); + return ImmutableList.of(Reporter.class.cast(instance)); + } catch (ClassNotFoundException | IllegalAccessException | InstantiationException e) { + e.printStackTrace(); + } + + return ImmutableList.of(); + } + } diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/core/Collection.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/core/Collection.java index 4951af1429abb3074ffc4576bdaabeb567fb4d0f..02da406097d8ba8cc7ff27a30ed32560309a20dc 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/core/Collection.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/core/Collection.java @@ -32,6 +32,7 @@ package edu.umiacs.ace.monitor.core; import edu.umiacs.ace.monitor.peers.PeerCollection; +import edu.umiacs.ace.monitor.reporting.ReportPlugin; import edu.umiacs.util.Argument; import javax.persistence.CascadeType; @@ -39,6 +40,7 @@ import javax.persistence.CollectionTable; import javax.persistence.Column; import javax.persistence.ElementCollection; import javax.persistence.Entity; +import javax.persistence.FetchType; import javax.persistence.GeneratedValue; import javax.persistence.GenerationType; import javax.persistence.Id; @@ -115,6 +117,9 @@ public class Collection implements Serializable { @Column(name="VALUE") private Map settings; + @OneToMany(mappedBy = "parent", cascade = CascadeType.ALL, fetch = FetchType.LAZY) + private List reportPlugins; + public void setId( Long id ) { this.id = id; } @@ -130,10 +135,19 @@ public class Collection implements Serializable { public void setSettings(Map settings) { this.settings = settings; } + + public void setReportPlugins(List plugins) { + this.reportPlugins = plugins; + } + public void setPeerCollections( List peerCollections ) { this.peerCollections = peerCollections; } + public List getReportPlugins() { + return reportPlugins; + } + public List getPeerCollections() { return peerCollections; } diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/log/LogEvent.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/log/LogEvent.java index 5f5b1863e4a47c5618bedbd0bb0c93c429870942..108a315085372cfbadf6618fbdc2a55fda460142 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/log/LogEvent.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/log/LogEvent.java @@ -54,7 +54,9 @@ import javax.persistence.Temporal; @Table(name = "logevent") @NamedQueries({ @NamedQuery(name = "LogEvent.deleteByCollection", query = - "DELETE FROM LogEvent e WHERE e.collection = :coll") + "DELETE FROM LogEvent e WHERE e.collection = :coll"), + @NamedQuery(name= "LogEvent.selectByCollectionAndSession", query= + "SELECT e FROM LogEvent e WHERE e.collection = :coll AND e.session = :session") }) public class LogEvent implements Serializable { diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/DbReportInfo.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/DbReportInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..fb14d609410f3c4fc32c2612762b8d0b202390ed --- /dev/null +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/DbReportInfo.java @@ -0,0 +1,81 @@ +package edu.umiacs.ace.monitor.reporting; + +import edu.umiacs.ace.am.Item; +import edu.umiacs.ace.am.LogEnum; +import edu.umiacs.ace.am.LogItem; +import edu.umiacs.ace.am.ReportInfo; +import edu.umiacs.ace.monitor.core.Collection; +import edu.umiacs.ace.monitor.core.MonitoredItem; +import edu.umiacs.ace.monitor.log.LogEvent; +import edu.umiacs.ace.util.PersistUtil; + +import javax.persistence.EntityManager; +import javax.persistence.Query; +import java.util.ArrayList; +import java.util.List; + +/** + * Class which we supply to a report to get back information from the database + * + * Created by shake on 6/15/16. + */ +public class DbReportInfo implements ReportInfo { + + private long session; + private Collection collection; + + public DbReportInfo(Collection collection, long session) { + this.collection = collection; + this.session = session; + } + + @Override + public List getErrors() { + List errorItems = new ArrayList<>(); + + EntityManager em = PersistUtil.getEntityManager(); + Query query = em.createNamedQuery("MonitoredItem.listLocalErrors"); + query.setParameter("coll", collection); + List errors = query.getResultList(); + em.close(); + + for (MonitoredItem error : errors) { + Item errorItem = new Item(); + errorItem.setState(error.getState()); + errorItem.setFileDigest(error.getFileDigest()); + // errorItem.setLastDigest() + errorItem.setLastSeen(error.getLastSeen()); + errorItem.setPath(error.getPath()); + errorItem.setSize(error.getSize()); + errorItem.setStateChange(error.getStateChange()); + errorItem.setLastVisited(error.getLastVisited()); + errorItems.add(errorItem); + } + + return errorItems; + } + + @Override + public List getLogEntries() { + List logItems = new ArrayList<>(); + + EntityManager em = PersistUtil.getEntityManager(); + Query query = em.createNamedQuery("LogEvent.selectByCollectionAndSession"); + query.setParameter("coll", collection); + query.setParameter("session", session); + List events = query.getResultList(); + em.close(); + + for (LogEvent event : events) { + LogItem item = new LogItem(); + item.setPath(event.getPath()); + item.setDate(event.getDate()); + item.setDescription(event.getDescription()); + item.setType(LogEnum.valueOf(event.getLogType())); + item.setSession(session); + logItems.add(item); + } + + return logItems; + } +} diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/ManageReporterServlet.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/ManageReporterServlet.java new file mode 100644 index 0000000000000000000000000000000000000000..0797b79685ad1ff620c8fec4ce1810da196d7649 --- /dev/null +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/ManageReporterServlet.java @@ -0,0 +1,114 @@ +package edu.umiacs.ace.monitor.reporting; + +import edu.umiacs.ace.monitor.core.Collection; +import edu.umiacs.ace.util.EntityManagerServlet; +import edu.umiacs.util.Strings; +import org.apache.log4j.Logger; + +import javax.persistence.EntityManager; +import javax.persistence.EntityTransaction; +import javax.persistence.TypedQuery; +import javax.servlet.RequestDispatcher; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Servlet to manage Reporters attached to a collection + *

+ * Used for adding and remove plugins I guess + *

+ * Created by shake on 6/17/16. + */ +public class ManageReporterServlet extends EntityManagerServlet { + + private static final Logger LOG = Logger.getLogger(ManageReporterServlet.class); + + private static final String PARAM_CLASS = "class"; // class name of our plugin + private static final String PARAM_REMOVE = "remove"; // long internal id of relation + + + @Override + protected void processRequest(HttpServletRequest request, HttpServletResponse response, EntityManager em) throws ServletException, IOException { + Collection coll = getCollection(request, em); + String namedClass = request.getParameter(PARAM_CLASS); + String remove = request.getParameter(PARAM_REMOVE); + + RequestDispatcher dispatcher; + if (coll == null) { + LOG.trace("No collection to process for managing reporter plugins"); + dispatcher = request.getRequestDispatcher("Status"); + } else { + // Begin our transaction + dispatcher = request.getRequestDispatcher("ManageCollection?collectionid=" + coll.getId()); + EntityTransaction transaction = em.getTransaction(); + transaction.begin(); + + if (!Strings.isEmpty(remove)) { // request to remove a plugin + remove(coll, remove, em); + } else if (!Strings.isEmpty(namedClass)) { // request to add a plugin + save(coll, namedClass, request, em); + } else { // GET on /ManageReporter + dispatcher = request.getRequestDispatcher("configure_reporting.jsp?collectionid=" + coll.getId()); + // Don't really like this.. but... it will do for now I suppose + List reporters = ReporterContextListener.reporters.stream() + .filter(r -> !coll.getReportPlugins().contains( + // create a report plugin to check against + new ReportPlugin() + .setNamedClass(r.getName()) + .setParent(coll))) + .collect(Collectors.toList()); + request.setAttribute("reporters", reporters); + } + + transaction.commit(); + } + + dispatcher.forward(request, response); + } + + private void remove(Collection coll, String remove, EntityManager em) { + Map settings = coll.getSettings(); + List plugins = coll.getReportPlugins(); + + TypedQuery query = em.createNamedQuery("ReportPlugin.selectByCollectionAndName", ReportPlugin.class); + query.setParameter("coll", coll); + query.setParameter("named", remove); + + // Find our plugin and remove it + ReportPlugin plugin = query.getSingleResult(); + em.remove(plugin); + + // Remove the plugin settings from the collection + ReporterContextListener.Plugin rcl = ReporterContextListener.map.get(remove); + rcl.getSettings().forEach(settings::remove); + plugins.remove(plugin); + em.merge(coll); + } + + private void save(Collection coll, String namedClass, HttpServletRequest request, EntityManager em) { + Map settings = coll.getSettings(); + List plugins = coll.getReportPlugins(); + + ReportPlugin savedPlugin = new ReportPlugin(); + savedPlugin.setNamedClass(namedClass); + savedPlugin.setParent(coll); + plugins.add(savedPlugin); + // Create an instance of our class + ReporterContextListener.Plugin plugin = ReporterContextListener.map.get(namedClass); + for (String param : plugin.getSettings()) { + String pluginSetting = request.getParameter(param); + LOG.trace("Adding " + param + " with val " + pluginSetting); + settings.put(param, pluginSetting); + } + + em.persist(savedPlugin); + em.merge(coll); + } + + +} diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/ReportPlugin.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/ReportPlugin.java new file mode 100644 index 0000000000000000000000000000000000000000..97ab2ed4f0ca41eef22b6557e67c95a4b27e4876 --- /dev/null +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/ReportPlugin.java @@ -0,0 +1,83 @@ +package edu.umiacs.ace.monitor.reporting; + +import edu.umiacs.ace.monitor.core.Collection; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.ManyToOne; +import javax.persistence.NamedQueries; +import javax.persistence.NamedQuery; +import javax.persistence.Table; +import java.io.Serializable; + +/** + * + * Created by shake on 6/20/16. + */ +@Entity +@Table(name = "report_plugin") +@NamedQueries({ + @NamedQuery(name = "ReportPlugin.deleteByCollection", query = + "DELETE FROM ReportPlugin WHERE parent = :coll"), + @NamedQuery(name = "ReportPlugin.selectByCollectionAndName", query = + "SELECT p FROM ReportPlugin p WHERE p.parent = :coll AND p.namedClass = :named") +}) +public class ReportPlugin implements Serializable { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @ManyToOne + private Collection parent; + private String namedClass; + + public Long getId() { + return id; + } + + public ReportPlugin setId(Long id) { + this.id = id; + return this; + } + + public Collection getParent() { + return parent; + } + + public ReportPlugin setParent(Collection parent) { + this.parent = parent; + return this; + } + + public String getNamedClass() { + return namedClass; + } + + public ReportPlugin setNamedClass(String namedClass) { + this.namedClass = namedClass; + return this; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ReportPlugin that = (ReportPlugin) o; + + if (!parent.equals(that.parent)) return false; + return namedClass.equals(that.namedClass); + + } + + @Override + public int hashCode() { + int result = parent.hashCode(); + result = 31 * result + namedClass.hashCode(); + return result; + } +} diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/ReporterContextListener.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/ReporterContextListener.java new file mode 100644 index 0000000000000000000000000000000000000000..fac5b69ea2c446d179711068a3e20df91e254574 --- /dev/null +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/ReporterContextListener.java @@ -0,0 +1,86 @@ +package edu.umiacs.ace.monitor.reporting; + +import com.google.common.collect.ImmutableMap; +import edu.umiacs.ace.am.Reporter; +import io.github.lukehutch.fastclasspathscanner.FastClasspathScanner; +import org.apache.log4j.Logger; + +import javax.servlet.ServletContextEvent; +import javax.servlet.ServletContextListener; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * + * Created by shake on 6/16/16. + */ +public class ReporterContextListener implements ServletContextListener { + private static final Logger log = Logger.getLogger(ReporterContextListener.class); + + public static Map map; + static List reporters; + private static final String PAGE_REPORTERS = "reporters"; + + @Override + public void contextInitialized(ServletContextEvent servletContextEvent) { + reporters = new ArrayList<>(); + Map tempMap = new HashMap<>(); + + new FastClasspathScanner() + .matchClassesImplementing(Reporter.class, c -> { + Plugin plugin = new Plugin(c); + tempMap.put(c.getSimpleName(), plugin); + reporters.add(plugin); + }) + .scan(); + + log.info("Found " + reporters.size() + " reporting classes: " + reporters.toString()); + + map = ImmutableMap.copyOf(tempMap); + } + + @Override + public void contextDestroyed(ServletContextEvent servletContextEvent) { + reporters.clear(); + } + + public static class Plugin { + private Class clazz; + + Plugin(Class clazz) { + this.clazz = clazz; + } + + public Class heldClass() { + return clazz; + } + + public String getLongName() { + return this.clazz.getName(); + } + + public String getName() { + return clazz.getSimpleName(); + } + + public List getSettings() { + log.trace("Retrieving settings for " + getName()); + List settings = new ArrayList<>(); + + try { + // log.trace("clazz classloader: " + clazz.getClassLoader()); + // log.trace("Reporter classloader: " + Reporter.class.getClassLoader()); + Reporter reporter = Reporter.class.cast(clazz.newInstance()); + settings = reporter.settings(); + log.trace("Found " + settings.size() + " settings"); + } catch (InstantiationException | IllegalAccessException e) { + log.error("could not instantiate class " + clazz, e); + } + + return settings; + } + } + +} diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/SMTPReporter.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/SMTPReporter.java new file mode 100644 index 0000000000000000000000000000000000000000..204f869a8cb8e18f0137bf461b8c83ce00638422 --- /dev/null +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/SMTPReporter.java @@ -0,0 +1,162 @@ +package edu.umiacs.ace.monitor.reporting; + +import com.google.common.collect.ImmutableList; +import edu.umiacs.ace.am.LogEnum; +import edu.umiacs.ace.am.LogItem; +import edu.umiacs.ace.am.Report; +import edu.umiacs.ace.am.Reporter; +import org.apache.log4j.Logger; + +import javax.mail.Message; +import javax.mail.MessagingException; +import javax.mail.Session; +import javax.mail.Transport; +import javax.mail.internet.InternetAddress; +import javax.mail.internet.MimeMessage; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +/** + * Class which makes smtp class after an audit has finished + * + * Created by shake on 6/13/16. + */ +public class SMTPReporter implements Reporter { + + private static final Logger LOG = Logger.getLogger(SMTPReporter.class); + + private final String RECIPIENTS_KEY = "smtp.recipients"; + + private String recipients; + private Map> grouped = new HashMap<>(); + + @Override + public void init(Map settings) { + recipients = settings.get(RECIPIENTS_KEY); + } + + @Override + public List settings() { + return ImmutableList.of(RECIPIENTS_KEY); + } + + @Override + public void report(Report report) { + groupLogEntries(report); + String body = createBody(report); + try { + send(body); + } catch (MessagingException e) { + LOG.error("Unable to send report", e); + } + } + + private void groupLogEntries(Report report) { + for (LogItem logItem : report.getInfo().getLogEntries()) { + LogEnum type = logItem.getType(); + List items = grouped.get(type); + if (items == null) { + items = new ArrayList<>(); + } + + items.add(logItem); + grouped.put(type, items); + } + } + + private void send(String body) throws MessagingException { + //Set the host smtp address + Properties props = new Properties(); + props.put("mail.smtp.host", "127.0.0.1"); + + // create some properties and get the default Session + Session session = Session.getDefaultInstance(props, null); + session.setDebug(false); + + // create a message + Message msg = new MimeMessage(session); + + // set the from and to address + InternetAddress addressFrom = new InternetAddress("ace-report@umiacs.umd.edu"); + msg.setFrom(addressFrom); + + String[] to = (recipients == null ? null : recipients.split("\\s*,\\s*")); + if (to == null) { + throw new RuntimeException("No email recipients to send to"); + } + + InternetAddress[] addressTo = new InternetAddress[to.length]; + for (int i = 0; i < to.length; i++) { + addressTo[i] = new InternetAddress(to[i]); + } + addressTo[0] = new InternetAddress(recipients); + msg.setRecipients(Message.RecipientType.TO, addressTo); + + // Setting the Subject and Content Type + msg.setSubject("Ace Report: Test Reporter"); + msg.setContent(body, "text/plain"); + Transport.send(msg); + // LOG.trace("Successfully mailed report to: " + Strings.join(',', mailList)); + } + + private String createBody(Report report) { + StringBuilder sb = new StringBuilder(); + + sb.append("Report Name: \t"); + sb.append(report.getSession()); + sb.append("\r\n"); + + sb.append("Collection: \t"); + sb.append(report.getCollection().getName()); + sb.append("\r\n"); + + sb.append("Generated on: \t"); + sb.append(report.getGenerated()); + sb.append("\r\n"); + + sb.append("Start Date: \t"); + sb.append(report.getStart()); + sb.append("\r\n"); + + sb.append("End Date: \t"); + sb.append(report.getEnd()); + sb.append("\r\n"); + + sb.append("\r\n----------------------\r\nCollection summary\r\n\r\n"); + sb.append("Collection state: \t").append(report.getCollection().getState()).append("\r\n"); + sb.append("Errors: \t").append(report.getInfo().getErrors().size()).append("\r\n"); + + sb.append("\r\n----------------------\r\nTotal Log Entries\r\n\r\n"); + for (Map.Entry> entry : grouped.entrySet()) { + sb.append(entry.getKey().getShortName()) + .append(": \t") + .append(entry.getValue().size()) + .append("\r\n"); + } + + + /* + for ( ReportItem ri : summaryItems ) { + if ( !ri.isLogType() ) { + sb.append(ri.getAttribute()); + sb.append(": \t"); + sb.append(ri.getValue()); + sb.append("\r\n"); + } + } + sb.append("\r\n----------------------\r\nTotal Log Entries\r\n\r\n"); + for ( ReportItem ri : summaryItems ) { + if ( ri.isLogType() ) { + sb.append(ri.getAttribute()); + sb.append(": \t"); + sb.append(ri.getValue()); + sb.append("\r\n"); + } + } + */ + return sb.toString(); + } +} diff --git a/ace-am/src/main/java/edu/umiacs/ace/remote/JsonGateway.java b/ace-am/src/main/java/edu/umiacs/ace/remote/JsonGateway.java index 4a3b2224f3659e3744b5794e8b93cf3a6aa97c2e..d69e5047458b51721ec9ea28a8eef8a2ae737211 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/remote/JsonGateway.java +++ b/ace-am/src/main/java/edu/umiacs/ace/remote/JsonGateway.java @@ -264,12 +264,12 @@ public class JsonGateway { private long statusUpdate = 0; private SummaryBean summary = null; private long summaryUpdate = 0; - private Map reportMap = new HashMap(); - private Map reportMapUpdate = new HashMap(); - private Map itemMap = new HashMap(); - private Map itemMapUpdate = new HashMap(); - private Map itemRootMap = new HashMap(); - private Map itemRootMapUpdate = new HashMap(); + private Map reportMap = new HashMap<>(); + private Map reportMapUpdate = new HashMap<>(); + private Map itemMap = new HashMap<>(); + private Map itemMapUpdate = new HashMap<>(); + private Map itemRootMap = new HashMap<>(); + private Map itemRootMapUpdate = new HashMap<>(); } private T readJSONValue( URL u, String auth, Class clazz ) throws IOException { diff --git a/ace-am/src/main/java/edu/umiacs/ace/rest/CollectionManagement.java b/ace-am/src/main/java/edu/umiacs/ace/rest/CollectionManagement.java index 6547bd87922e9458e90d2e1f5f6330029418a33b..fe801bed2c8eeea79bfde0680640f1f90dbbda75 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/rest/CollectionManagement.java +++ b/ace-am/src/main/java/edu/umiacs/ace/rest/CollectionManagement.java @@ -11,6 +11,8 @@ import edu.umiacs.ace.monitor.access.CollectionCountContext; import edu.umiacs.ace.monitor.audit.AuditThreadFactory; import edu.umiacs.ace.monitor.core.Collection; import edu.umiacs.ace.monitor.core.MonitoredItem; +import edu.umiacs.ace.monitor.reporting.ReportPlugin; +import edu.umiacs.ace.rest.model.RPluginRequest; import edu.umiacs.ace.util.PersistUtil; import org.apache.log4j.Logger; import org.codehaus.jettison.json.JSONException; @@ -36,6 +38,8 @@ import java.util.List; /** * REST Service for adding and viewing collections * + * TODO: We should do /{id}/verb instead of {/verb/{id}} + * * @author shake */ @Path("collection") @@ -68,11 +72,45 @@ public class CollectionManagement { return false; } + @POST + @Consumes(MediaType.APPLICATION_JSON) + @Path("plugin/{id}") + public Response configurePlugin(@PathParam("id") long id, RPluginRequest request) { + EntityManager em = PersistUtil.getEntityManager(); + Collection c = em.find(Collection.class, id); + + if (c == null || !RPluginRequest.isValid(request)) { + return Response.status(400).build(); + } + + ReportPlugin plugin = new ReportPlugin() + .setNamedClass(request.getNamedClass()) + .setParent(c); + + if (c.getReportPlugins().contains(plugin)) { + return Response.status(400).build(); + } + + c.getReportPlugins().add(plugin); + // TODO: Ensure settings is at least an empty map + c.getSettings().putAll(request.getSettings()); + + EntityTransaction transaction = em.getTransaction(); + transaction.begin(); + em.persist(plugin); + em.merge(c); + transaction.commit(); + em.close(); + + return Response.ok().build(); + } + @POST @Path("audit/{id}") - public Response startAudit(@PathParam("id")long id) { + public Response startAudit(@PathParam("id") long id) { EntityManager em = PersistUtil.getEntityManager(); Collection c = em.find(Collection.class, id); + em.close(); if ( c == null ) { return Response.status(Status.NOT_FOUND).build(); } diff --git a/ace-am/src/main/java/edu/umiacs/ace/rest/model/RPluginRequest.java b/ace-am/src/main/java/edu/umiacs/ace/rest/model/RPluginRequest.java new file mode 100644 index 0000000000000000000000000000000000000000..99ca79b5c57b13bcbe4094bac7d6299a5426f73e --- /dev/null +++ b/ace-am/src/main/java/edu/umiacs/ace/rest/model/RPluginRequest.java @@ -0,0 +1,35 @@ +package edu.umiacs.ace.rest.model; + +import java.util.Map; + +/** + * + * Created by shake on 6/21/16. + */ +public class RPluginRequest { + + private String namedClass; + private Map settings; + + public String getNamedClass() { + return namedClass; + } + + public RPluginRequest setNamedClass(String namedClass) { + this.namedClass = namedClass; + return this; + } + + public Map getSettings() { + return settings; + } + + public RPluginRequest setSettings(Map settings) { + this.settings = settings; + return this; + } + + public static boolean isValid(RPluginRequest request) { + return request != null && request.namedClass != null; + } +} diff --git a/ace-am/src/main/resources/META-INF/persistence.xml b/ace-am/src/main/resources/META-INF/persistence.xml index ff2ccf4b97528f5e09506402c62945bdd404fa75..1119ded74276a19c0377c424f68bc94912c73d70 100644 --- a/ace-am/src/main/resources/META-INF/persistence.xml +++ b/ace-am/src/main/resources/META-INF/persistence.xml @@ -20,6 +20,7 @@ edu.umiacs.ace.monitor.core.MonitoredItem edu.umiacs.ace.driver.swap.SwapSettings edu.umiacs.ace.monitor.settings.SettingsParameter + edu.umiacs.ace.monitor.reporting.ReportPlugin true diff --git a/ace-am/src/main/sql/ace-am.sql b/ace-am/src/main/sql/ace-am.sql index f06618206e8892ec421f27953003a9fbb992658d..27540bd3ac961928db758c712368e77ac6f794c0 100644 --- a/ace-am/src/main/sql/ace-am.sql +++ b/ace-am/src/main/sql/ace-am.sql @@ -20,9 +20,9 @@ CREATE TABLE `monitored_item` ( `LASTSEEN` datetime default NULL, `STATECHANGE` datetime default NULL, `LASTVISITED` datetime default NULL, - `PARENTPATH` varchar(512) character set utf8 collate utf8_general_ci default NULL, + `PARENTPATH` text character set utf8 collate utf8_general_ci default NULL, `STATE` char(1) NOT NULL, - `PATH` varchar(512) character set utf8 collate utf8_general_ci default NULL, + `PATH` text character set utf8 collate utf8_general_ci default NULL, `DIRECTORY` tinyint(1) NOT NULL default '0', `TOKEN_ID` bigint(20) default NULL, `PARENTCOLLECTION_ID` bigint(20) NOT NULL, @@ -218,3 +218,12 @@ CREATE TABLE `system_settings` ( PRIMARY KEY (`ID`), UNIQUE (`ATTR`) ) ENGINE=MyISAM DEFAULT CHARSET=utf8; + +CREATE TABLE `report_plugin` ( + `ID` bigint(20) NOT NULL auto_increment, + `PARENT_ID` bigint(20) NOT NULL, + `NAMEDCLASS` varchar(255) NOT NULL, + UNIQUE `pc_idx`(`PARENT_ID`, `NAMEDCLASS`), + PRIMARY KEY (`ID`) +) ENGINE=MyISAM DEFAULT CHARSET=utf8; + diff --git a/ace-am/src/main/webapp/WEB-INF/web.xml b/ace-am/src/main/webapp/WEB-INF/web.xml index 25b297515b3f31a02db46e821f0b4991dab0c070..035e566f6ed992cfe281ccb09beaeb477a9cc403 100644 --- a/ace-am/src/main/webapp/WEB-INF/web.xml +++ b/ace-am/src/main/webapp/WEB-INF/web.xml @@ -80,6 +80,10 @@ Initial ingestion configuration edu.umiacs.ace.monitor.register.IngestContextListener + + Reporter scanning + edu.umiacs.ace.monitor.reporting.ReporterContextListener + ServletAdaptor com.sun.jersey.spi.container.servlet.ServletContainer @@ -202,6 +206,11 @@ DeleteSettingsServlet edu.umiacs.ace.monitor.settings.DeleteSettingsServlet + + ManageReporter + edu.umiacs.ace.monitor.reporting.ManageReporterServlet + + ServletAdaptor /rest/* @@ -323,6 +332,10 @@ DeleteSettingsServlet /DeleteSettings + + ManageReporter + /ManageReporter + 30 diff --git a/ace-am/src/main/webapp/collectionmodify.jsp b/ace-am/src/main/webapp/collectionmodify.jsp index e2e735b86c84832e1d46a1a0b1134a87fdb5247a..1afd5625fe83fcfc11209ae0090df5e27be747ac 100644 --- a/ace-am/src/main/webapp/collectionmodify.jsp +++ b/ace-am/src/main/webapp/collectionmodify.jsp @@ -145,7 +145,19 @@ Author : toaster - + + + + Configured Plugin + ${plugin.namedClass} Remove + + + + + Configure reporting + Configure reporting which occurs after auditing + + Peer Collection diff --git a/ace-am/src/main/webapp/configure_reporting.jsp b/ace-am/src/main/webapp/configure_reporting.jsp new file mode 100644 index 0000000000000000000000000000000000000000..d3c5cbb0750246cc5cc19732a1edbb60b20e3967 --- /dev/null +++ b/ace-am/src/main/webapp/configure_reporting.jsp @@ -0,0 +1,98 @@ + +<%@page pageEncoding="UTF-8"%> +<%-- +The taglib directive below imports the JSTL library. If you uncomment it, +you must also add the JSTL library to the project. The Add Library... action +on Libraries node in Projects view can be used to add the JSTL 1.1 library. +--%> +<%@taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"%> +<%@ taglib prefix="fn" uri="http://java.sun.com/jsp/jstl/functions" %> + + + + + + + + + + Modify Audit Reporting + + + + + + +

+

Configure audit reporting for ${workingCollection.collection.name}

+ +
+ + +
1. Select Plugin: + +
+ 2. Configure Collection: +
+
+
+
+
+
+ + + + diff --git a/ace-am/src/main/webapp/settings.jsp b/ace-am/src/main/webapp/settings.jsp index b672f2fd825023a791bca82153b0c80ea7f53608..fe6c8f45976906d6c5b101506faa9be09d43bf6b 100644 --- a/ace-am/src/main/webapp/settings.jsp +++ b/ace-am/src/main/webapp/settings.jsp @@ -157,7 +157,7 @@
-
UMIACS Connection:
+
ACE Log Level:
diff --git a/pom.xml b/pom.xml index a25e606abb1056a19a4e8b6d048521bedc644ea8..8d0a8879ad7d2718c557dbf2057a5d39cf410b69 100644 --- a/pom.xml +++ b/pom.xml @@ -17,10 +17,10 @@ org.apache.maven.plugins maven-compiler-plugin - 2.0.2 + 3.1 - 1.7 - 1.7 + 1.8 + 1.8 @@ -91,7 +91,7 @@ - ${project.version} (hudson build:${BUILD_NUMBER} revision:${SVN_REVISION}) + ${project.version} @@ -100,6 +100,7 @@ ace-ims-ws ace-ims-api ace-am + ace-am-api ace-dist audit-core ace-ims-server