From 075b6a3279e7d92b8ca12b0ee8583ba0f5864f36 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Wed, 15 Jun 2016 16:16:26 -0400 Subject: [PATCH 01/39] Add ace-am-api for reporting plugins --- ace-am-api/pom.xml | 17 +++++++++++++++++ ace-am/pom.xml | 6 ++++++ pom.xml | 3 ++- 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 ace-am-api/pom.xml diff --git a/ace-am-api/pom.xml b/ace-am-api/pom.xml new file mode 100644 index 0000000..c6df989 --- /dev/null +++ b/ace-am-api/pom.xml @@ -0,0 +1,17 @@ + + + 4.0.0 + + + edu.umiacs.ace + ace + 1.11-SNAPSHOT + + + ace-am-api + 1.11-SNAPSHOT + + + \ No newline at end of file diff --git a/ace-am/pom.xml b/ace-am/pom.xml index 869422c..b12256c 100644 --- a/ace-am/pom.xml +++ b/ace-am/pom.xml @@ -85,6 +85,12 @@ + + edu.umiacs.ace + ace-am-api + ${project.version} + + edu.umiacs.ace ace-ims-api diff --git a/pom.xml b/pom.xml index a25e606..b2e1e08 100644 --- a/pom.xml +++ b/pom.xml @@ -91,7 +91,7 @@ - ${project.version} (hudson build:${BUILD_NUMBER} revision:${SVN_REVISION}) + ${project.version} (hudson build:${BUILD_NUMBER} revision:${GIT_REVISION}) @@ -100,6 +100,7 @@ ace-ims-ws ace-ims-api ace-am + ace-am-api ace-dist audit-core ace-ims-server -- GitLab From cfe9415a77df289d4ec8aeff300c07c91e25f2ce Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Wed, 15 Jun 2016 16:22:12 -0400 Subject: [PATCH 02/39] Add two interfaces for reporting/gathering info from ace --- ace-am-api/src/edu/umiacs/ace/am/ReportInfo.java | 16 ++++++++++++++++ ace-am-api/src/edu/umiacs/ace/am/Reporter.java | 12 ++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 ace-am-api/src/edu/umiacs/ace/am/ReportInfo.java create mode 100644 ace-am-api/src/edu/umiacs/ace/am/Reporter.java diff --git a/ace-am-api/src/edu/umiacs/ace/am/ReportInfo.java b/ace-am-api/src/edu/umiacs/ace/am/ReportInfo.java new file mode 100644 index 0000000..4db21d2 --- /dev/null +++ b/ace-am-api/src/edu/umiacs/ace/am/ReportInfo.java @@ -0,0 +1,16 @@ +package edu.umiacs.ace.am; + +import java.util.List; + +/** + * Interface from which we get information about our audit; implemented in am to allow db access + * + * + * Created by shake on 6/15/16. + */ +public interface ReportInfo { + + List getErrors(); + List getLogEntries(); + +} diff --git a/ace-am-api/src/edu/umiacs/ace/am/Reporter.java b/ace-am-api/src/edu/umiacs/ace/am/Reporter.java new file mode 100644 index 0000000..eae5793 --- /dev/null +++ b/ace-am-api/src/edu/umiacs/ace/am/Reporter.java @@ -0,0 +1,12 @@ +package edu.umiacs.ace.am; + +/** + * Interface from which we send Reports out + * + * Created by shake on 6/13/16. + */ +public interface Reporter { + + void report(Report report); + +} -- GitLab From 4fbcbab6bc2a02cab315f3a409084c4f1fe6fc94 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Wed, 15 Jun 2016 16:22:59 -0400 Subject: [PATCH 03/39] Initial base data classes for reporting --- .../src/edu/umiacs/ace/am/Collection.java | 92 +++++++++ ace-am-api/src/edu/umiacs/ace/am/Item.java | 111 ++++++++++ ace-am-api/src/edu/umiacs/ace/am/LogEnum.java | 189 ++++++++++++++++++ ace-am-api/src/edu/umiacs/ace/am/LogItem.java | 81 ++++++++ ace-am-api/src/edu/umiacs/ace/am/Report.java | 75 +++++++ 5 files changed, 548 insertions(+) create mode 100644 ace-am-api/src/edu/umiacs/ace/am/Collection.java create mode 100644 ace-am-api/src/edu/umiacs/ace/am/Item.java create mode 100644 ace-am-api/src/edu/umiacs/ace/am/LogEnum.java create mode 100644 ace-am-api/src/edu/umiacs/ace/am/LogItem.java create mode 100644 ace-am-api/src/edu/umiacs/ace/am/Report.java diff --git a/ace-am-api/src/edu/umiacs/ace/am/Collection.java b/ace-am-api/src/edu/umiacs/ace/am/Collection.java new file mode 100644 index 0000000..86789df --- /dev/null +++ b/ace-am-api/src/edu/umiacs/ace/am/Collection.java @@ -0,0 +1,92 @@ +package edu.umiacs.ace.am; + +import java.util.Date; + +/** + * Simplified view of a collection + * + * Created by shake on 6/13/16. + */ +public class Collection { + + private static final long serialVersionUID = 1L; + + private String name; + private String group; + + private Date lastSync; + private State state; + + private String digestAlgorithm; + + public String getName() { + return name; + } + + public Collection setName(String name) { + this.name = name; + return this; + } + + public String getGroup() { + return group; + } + + public Collection setGroup(String group) { + this.group = group; + return this; + } + + public Date getLastSync() { + return lastSync; + } + + public Collection setLastSync(Date lastSync) { + this.lastSync = lastSync; + return this; + } + + public State getState() { + return state; + } + + public Collection setState(char state) { + this.state = State.fromIdentifier(state); + return this; + } + + public String getDigestAlgorithm() { + return digestAlgorithm; + } + + public Collection setDigestAlgorithm(String digestAlgorithm) { + this.digestAlgorithm = digestAlgorithm; + return this; + } + + enum State { + + ACTIVE('A', "Active"), + NEW('N', "New"), + ERROR('E', "Error"); + + final char identifier; + final String description; + + State(char identifier, String description) { + this.identifier = identifier; + this.description = description; + } + + static State fromIdentifier(char identifier) { + switch (identifier) { + case 'A': return ACTIVE; + case 'N': return NEW; + case 'E': return ERROR; + default: + throw new IllegalArgumentException("Identifier " + identifier + " does not match any known type"); + } + } + + } +} diff --git a/ace-am-api/src/edu/umiacs/ace/am/Item.java b/ace-am-api/src/edu/umiacs/ace/am/Item.java new file mode 100644 index 0000000..b7c5c0d --- /dev/null +++ b/ace-am-api/src/edu/umiacs/ace/am/Item.java @@ -0,0 +1,111 @@ +package edu.umiacs.ace.am; + +import java.util.Date; + +/** + * Simplified view of a MonitoredItem from aceam + * + * + * Created by shake on 6/13/16. + */ +public class Item { + + // path relative to base directory + private String path; + + private Date lastSeen; + private Date stateChange; + private Date lastVisited; + + private Collection parentCollection; + + // registered digest + private String fileDigest; + + // current digest + private String lastDigest; + + private char state; + private long size; + + public String getPath() { + return path; + } + + public Item setPath(String path) { + this.path = path; + return this; + } + + public Date getLastSeen() { + return lastSeen; + } + + public Item setLastSeen(Date lastSeen) { + this.lastSeen = lastSeen; + return this; + } + + public Date getStateChange() { + return stateChange; + } + + public Item setStateChange(Date stateChange) { + this.stateChange = stateChange; + return this; + } + + public Date getLastVisited() { + return lastVisited; + } + + public Item setLastVisited(Date lastVisited) { + this.lastVisited = lastVisited; + return this; + } + + public Collection getParentCollection() { + return parentCollection; + } + + public Item setParentCollection(Collection parentCollection) { + this.parentCollection = parentCollection; + return this; + } + + public String getFileDigest() { + return fileDigest; + } + + public Item setFileDigest(String fileDigest) { + this.fileDigest = fileDigest; + return this; + } + + public String getLastDigest() { + return lastDigest; + } + + public Item setLastDigest(String lastDigest) { + this.lastDigest = lastDigest; + return this; + } + + public char getState() { + return state; + } + + public Item setState(char state) { + this.state = state; + return this; + } + + public long getSize() { + return size; + } + + public Item setSize(long size) { + this.size = size; + return this; + } +} diff --git a/ace-am-api/src/edu/umiacs/ace/am/LogEnum.java b/ace-am-api/src/edu/umiacs/ace/am/LogEnum.java new file mode 100644 index 0000000..da7c74a --- /dev/null +++ b/ace-am-api/src/edu/umiacs/ace/am/LogEnum.java @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2007-2010, University of Maryland + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided + * that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions + * and the following disclaimer in the documentation and/or other materials provided with the + * distribution. + * + * Neither the name of the University of Maryland nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR + * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ACE Components were written in the ADAPT Project at the University of + * Maryland Institute for Advanced Computer Study. + */ +// $Id$ + +package edu.umiacs.ace.am; + +/** + * Enumeration of all possible log types used in LogEvent + * + * @author toaster + */ +public enum LogEnum { + + LOG_TYPE_UNKNOWN(0, "Unknown", "Unknown Log Entry Type"), + /** + * New file registered into the system + */ + FILE_NEW(1, "New File", "New file added for auditing"), + /** + * Previously registered file no longer exists + */ + FILE_MISSING(2, "File Missing", "File could not be accessed for auditing"), + /** + * file checksums do not match + */ + FILE_CORRUPT(3, "Corrupt File", "File's checksum does not match stored checksum"), + /** + * Found a previously missing file and marked it intact + */ + FILE_ONLINE(4, "File Online", "Previously offline file is available"), + ADD_TOKEN(5, "Add Token", "New token from IMS added for this file"), + CREATE_TOKEN_ERROR(6, "Create Token Error", "Error creating token for this file"), + MISSING_TOKEN(7, "Missing Token", "File is registered, but has no token"), + REMOVE_ITEM(8, "Item Removed", "File or directory and tokens removed from monitoring"), + TOKEN_INVALID(9, "Digest Token Mismatch", "Possibly corrupt checksum or proof, local hash with proof does not match IMS CSI"), + SITE_UNACCESSIBLE(10, "Collection Unaccessible", "Unable to connect to collection for auditing"), //TODO + /** + * Error reading file from local storage + */ + ERROR_READING(11, "Read Error", "Error reading the file locally, check local file"), + /** + * unknown error returned from ims + */ + UNKNOWN_IMS_COMMUNICATION_ERROR(12, "IMS Error", "Unkown error trying to communicate with the IMS, check IMS logs"), + /** + * token state changed from invalid token to valid, this should be VERY VERY rare + */ + TOKEN_VALID(13, "Token Digest Revalidated", "hash and proof match CSI"), + REMOTE_FILE_CORRUPT(14, "Remote File Corrupt", "remote file digest differs from local file"), + REMOTE_FILE_MISSING(15, "Remote File Missing", "not corresponding file on remote site"), + REMOTE_FILE_ONLINE(16, "Remote File Online", "Remote file marked as active"), + REMOVE_STORAGE_DRIVER(17, "Remove Storage Driver", "Configured storage driver was removed"), + //STORAGE_DRIVER_ADDED(18, "Add Storage Driver", "New Storage Driver configured"), + COLLECTION_REGISTERED(19,"Collection Registered", "Collection registered for monitoring"), + /////////// audit start/stops + /** + * Start a synchronization run on a master site + */ + FILE_AUDIT_START(20, "File Audit Start", "Auditing of this collection's files started"), + /** + * Finish a sync run on a master site + */ + FILE_AUDIT_FINISH(21, "File Audit Finish", "Auditing of this collection's files finished"), + /** + * Start a synchronization run on a master site + */ + TOKEN_AUDIT_START(22, "Token Audit Start", "Auditing of this collection's tokens started"), + /** + * Finish a sync run on a master site + */ + TOKEN_AUDIT_FINISH(23, "Token Audit Finish", "Auditing of this collection's tokens finished"), + TOKEN_INGEST_UPDATE(24, "Token Ingest Update", "Token was out of date and has been updated"), + FILE_REGISTER(25, "File Registered", "New file registered but is not ready for auditing"), + FILE_AUDIT_FALLBACK(26, "File Audit Fallback", "File Audit could not connect to the IMS, falling back to audit-only mode"), + SMTP_ERROR(27, "SMTP Communication Error", "Could not connect to designated SMTP host"), + SYSTEM_ERROR(99, "System Error", "Unknown system error occurred, check server logs"); + private int type; + private String shortName; + private String details; + + LogEnum(int i, String shortName, String details ) { + this.type = i; + this.shortName = shortName; + this.details = details; + } + + public int getType() { + return type; + } + + public String getDetails() { + return details; + } + + public String getShortName() { + return shortName; + } + + public static LogEnum valueOf( int i ) { + switch ( i ) { + case 0: + return LOG_TYPE_UNKNOWN; + case 1: + return FILE_NEW; + case 2: + return FILE_MISSING; + case 3: + return FILE_CORRUPT; + case 4: + return FILE_ONLINE; + case 5: + return ADD_TOKEN; + case 6: + return CREATE_TOKEN_ERROR; + case 7: + return MISSING_TOKEN; + case 8: + return REMOVE_ITEM; + case 9: + return TOKEN_INVALID; + case 10: + return SITE_UNACCESSIBLE; + case 11: + return ERROR_READING; + case 12: + return UNKNOWN_IMS_COMMUNICATION_ERROR; + case 13: + return TOKEN_VALID; + case 14: + return REMOTE_FILE_CORRUPT; + case 15: + return REMOTE_FILE_MISSING; + case 16: + return REMOTE_FILE_ONLINE; + case 17: + return REMOVE_STORAGE_DRIVER; +// case 18: +// return STORAGE_DRIVER_ADDED; + case 19: + return COLLECTION_REGISTERED; + case 20: + return FILE_AUDIT_START; + case 21: + return FILE_AUDIT_FINISH; + case 22: + return TOKEN_AUDIT_START; + case 23: + return TOKEN_AUDIT_FINISH; + case 24: + return TOKEN_INGEST_UPDATE; + case 25: + return FILE_REGISTER; + case 26: + return FILE_AUDIT_FALLBACK; + + case 99: + return SYSTEM_ERROR; + } + + throw new IllegalArgumentException("Int " + i + " does not match any known type"); + } +} diff --git a/ace-am-api/src/edu/umiacs/ace/am/LogItem.java b/ace-am-api/src/edu/umiacs/ace/am/LogItem.java new file mode 100644 index 0000000..68cd55d --- /dev/null +++ b/ace-am-api/src/edu/umiacs/ace/am/LogItem.java @@ -0,0 +1,81 @@ +package edu.umiacs.ace.am; + +import java.util.Date; + +/** + * + * Created by shake on 6/14/16. + */ +public class LogItem { + + private LogEnum type; + private String description; + private long session; + private String path; + private Collection collection; + private Date date; + + public LogEnum getType() { + return type; + } + + public LogItem setType(LogEnum type) { + this.type = type; + return this; + } + + public String getDescription() { + return description; + } + + public LogItem setDescription(String description) { + this.description = description; + return this; + } + + public long getSession() { + return session; + } + + public LogItem setSession(long session) { + this.session = session; + return this; + } + + public String getPath() { + return path; + } + + public LogItem setPath(String path) { + this.path = path; + return this; + } + + public Collection getCollection() { + return collection; + } + + public LogItem setCollection(Collection collection) { + this.collection = collection; + return this; + } + + public Date getDate() { + return date; + } + + public LogItem setDate(Date date) { + this.date = date; + return this; + } + + /** + * What we want: + * Map ??? + * -> Want to map from a log type to the list of log entries for that type + * -> can keep the LogItem (hold all the columns) in its own class + * -> Have a LogGroup class which holds the grouping of log items + * -> LogGroup should keep a Cursor for iterating over the entries + */ + +} diff --git a/ace-am-api/src/edu/umiacs/ace/am/Report.java b/ace-am-api/src/edu/umiacs/ace/am/Report.java new file mode 100644 index 0000000..d3eeba3 --- /dev/null +++ b/ace-am-api/src/edu/umiacs/ace/am/Report.java @@ -0,0 +1,75 @@ +package edu.umiacs.ace.am; + +import java.util.Date; + +/** + * Class which makes up a Report to send out + * Should contain all the necessary information from an audit + * + * Created by shake on 6/13/16. + */ +public class Report { + + private Date generated; + private Date start; + private Date end; + + private long session; + + private Collection collection; + private ReportInfo info; + + public Collection getCollection() { + return collection; + } + + public Report setCollection(Collection collection) { + this.collection = collection; + return this; + } + + public long getSession() { + return session; + } + + public Report setSession(long session) { + this.session = session; + return this; + } + + public Date getEnd() { + return end; + } + + public Report setEnd(Date end) { + this.end = end; + return this; + } + + public Date getStart() { + return start; + } + + public Report setStart(Date start) { + this.start = start; + return this; + } + + public Date getGenerated() { + return generated; + } + + public Report setGenerated(Date generated) { + this.generated = generated; + return this; + } + + public ReportInfo getInfo() { + return info; + } + + public Report setInfo(ReportInfo info) { + this.info = info; + return this; + } +} -- GitLab From 0e35e64d0d06b30808b03ba501da1d93800496a2 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Wed, 15 Jun 2016 16:23:24 -0400 Subject: [PATCH 04/39] Add named query for selecting log events --- ace-am/src/main/java/edu/umiacs/ace/monitor/log/LogEvent.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/log/LogEvent.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/log/LogEvent.java index 5f5b186..108a315 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/log/LogEvent.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/log/LogEvent.java @@ -54,7 +54,9 @@ import javax.persistence.Temporal; @Table(name = "logevent") @NamedQueries({ @NamedQuery(name = "LogEvent.deleteByCollection", query = - "DELETE FROM LogEvent e WHERE e.collection = :coll") + "DELETE FROM LogEvent e WHERE e.collection = :coll"), + @NamedQuery(name= "LogEvent.selectByCollectionAndSession", query= + "SELECT e FROM LogEvent e WHERE e.collection = :coll AND e.session = :session") }) public class LogEvent implements Serializable { -- GitLab From 61508c53426e98617eebd490698255a33e398b23 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Wed, 15 Jun 2016 16:23:41 -0400 Subject: [PATCH 05/39] ReportInfo implementation --- .../ace/monitor/reporting/DbReportInfo.java | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/DbReportInfo.java diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/DbReportInfo.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/DbReportInfo.java new file mode 100644 index 0000000..fb14d60 --- /dev/null +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/DbReportInfo.java @@ -0,0 +1,81 @@ +package edu.umiacs.ace.monitor.reporting; + +import edu.umiacs.ace.am.Item; +import edu.umiacs.ace.am.LogEnum; +import edu.umiacs.ace.am.LogItem; +import edu.umiacs.ace.am.ReportInfo; +import edu.umiacs.ace.monitor.core.Collection; +import edu.umiacs.ace.monitor.core.MonitoredItem; +import edu.umiacs.ace.monitor.log.LogEvent; +import edu.umiacs.ace.util.PersistUtil; + +import javax.persistence.EntityManager; +import javax.persistence.Query; +import java.util.ArrayList; +import java.util.List; + +/** + * Class which we supply to a report to get back information from the database + * + * Created by shake on 6/15/16. + */ +public class DbReportInfo implements ReportInfo { + + private long session; + private Collection collection; + + public DbReportInfo(Collection collection, long session) { + this.collection = collection; + this.session = session; + } + + @Override + public List getErrors() { + List errorItems = new ArrayList<>(); + + EntityManager em = PersistUtil.getEntityManager(); + Query query = em.createNamedQuery("MonitoredItem.listLocalErrors"); + query.setParameter("coll", collection); + List errors = query.getResultList(); + em.close(); + + for (MonitoredItem error : errors) { + Item errorItem = new Item(); + errorItem.setState(error.getState()); + errorItem.setFileDigest(error.getFileDigest()); + // errorItem.setLastDigest() + errorItem.setLastSeen(error.getLastSeen()); + errorItem.setPath(error.getPath()); + errorItem.setSize(error.getSize()); + errorItem.setStateChange(error.getStateChange()); + errorItem.setLastVisited(error.getLastVisited()); + errorItems.add(errorItem); + } + + return errorItems; + } + + @Override + public List getLogEntries() { + List logItems = new ArrayList<>(); + + EntityManager em = PersistUtil.getEntityManager(); + Query query = em.createNamedQuery("LogEvent.selectByCollectionAndSession"); + query.setParameter("coll", collection); + query.setParameter("session", session); + List events = query.getResultList(); + em.close(); + + for (LogEvent event : events) { + LogItem item = new LogItem(); + item.setPath(event.getPath()); + item.setDate(event.getDate()); + item.setDescription(event.getDescription()); + item.setType(LogEnum.valueOf(event.getLogType())); + item.setSession(session); + logItems.add(item); + } + + return logItems; + } +} -- GitLab From 018a6f5416b83bc28962f5c440c56feb46d6689b Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Wed, 15 Jun 2016 16:24:07 -0400 Subject: [PATCH 06/39] Add test reporter --- .../ace/monitor/reporting/SMTPReporter.java | 139 ++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/SMTPReporter.java diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/SMTPReporter.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/SMTPReporter.java new file mode 100644 index 0000000..7e36e8c --- /dev/null +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/SMTPReporter.java @@ -0,0 +1,139 @@ +package edu.umiacs.ace.monitor.reporting; + +import edu.umiacs.ace.am.LogEnum; +import edu.umiacs.ace.am.LogItem; +import edu.umiacs.ace.am.Report; +import edu.umiacs.ace.am.Reporter; +import org.apache.log4j.Logger; + +import javax.mail.Message; +import javax.mail.MessagingException; +import javax.mail.Session; +import javax.mail.Transport; +import javax.mail.internet.InternetAddress; +import javax.mail.internet.MimeMessage; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +/** + * Class which makes smtp class after an audit has finished + * + * Created by shake on 6/13/16. + */ +public class SMTPReporter implements Reporter { + + private static final Logger LOG = Logger.getLogger(SMTPReporter.class); + private Map> grouped = new HashMap<>(); + + @Override + public void report(Report report) { + groupLogEntries(report); + String body = createBody(report); + try { + send(body); + } catch (MessagingException e) { + LOG.error("Unable to send report", e); + } + } + + private void groupLogEntries(Report report) { + for (LogItem logItem : report.getInfo().getLogEntries()) { + LogEnum type = logItem.getType(); + List items = grouped.get(type); + if (items == null) { + items = new ArrayList<>(); + } + + items.add(logItem); + grouped.put(type, items); + } + } + + private void send(String body) throws MessagingException { + //Set the host smtp address + Properties props = new Properties(); + props.put("mail.smtp.host", "127.0.0.1"); + + // create some properties and get the default Session + Session session = Session.getDefaultInstance(props, null); + session.setDebug(false); + + // create a message + Message msg = new MimeMessage(session); + + // set the from and to address + InternetAddress addressFrom = new InternetAddress("ace-report@umiacs.umd.edu"); + msg.setFrom(addressFrom); + + InternetAddress[] addressTo = new InternetAddress[1]; + addressTo[0] = new InternetAddress("shake@umiacs.umd.edu"); + msg.setRecipients(Message.RecipientType.TO, addressTo); + + // Setting the Subject and Content Type + msg.setSubject("Ace Report: Test Reporter"); + msg.setContent(body, "text/plain"); + Transport.send(msg); + // LOG.trace("Successfully mailed report to: " + Strings.join(',', mailList)); + } + + private String createBody(Report report) { + StringBuilder sb = new StringBuilder(); + + sb.append("Report Name: \t"); + sb.append(report.getSession()); + sb.append("\r\n"); + + sb.append("Collection: \t"); + sb.append(report.getCollection().getName()); + sb.append("\r\n"); + + sb.append("Generated on: \t"); + sb.append(report.getGenerated()); + sb.append("\r\n"); + + sb.append("Start Date: \t"); + sb.append(report.getStart()); + sb.append("\r\n"); + + sb.append("End Date: \t"); + sb.append(report.getEnd()); + sb.append("\r\n"); + + sb.append("\r\n----------------------\r\nCollection summary\r\n\r\n"); + sb.append("Collection state: \t").append(report.getCollection().getState()).append("\r\n"); + sb.append("Errors: \t").append(report.getInfo().getErrors().size()).append("\r\n"); + + sb.append("\r\n----------------------\r\nTotal Log Entries\r\n\r\n"); + for (Map.Entry> entry : grouped.entrySet()) { + sb.append(entry.getKey().getShortName()) + .append(": \t") + .append(entry.getValue().size()) + .append("\r\n"); + } + + + /* + for ( ReportItem ri : summaryItems ) { + if ( !ri.isLogType() ) { + sb.append(ri.getAttribute()); + sb.append(": \t"); + sb.append(ri.getValue()); + sb.append("\r\n"); + } + } + sb.append("\r\n----------------------\r\nTotal Log Entries\r\n\r\n"); + for ( ReportItem ri : summaryItems ) { + if ( ri.isLogType() ) { + sb.append(ri.getAttribute()); + sb.append(": \t"); + sb.append(ri.getValue()); + sb.append("\r\n"); + } + } + */ + return sb.toString(); + } +} -- GitLab From 0af1067a12979537e0a26e78e3f43be0a1996f5e Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Wed, 15 Jun 2016 16:24:39 -0400 Subject: [PATCH 07/39] Build report/call reporter when audits end --- .../umiacs/ace/monitor/audit/AuditThread.java | 71 +++++++++++++++---- .../ace/monitor/audit/AuditThreadFactory.java | 9 +++ 2 files changed, 65 insertions(+), 15 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java index 904f4f8..9d6dfa6 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java @@ -30,50 +30,56 @@ // $Id$ package edu.umiacs.ace.monitor.audit; +import edu.umiacs.ace.am.Item; +import edu.umiacs.ace.am.Report; +import edu.umiacs.ace.am.ReportInfo; +import edu.umiacs.ace.am.Reporter; import edu.umiacs.ace.driver.AuditIterable; import edu.umiacs.ace.driver.DriverStateBean; -import edu.umiacs.ace.driver.filter.PathFilter; import edu.umiacs.ace.driver.FileBean; import edu.umiacs.ace.driver.StorageDriver; +import edu.umiacs.ace.driver.filter.PathFilter; +import edu.umiacs.ace.driver.filter.SimpleFilter; import edu.umiacs.ace.ims.api.IMSException; import edu.umiacs.ace.ims.api.IMSService; import edu.umiacs.ace.ims.api.TokenRequestBatch; import edu.umiacs.ace.ims.api.TokenValidator; import edu.umiacs.ace.ims.ws.TokenRequest; import edu.umiacs.ace.monitor.access.CollectionCountContext; -import edu.umiacs.ace.monitor.register.IngestThreadPool; -import edu.umiacs.ace.util.PersistUtil; -import edu.umiacs.ace.driver.filter.SimpleFilter; import edu.umiacs.ace.monitor.compare.CollectionCompare2; import edu.umiacs.ace.monitor.compare.CompareResults; -import edu.umiacs.ace.remote.JsonGateway; +import edu.umiacs.ace.monitor.core.Collection; +import edu.umiacs.ace.monitor.core.ConfigConstants; import edu.umiacs.ace.monitor.core.MonitoredItem; import edu.umiacs.ace.monitor.core.MonitoredItemManager; +import edu.umiacs.ace.monitor.log.LogEnum; +import edu.umiacs.ace.monitor.log.LogEvent; import edu.umiacs.ace.monitor.log.LogEventManager; +import edu.umiacs.ace.monitor.peers.PeerCollection; +import edu.umiacs.ace.monitor.reporting.DbReportInfo; import edu.umiacs.ace.monitor.reporting.ReportSummary; import edu.umiacs.ace.monitor.reporting.SchedulerContextListener; import edu.umiacs.ace.monitor.reporting.SummaryGenerator; -import edu.umiacs.ace.monitor.core.Collection; -import edu.umiacs.ace.monitor.core.ConfigConstants; import edu.umiacs.ace.monitor.settings.SettingsUtil; -import edu.umiacs.ace.monitor.log.LogEnum; -import edu.umiacs.ace.monitor.log.LogEvent; -import edu.umiacs.ace.monitor.peers.PeerCollection; +import edu.umiacs.ace.remote.JsonGateway; import edu.umiacs.ace.token.AceToken; +import edu.umiacs.ace.util.PersistUtil; import edu.umiacs.ace.util.TokenUtil; import edu.umiacs.util.Strings; +import org.apache.log4j.Logger; +import org.apache.log4j.NDC; + +import javax.mail.MessagingException; +import javax.persistence.EntityManager; +import javax.persistence.EntityTransaction; import java.io.InputStream; import java.security.MessageDigest; +import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import javax.mail.MessagingException; -import javax.persistence.EntityManager; -import javax.persistence.EntityTransaction; -import org.apache.log4j.Logger; -import org.apache.log4j.NDC; /** * @@ -280,6 +286,11 @@ public final class AuditThread extends Thread implements CancelCallback { if ( verbose ) { logAuditFinish(); generateAuditReport(); + try { + handleReporters(); + } catch (Exception e) { + LOG.error("Caught exception during extra reporting", e); + } } AuditThreadFactory.finished(coll); @@ -289,6 +300,36 @@ public final class AuditThread extends Thread implements CancelCallback { LOG.info("Exiting audit thread"); } + private void handleReporters() { + LOG.trace("Generating extra reports on " + session + " coll " + + coll.getName()); + Report report = new Report(); + ReportInfo info = new DbReportInfo(coll, session); + + edu.umiacs.ace.am.Collection c = new edu.umiacs.ace.am.Collection(); + + c.setDigestAlgorithm(coll.getDigestAlgorithm()); + c.setGroup(coll.getGroup()); + c.setLastSync(coll.getLastSync()); + c.setName(coll.getName()); + c.setState(coll.getState()); + + report.setEnd(new Date()); + report.setStart(new Date(session)); + report.setGenerated(new Date()); + + report.setInfo(info); + report.setCollection(c); + report.setSession(session); + + List reporters = AuditThreadFactory.getReporters(); + for (Reporter reporter : reporters) { + reporter.report(report); + } + LOG.trace("Finished extra reports on " + session + " coll " + + coll.getName()); + } + private boolean openIms() { try { IMSService ims; diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThreadFactory.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThreadFactory.java index bea5a44..66c45b8 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThreadFactory.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThreadFactory.java @@ -30,9 +30,12 @@ // $Id$ package edu.umiacs.ace.monitor.audit; +import com.google.common.collect.ImmutableList; +import edu.umiacs.ace.am.Reporter; import edu.umiacs.ace.driver.StorageDriver; import edu.umiacs.ace.monitor.core.Collection; import edu.umiacs.ace.monitor.core.MonitoredItem; +import edu.umiacs.ace.monitor.reporting.SMTPReporter; import edu.umiacs.ace.util.CollectionThreadPoolExecutor; import edu.umiacs.ace.util.KSFuture; import edu.umiacs.ace.util.PersistUtil; @@ -69,6 +72,8 @@ public class AuditThreadFactory { public static boolean blocking = false; public static int maxBlockTime = 0; + private static List reporters; + public static void setIMS( String ims ) { if (Strings.isEmpty(ims)) { LOG.error("Empty ims string, ignoring"); @@ -291,4 +296,8 @@ public class AuditThreadFactory { future.cancel(true); } } + + public static List getReporters() { + return ImmutableList.of(new SMTPReporter()); + } } -- GitLab From 516bdc5d7aa0c8e3763fd6b76ccb073508518a9b Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Mon, 20 Jun 2016 17:18:15 -0400 Subject: [PATCH 08/39] Add jsp for configuring plugins --- .../src/main/webapp/configure_reporting.jsp | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 ace-am/src/main/webapp/configure_reporting.jsp diff --git a/ace-am/src/main/webapp/configure_reporting.jsp b/ace-am/src/main/webapp/configure_reporting.jsp new file mode 100644 index 0000000..375701d --- /dev/null +++ b/ace-am/src/main/webapp/configure_reporting.jsp @@ -0,0 +1,98 @@ + +<%@page pageEncoding="UTF-8"%> +<%-- +The taglib directive below imports the JSTL library. If you uncomment it, +you must also add the JSTL library to the project. The Add Library... action +on Libraries node in Projects view can be used to add the JSTL 1.1 library. +--%> +<%@taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"%> +<%@ taglib prefix="fn" uri="http://java.sun.com/jsp/jstl/functions" %> + + + + + + + + + + Modify Audit Reporting + + + + + + +
+

Configure audit reporting for ${workingCollection.collection.name}

+ +
+ + +
1. Select Plugin: + +
+ 2. Configure Collection: +
+
+
+
+
+
+ + + + -- GitLab From ba9bdd20b9896143e6e7b4e80b7db987c3986220 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Mon, 20 Jun 2016 17:19:28 -0400 Subject: [PATCH 09/39] Add reporting plugins in the collectionmodify jsp --- ace-am/src/main/webapp/collectionmodify.jsp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/ace-am/src/main/webapp/collectionmodify.jsp b/ace-am/src/main/webapp/collectionmodify.jsp index e2e735b..f420ff6 100644 --- a/ace-am/src/main/webapp/collectionmodify.jsp +++ b/ace-am/src/main/webapp/collectionmodify.jsp @@ -145,7 +145,19 @@ Author : toaster - + + + + Configured Plugin + ${plugin.namedClass} Remove + + + + + Configure reporting + Configure reporting which occurs after auditing + + Peer Collection -- GitLab From 1b5989762cfaaf313f79f6d3e813c029e1445436 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Mon, 20 Jun 2016 17:20:35 -0400 Subject: [PATCH 10/39] Add context listener for scanning Reporters in the classpath --- .../reporting/ReporterContextListener.java | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/ReporterContextListener.java diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/ReporterContextListener.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/ReporterContextListener.java new file mode 100644 index 0000000..640c5fb --- /dev/null +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/ReporterContextListener.java @@ -0,0 +1,81 @@ +package edu.umiacs.ace.monitor.reporting; + +import com.google.common.collect.ImmutableMap; +import edu.umiacs.ace.am.Reporter; +import io.github.lukehutch.fastclasspathscanner.FastClasspathScanner; +import org.apache.log4j.Logger; + +import javax.servlet.ServletContextEvent; +import javax.servlet.ServletContextListener; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * + * Created by shake on 6/16/16. + */ +public class ReporterContextListener implements ServletContextListener { + private static final Logger log = Logger.getLogger(ReporterContextListener.class); + + public static Map map; + private static List reporters; + private static final String PAGE_REPORTERS = "reporters"; + + @Override + public void contextInitialized(ServletContextEvent servletContextEvent) { + reporters = new ArrayList<>(); + Map tempMap = new HashMap<>(); + + new FastClasspathScanner() + .matchClassesImplementing(Reporter.class, c -> { + Plugin plugin = new Plugin(c); + tempMap.put(c.getSimpleName(), plugin); + reporters.add(plugin); + }) + .scan(); + + log.info("Found " + reporters.size() + " reporting classes: " + reporters.toString()); + + map = ImmutableMap.copyOf(tempMap); + servletContextEvent.getServletContext().setAttribute(PAGE_REPORTERS, reporters); + } + + @Override + public void contextDestroyed(ServletContextEvent servletContextEvent) { + reporters.clear(); + } + + public static class Plugin { + private Class clazz; + + Plugin(Class clazz) { + this.clazz = clazz; + } + + public String getLongName() { + return this.clazz.getName(); + } + + public String getName() { + return clazz.getSimpleName(); + } + + public List getSettings() { + log.trace("Retrieving settings for " + getName()); + List settings = new ArrayList<>(); + + try { + Reporter reporter = Reporter.class.cast(clazz.newInstance()); + settings = reporter.settings(); + log.trace("Found " + settings.size() + " settings"); + } catch (InstantiationException | IllegalAccessException e) { + log.error("could not instantiate class " + clazz, e); + } + + return settings; + } + } + +} -- GitLab From 8098c7fc2d74f5ec554bd838b58d45f4b5824407 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Mon, 20 Jun 2016 17:45:07 -0400 Subject: [PATCH 11/39] Add entity to track plugins to collection association --- .../ace/monitor/reporting/ReportPlugin.java | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/ReportPlugin.java diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/ReportPlugin.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/ReportPlugin.java new file mode 100644 index 0000000..ee032c0 --- /dev/null +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/reporting/ReportPlugin.java @@ -0,0 +1,62 @@ +package edu.umiacs.ace.monitor.reporting; + +import edu.umiacs.ace.monitor.core.Collection; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.ManyToOne; +import javax.persistence.NamedQueries; +import javax.persistence.NamedQuery; +import javax.persistence.Table; +import java.io.Serializable; + +/** + * + * Created by shake on 6/20/16. + */ +@Entity +@Table(name = "report_plugin") +@NamedQueries({ + @NamedQuery(name = "ReportPlugin.deleteByCollection", query = + "DELETE FROM ReportPlugin WHERE parent = :coll") +}) +public class ReportPlugin implements Serializable { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @ManyToOne + private Collection parent; + private String namedClass; + + public Long getId() { + return id; + } + + public ReportPlugin setId(Long id) { + this.id = id; + return this; + } + + public Collection getParent() { + return parent; + } + + public ReportPlugin setParent(Collection parent) { + this.parent = parent; + return this; + } + + public String getNamedClass() { + return namedClass; + } + + public ReportPlugin setNamedClass(String namedClass) { + this.namedClass = namedClass; + return this; + } +} -- GitLab From 1dc8bb119cc04ca7df3878e75dc2773c1b4c4346 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Mon, 20 Jun 2016 17:45:37 -0400 Subject: [PATCH 12/39] Add new Entity to the persistence xml --- ace-am/src/main/resources/META-INF/persistence.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/ace-am/src/main/resources/META-INF/persistence.xml b/ace-am/src/main/resources/META-INF/persistence.xml index ff2ccf4..1119ded 100644 --- a/ace-am/src/main/resources/META-INF/persistence.xml +++ b/ace-am/src/main/resources/META-INF/persistence.xml @@ -20,6 +20,7 @@ edu.umiacs.ace.monitor.core.MonitoredItem edu.umiacs.ace.driver.swap.SwapSettings edu.umiacs.ace.monitor.settings.SettingsParameter + edu.umiacs.ace.monitor.reporting.ReportPlugin true -- GitLab From 9617c31908e668de2da35004af7ea1bdd644d77c Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Mon, 20 Jun 2016 17:46:07 -0400 Subject: [PATCH 13/39] Add fastclasspath scanner + java8 --- ace-am/pom.xml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/ace-am/pom.xml b/ace-am/pom.xml index b12256c..b44c4f5 100644 --- a/ace-am/pom.xml +++ b/ace-am/pom.xml @@ -7,7 +7,6 @@ 1.11-SNAPSHOT ../pom.xml - edu.umiacs.ace ace-am ace-am war @@ -42,6 +41,14 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + +