From 9c4959e99f6149a1837e7050a2d4797d70f713fc Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Thu, 22 Sep 2016 12:32:37 -0400 Subject: [PATCH 1/7] Add some notes --- .../java/edu/umiacs/ace/monitor/register/IngestThread.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/register/IngestThread.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/register/IngestThread.java index d8a2b2f..f9787a2 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/register/IngestThread.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/register/IngestThread.java @@ -62,7 +62,7 @@ import java.util.concurrent.RecursiveAction; public class IngestThread extends RecursiveAction { private static final Logger LOG = Logger.getLogger(IngestThread.class); - // These only gets read from, never written to + // These only get read from, never written to private Map tokens; private Collection coll; private List identifiers; @@ -173,7 +173,8 @@ public class IngestThread extends RecursiveAction { token.setParentCollection(coll); - // Token + // Token + // TODO: may be be able to do em.persist(item) and let the token get persisted along w/ it em.persist(event[1]); em.persist(token); item.setToken(token); -- GitLab From 528e742810b7e189fa7975682a502e57db658ea9 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Thu, 22 Sep 2016 12:33:46 -0400 Subject: [PATCH 2/7] Move search table css into style.css --- ace-am/src/main/webapp/status.jsp | 54 -------------- ace-am/src/main/webapp/style.css | 116 ++++++++++++++++++++++-------- 2 files changed, 88 insertions(+), 82 deletions(-) diff --git a/ace-am/src/main/webapp/status.jsp b/ace-am/src/main/webapp/status.jsp index 3306713..4d16aa2 100644 --- a/ace-am/src/main/webapp/status.jsp +++ b/ace-am/src/main/webapp/status.jsp @@ -133,56 +133,7 @@ background-color: #e8e8e8; } - #searchtable { - margin-top: 10px; - margin-bottom: -10px; - margin-left: auto; - margin-right: auto; - width: 650px; - } - - .input { - padding: 2px; - display: flex; - width: 650px; - } - - .input-group-addon { - border: 1px solid #ccc; - font-size: 12px; - background-color: #e8e8e8; - text-align: center; - width: 75px; - height: 20px; - line-height: 20px; - padding: 3px 10px; - } - - .form-input { - border: 1px solid #ccc; - width: 100%; - height: 20px; - padding: 3px 8px; - margin-left: -1px; - margin-right: 5px; - } - .form-select { - border: 1px solid #ccc; - width: 100%; - padding: 3px 8px; - margin-left: -3px; - margin-right: 5px; - } - - .btn { - padding: 2px; - width: 75px; - height: 25px; - border: 1px solid #e8e8e8; - margin-left: 2px; - margin-top: 2px; - } @@ -344,8 +295,6 @@ @@ -356,9 +305,6 @@ 1000 diff --git a/ace-am/src/main/webapp/style.css b/ace-am/src/main/webapp/style.css index 36fe6ca..3f4a43a 100644 --- a/ace-am/src/main/webapp/style.css +++ b/ace-am/src/main/webapp/style.css @@ -1,5 +1,5 @@ body { - font-family: Arial,Helvetica,sans-serif; + font-family: Arial, Helvetica, sans-serif; font-size: 10px; width: 750px; border: 1px solid #000000; @@ -13,10 +13,19 @@ h1 { margin-left: 10px; } +a:link, a:visited { + font-size: 12px; + color: #003388; + text-decoration: none; +} + +a:hover { + color: #0066CC; +} -a:link, a:visited {font-size: 12px; color: #003388; text-decoration: none;} -a:hover {color: #0066CC;} -a img {border:none;} +a img { + border: none; +} input { font-size: 12px; @@ -29,13 +38,16 @@ input { margin-left: 50px; margin-right: 50px; } + .menuheader { float: right; - + } + .menuheader table { - + } + /** .menubar { padding: 0px; @@ -79,7 +91,6 @@ width: 100%; text-align: center; } - .footer { /*padding-left: 50px;*/ text-align: center; @@ -101,7 +112,7 @@ vertical-align: top; .submitLink { color: #4444cc; - font-family: Arial,Helvetica,sans-serif; + font-family: Arial, Helvetica, sans-serif; background-color: transparent; text-decoration: underline; border: none; @@ -120,8 +131,6 @@ vertical-align: top; font-size: small; } - - #statustable thead { background-color: #e8e8e8; } @@ -130,7 +139,7 @@ vertical-align: top; border-bottom: 1px dotted #555; } -#statustable tr td{ +#statustable tr td { padding-left: 5px; } @@ -163,10 +172,10 @@ vertical-align: top; } #settingsTable { - font-size: small; - border: 1px solid #000000; - margin-left: 5%; - margin-right: 5%; + font-size: small; + border: 1px solid #000000; + margin-left: 5%; + margin-right: 5%; } .settingsRow { @@ -178,9 +187,9 @@ vertical-align: top; } .settingsName { - width: 200px; - float: left; - margin-left: 2px; + width: 200px; + float: left; + margin-left: 2px; } .settingsVal { @@ -193,7 +202,7 @@ vertical-align: top; float: left; } -.dropDownContainer{ +.dropDownContainer { margin: 0; padding: 0; width: 100%; @@ -201,15 +210,66 @@ vertical-align: top; } .collName { -margin: 1px; -color: #000; -padding: 3px 10px; -cursor: pointer; -position: relative; -background-color:#eee; + margin: 1px; + color: #000; + padding: 3px 10px; + cursor: pointer; + position: relative; + background-color: #eee; } .ingestContent { -padding: 5px 10px; -background-color:#fff; -} \ No newline at end of file + padding: 5px 10px; + background-color: #fff; +} + +#searchtable { + margin-top: 10px; + margin-bottom: -10px; + margin-left: auto; + margin-right: auto; + width: 650px; +} + +.input { + padding: 2px; + display: flex; + width: 650px; +} + +.input-group-addon { + border: 1px solid #ccc; + font-size: 12px; + background-color: #e8e8e8; + text-align: center; + width: 75px; + height: 20px; + line-height: 20px; + padding: 3px 10px; +} + +.form-input { + border: 1px solid #ccc; + width: 100%; + /* height: 20px; */ + padding: 3px 8px; + margin-left: -1px; + margin-right: 5px; +} + +.form-select { + border: 1px solid #ccc; + width: 100%; + padding: 3px 8px; + margin-left: -3px; + margin-right: 5px; +} + +.btn { + padding: 2px; + width: 75px; + height: 25px; + border: 1px solid #e8e8e8; + margin-left: 2px; + margin-top: 2px; +} -- GitLab From 16162d1b2fd2b16444239ebb91d06cbe3105b55f Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Thu, 22 Sep 2016 17:09:29 -0400 Subject: [PATCH 3/7] Add update query to allow modification of MonitoredItems in the db --- .../umiacs/ace/monitor/core/MonitoredItem.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/core/MonitoredItem.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/core/MonitoredItem.java index a93edf3..7d4d1a8 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/core/MonitoredItem.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/core/MonitoredItem.java @@ -101,35 +101,48 @@ import javax.persistence.TemporalType; @NamedQuery(name = "MonitoredItem.listRemoteErrors", query = "SELECT m FROM MonitoredItem m WHERE (m.state = 'P' OR m.state = 'D') AND m.directory = false AND m.parentCollection = :coll"), @NamedQuery(name = "MonitoredItem.listLocalErrors", query = - "SELECT m FROM MonitoredItem m WHERE (m.state = 'C' OR m.state = 'M' OR m.state = 'T' OR m.state = 'I') AND m.directory = false AND m.parentCollection = :coll") + "SELECT m FROM MonitoredItem m WHERE (m.state = 'C' OR m.state = 'M' OR m.state = 'T' OR m.state = 'I') AND m.directory = false AND m.parentCollection = :coll"), + @NamedQuery(name = "MonitoredItem.updateMissing", query = + "UPDATE MonitoredItem SET state = 'M', stateChange = :date, lastVisited = :date WHERE parentCollection = :coll AND lastVisited < :date") }) public class MonitoredItem implements Serializable, Comparable { private static final long serialVersionUID = 1L; + @Id @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; + @Column(nullable = false, columnDefinition = "VARCHAR(255) COLLATE utf8") private String path; // path relative to base directory + @Column(columnDefinition = "VARCHAR(255) COLLATE utf8") private String parentPath; + @Column(nullable = false) private boolean directory; // true if directory + @Temporal(TemporalType.TIMESTAMP) private Date lastSeen; + @Temporal(TemporalType.TIMESTAMP) private Date stateChange; + @Temporal(TemporalType.TIMESTAMP) private Date lastVisited; + @ManyToOne @JoinColumn(nullable = false) private Collection parentCollection; - private String fileDigest; + @Column(nullable = false) private char state; + @ManyToOne(cascade = CascadeType.ALL) private Token token; + private long size; + private String fileDigest; public void setId( Long id ) { this.id = id; -- GitLab From 2c421359906082c54c31bf9624578467d2c4c025 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Thu, 22 Sep 2016 17:11:14 -0400 Subject: [PATCH 4/7] Update monitored items and add log events through the db --- .../umiacs/ace/monitor/audit/AuditThread.java | 86 +++++++++++++------ 1 file changed, 60 insertions(+), 26 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java index 904f4f8..a21fa5d 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java @@ -32,48 +32,54 @@ package edu.umiacs.ace.monitor.audit; import edu.umiacs.ace.driver.AuditIterable; import edu.umiacs.ace.driver.DriverStateBean; -import edu.umiacs.ace.driver.filter.PathFilter; import edu.umiacs.ace.driver.FileBean; import edu.umiacs.ace.driver.StorageDriver; +import edu.umiacs.ace.driver.filter.PathFilter; +import edu.umiacs.ace.driver.filter.SimpleFilter; import edu.umiacs.ace.ims.api.IMSException; import edu.umiacs.ace.ims.api.IMSService; import edu.umiacs.ace.ims.api.TokenRequestBatch; import edu.umiacs.ace.ims.api.TokenValidator; import edu.umiacs.ace.ims.ws.TokenRequest; import edu.umiacs.ace.monitor.access.CollectionCountContext; -import edu.umiacs.ace.monitor.register.IngestThreadPool; -import edu.umiacs.ace.util.PersistUtil; -import edu.umiacs.ace.driver.filter.SimpleFilter; import edu.umiacs.ace.monitor.compare.CollectionCompare2; import edu.umiacs.ace.monitor.compare.CompareResults; -import edu.umiacs.ace.remote.JsonGateway; +import edu.umiacs.ace.monitor.core.Collection; +import edu.umiacs.ace.monitor.core.ConfigConstants; import edu.umiacs.ace.monitor.core.MonitoredItem; import edu.umiacs.ace.monitor.core.MonitoredItemManager; +import edu.umiacs.ace.monitor.log.LogEnum; +import edu.umiacs.ace.monitor.log.LogEvent; import edu.umiacs.ace.monitor.log.LogEventManager; +import edu.umiacs.ace.monitor.peers.PeerCollection; import edu.umiacs.ace.monitor.reporting.ReportSummary; import edu.umiacs.ace.monitor.reporting.SchedulerContextListener; import edu.umiacs.ace.monitor.reporting.SummaryGenerator; -import edu.umiacs.ace.monitor.core.Collection; -import edu.umiacs.ace.monitor.core.ConfigConstants; import edu.umiacs.ace.monitor.settings.SettingsUtil; -import edu.umiacs.ace.monitor.log.LogEnum; -import edu.umiacs.ace.monitor.log.LogEvent; -import edu.umiacs.ace.monitor.peers.PeerCollection; +import edu.umiacs.ace.remote.JsonGateway; import edu.umiacs.ace.token.AceToken; +import edu.umiacs.ace.util.PersistUtil; import edu.umiacs.ace.util.TokenUtil; import edu.umiacs.util.Strings; +import org.apache.log4j.Logger; +import org.apache.log4j.NDC; + +import javax.mail.MessagingException; +import javax.persistence.EntityManager; +import javax.persistence.EntityTransaction; +import javax.persistence.Query; +import javax.sql.DataSource; import java.io.InputStream; import java.security.MessageDigest; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Timestamp; import java.util.Arrays; import java.util.Date; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import javax.mail.MessagingException; -import javax.persistence.EntityManager; -import javax.persistence.EntityTransaction; -import org.apache.log4j.Logger; -import org.apache.log4j.NDC; /** * @@ -222,12 +228,6 @@ public final class AuditThread extends Thread implements CancelCallback { session = System.currentTimeMillis(); logManager = new LogEventManager(session, coll); logAuditStart(); - /* - while (IngestThreadPool.isIngesting(coll)) { - LOG.debug("Waiting for ingest to finish"); - Thread.sleep(500); - } - */ callback = new FileAuditCallback(coll, session, this); boolean auditTokens = SettingsUtil.getBoolean(coll, @@ -254,7 +254,7 @@ public final class AuditThread extends Thread implements CancelCallback { // Let outstanding tokens finish, TODO, de-hackify this. sleep(2000); } catch (Throwable e) { - LOG.fatal("UNcaught exception in performAudit()", e); + LOG.fatal("Uncaught exception in performAudit()", e); if (abortException != null) { abortException = e; } @@ -595,7 +595,6 @@ public final class AuditThread extends Thread implements CancelCallback { } TokenRequest request = new TokenRequest(); - //request.setName(item.getPath()); request.setName(item.getId().toString()); request.setHashValue(currentFile.getHash()); if (!Strings.isEmpty(item.getPath()) && @@ -616,7 +615,6 @@ public final class AuditThread extends Thread implements CancelCallback { LogEvent event; // If we have a registered file, set the digested value - LOG.trace(null == item.getFileDigest()); if (null == item.getFileDigest()) { LOG.trace("Setting digest for registered file " + item.getPath()); item.setFileDigest(currentFile.getHash()); @@ -760,11 +758,43 @@ public final class AuditThread extends Thread implements CancelCallback { return; } + DataSource dataSource = PersistUtil.getDataSource(); EntityManager em = PersistUtil.getEntityManager(); MonitoredItemManager mim = new MonitoredItemManager(em); EntityTransaction trans = em.getTransaction(); trans.begin(); + try { + Query query = em.createNamedQuery("MonitoredItem.updateMissing") + .setParameter("coll", coll) + .setParameter("date", d); + + int i = query.executeUpdate(); + if (i > 0) { + LOG.info("Set " + i + " new missing items"); + + Connection connection = dataSource.getConnection(); + PreparedStatement ps = connection.prepareStatement("INSERT INTO logevent(session, path, date, logtype, collection_id) " + + "SELECT ?, path, NOW(), ?, parentcollection_id FROM monitored_item m WHERE m.parentcollection_id = ? AND m.state = ? AND m.statechange = ?"); + ps.setLong(1, session); + ps.setInt(2, LogEnum.FILE_MISSING.getType()); + ps.setLong(3, coll.getId()); + ps.setString(4, String.valueOf('M')); + ps.setTimestamp(5, new Timestamp(d.getTime())); + ps.executeUpdate(); + ps.close(); + connection.close(); + } + trans.commit(); + } catch (SQLException e) { + trans.rollback(); + LOG.error(e); + } finally { + em.close(); + } + + /* + int idx = 0; for (MonitoredItem mi : mim.listItemsBefore(coll, d)) { LOG.trace("Updating missing item: " + mi.getPath()); LOG.trace("Item information: LS= " + mi.getLastSeen() @@ -781,9 +811,13 @@ public final class AuditThread extends Thread implements CancelCallback { } mi.setLastVisited(new Date()); em.merge(mi); + idx++; + if (idx % 30 == 0) { + em.flush(); + em.clear(); + } } - trans.commit(); - em.close(); + */ } private void compareToPeers() { -- GitLab From b3330dc1d555379fd33567ecbce1e06f349af4d9 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Thu, 6 Oct 2016 17:23:42 -0400 Subject: [PATCH 5/7] Add some comments, better logging messages --- .../main/java/edu/umiacs/ace/monitor/audit/AuditThread.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java index a21fa5d..a5017f6 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java @@ -760,11 +760,11 @@ public final class AuditThread extends Thread implements CancelCallback { DataSource dataSource = PersistUtil.getDataSource(); EntityManager em = PersistUtil.getEntityManager(); - MonitoredItemManager mim = new MonitoredItemManager(em); EntityTransaction trans = em.getTransaction(); trans.begin(); try { + // Update the monitored item table Query query = em.createNamedQuery("MonitoredItem.updateMissing") .setParameter("coll", coll) .setParameter("date", d); @@ -773,6 +773,7 @@ public final class AuditThread extends Thread implements CancelCallback { if (i > 0) { LOG.info("Set " + i + " new missing items"); + // Add log entries for the new missing items Connection connection = dataSource.getConnection(); PreparedStatement ps = connection.prepareStatement("INSERT INTO logevent(session, path, date, logtype, collection_id) " + "SELECT ?, path, NOW(), ?, parentcollection_id FROM monitored_item m WHERE m.parentcollection_id = ? AND m.state = ? AND m.statechange = ?"); @@ -788,7 +789,7 @@ public final class AuditThread extends Thread implements CancelCallback { trans.commit(); } catch (SQLException e) { trans.rollback(); - LOG.error(e); + LOG.error("SQL error, rolling back missing items and log entries", e); } finally { em.close(); } -- GitLab From ffbdc26f0a4f78ef8bd6a7bc45eae2011774a299 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Thu, 6 Oct 2016 17:24:53 -0400 Subject: [PATCH 6/7] Only set missing if it is not already missing --- .../main/java/edu/umiacs/ace/monitor/core/MonitoredItem.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/core/MonitoredItem.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/core/MonitoredItem.java index 7d4d1a8..6c0860b 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/core/MonitoredItem.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/core/MonitoredItem.java @@ -103,7 +103,7 @@ import javax.persistence.TemporalType; @NamedQuery(name = "MonitoredItem.listLocalErrors", query = "SELECT m FROM MonitoredItem m WHERE (m.state = 'C' OR m.state = 'M' OR m.state = 'T' OR m.state = 'I') AND m.directory = false AND m.parentCollection = :coll"), @NamedQuery(name = "MonitoredItem.updateMissing", query = - "UPDATE MonitoredItem SET state = 'M', stateChange = :date, lastVisited = :date WHERE parentCollection = :coll AND lastVisited < :date") + "UPDATE MonitoredItem SET state = 'M', stateChange = :date, lastVisited = :date WHERE parentCollection = :coll AND lastVisited < :date AND state != 'M'") }) public class MonitoredItem implements Serializable, Comparable { -- GitLab From 9f6bb953a9aaa241c55037fe87cdf3c01719cb37 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Fri, 7 Oct 2016 13:42:45 -0400 Subject: [PATCH 7/7] Final cleanup before merge --- .../umiacs/ace/monitor/audit/AuditThread.java | 43 ++----------------- 1 file changed, 3 insertions(+), 40 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java index a5017f6..d7e9436 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java @@ -197,16 +197,6 @@ public final class AuditThread extends Thread implements CancelCallback { iterableItems.cancel(); } - /* - if (batch != null) { - batch.close(); - } - - if (validator != null) { - validator.close(); - } - */ - if (AuditThreadFactory.isRunning(coll) || AuditThreadFactory.isQueued(coll)) { AuditThreadFactory.finished(coll); } @@ -711,8 +701,8 @@ public final class AuditThread extends Thread implements CancelCallback { if (parentName != null) { parentName = Strings.cleanStringForXml(parentName, '_'); for (int i = 1; i < currentFile.getPathList().length; i++) { - String parent = (currentFile.getPathList().length > i + 1 - ? currentFile.getPathList()[i + 1] : null); + String parent = currentFile.getPathList().length > i + 1 ? + currentFile.getPathList()[i + 1] : null; parent = Strings.cleanStringForXml(parent, '_'); mim.createDirectory(currentFile.getPathList()[i], parent, coll); } @@ -723,8 +713,7 @@ public final class AuditThread extends Thread implements CancelCallback { private String[] createMailList() { String addrs = SettingsUtil.getString(coll, ConfigConstants.ATTR_EMAIL_RECIPIENTS); - String[] maillist = (addrs == null ? null : addrs.split("\\s*,\\s*")); - return maillist; + return addrs == null ? null : addrs.split("\\s*,\\s*"); } private void setCollectionState() { @@ -793,32 +782,6 @@ public final class AuditThread extends Thread implements CancelCallback { } finally { em.close(); } - - /* - int idx = 0; - for (MonitoredItem mi : mim.listItemsBefore(coll, d)) { - LOG.trace("Updating missing item: " + mi.getPath()); - LOG.trace("Item information: LS= " + mi.getLastSeen() - + ", LV = " + mi.getLastVisited() - + ", SC = " + mi.getStateChange() - + "Audit started on: " + d); - if (mi.getState() != 'M' - && (mi.getStateChange() == null - || d.after(mi.getStateChange()))) { - mi.setState('M'); - mi.setStateChange(new Date()); - em.persist(logManager.createItemEvent(LogEnum.FILE_MISSING, - mi.getPath())); - } - mi.setLastVisited(new Date()); - em.merge(mi); - idx++; - if (idx % 30 == 0) { - em.flush(); - em.clear(); - } - } - */ } private void compareToPeers() { -- GitLab
- <%-- |<    - << --%> |<    << - <%-- - >>    - >| --%> >>    >|