diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java index 904f4f8dfbb3f073fe16c9d5db48d2e12a10f512..d7e94367d62a448b112bf09539f93a529cf34e33 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java @@ -32,48 +32,54 @@ package edu.umiacs.ace.monitor.audit; import edu.umiacs.ace.driver.AuditIterable; import edu.umiacs.ace.driver.DriverStateBean; -import edu.umiacs.ace.driver.filter.PathFilter; import edu.umiacs.ace.driver.FileBean; import edu.umiacs.ace.driver.StorageDriver; +import edu.umiacs.ace.driver.filter.PathFilter; +import edu.umiacs.ace.driver.filter.SimpleFilter; import edu.umiacs.ace.ims.api.IMSException; import edu.umiacs.ace.ims.api.IMSService; import edu.umiacs.ace.ims.api.TokenRequestBatch; import edu.umiacs.ace.ims.api.TokenValidator; import edu.umiacs.ace.ims.ws.TokenRequest; import edu.umiacs.ace.monitor.access.CollectionCountContext; -import edu.umiacs.ace.monitor.register.IngestThreadPool; -import edu.umiacs.ace.util.PersistUtil; -import edu.umiacs.ace.driver.filter.SimpleFilter; import edu.umiacs.ace.monitor.compare.CollectionCompare2; import edu.umiacs.ace.monitor.compare.CompareResults; -import edu.umiacs.ace.remote.JsonGateway; +import edu.umiacs.ace.monitor.core.Collection; +import edu.umiacs.ace.monitor.core.ConfigConstants; import edu.umiacs.ace.monitor.core.MonitoredItem; import edu.umiacs.ace.monitor.core.MonitoredItemManager; +import edu.umiacs.ace.monitor.log.LogEnum; +import edu.umiacs.ace.monitor.log.LogEvent; import edu.umiacs.ace.monitor.log.LogEventManager; +import edu.umiacs.ace.monitor.peers.PeerCollection; import edu.umiacs.ace.monitor.reporting.ReportSummary; import edu.umiacs.ace.monitor.reporting.SchedulerContextListener; import edu.umiacs.ace.monitor.reporting.SummaryGenerator; -import edu.umiacs.ace.monitor.core.Collection; -import edu.umiacs.ace.monitor.core.ConfigConstants; import edu.umiacs.ace.monitor.settings.SettingsUtil; -import edu.umiacs.ace.monitor.log.LogEnum; -import edu.umiacs.ace.monitor.log.LogEvent; -import edu.umiacs.ace.monitor.peers.PeerCollection; +import edu.umiacs.ace.remote.JsonGateway; import edu.umiacs.ace.token.AceToken; +import edu.umiacs.ace.util.PersistUtil; import edu.umiacs.ace.util.TokenUtil; import edu.umiacs.util.Strings; +import org.apache.log4j.Logger; +import org.apache.log4j.NDC; + +import javax.mail.MessagingException; +import javax.persistence.EntityManager; +import javax.persistence.EntityTransaction; +import javax.persistence.Query; +import javax.sql.DataSource; import java.io.InputStream; import java.security.MessageDigest; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Timestamp; import java.util.Arrays; import java.util.Date; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import javax.mail.MessagingException; -import javax.persistence.EntityManager; -import javax.persistence.EntityTransaction; -import org.apache.log4j.Logger; -import org.apache.log4j.NDC; /** * @@ -191,16 +197,6 @@ public final class AuditThread extends Thread implements CancelCallback { iterableItems.cancel(); } - /* - if (batch != null) { - batch.close(); - } - - if (validator != null) { - validator.close(); - } - */ - if (AuditThreadFactory.isRunning(coll) || AuditThreadFactory.isQueued(coll)) { AuditThreadFactory.finished(coll); } @@ -222,12 +218,6 @@ public final class AuditThread extends Thread implements CancelCallback { session = System.currentTimeMillis(); logManager = new LogEventManager(session, coll); logAuditStart(); - /* - while (IngestThreadPool.isIngesting(coll)) { - LOG.debug("Waiting for ingest to finish"); - Thread.sleep(500); - } - */ callback = new FileAuditCallback(coll, session, this); boolean auditTokens = SettingsUtil.getBoolean(coll, @@ -254,7 +244,7 @@ public final class AuditThread extends Thread implements CancelCallback { // Let outstanding tokens finish, TODO, de-hackify this. sleep(2000); } catch (Throwable e) { - LOG.fatal("UNcaught exception in performAudit()", e); + LOG.fatal("Uncaught exception in performAudit()", e); if (abortException != null) { abortException = e; } @@ -595,7 +585,6 @@ public final class AuditThread extends Thread implements CancelCallback { } TokenRequest request = new TokenRequest(); - //request.setName(item.getPath()); request.setName(item.getId().toString()); request.setHashValue(currentFile.getHash()); if (!Strings.isEmpty(item.getPath()) && @@ -616,7 +605,6 @@ public final class AuditThread extends Thread implements CancelCallback { LogEvent event; // If we have a registered file, set the digested value - LOG.trace(null == item.getFileDigest()); if (null == item.getFileDigest()) { LOG.trace("Setting digest for registered file " + item.getPath()); item.setFileDigest(currentFile.getHash()); @@ -713,8 +701,8 @@ public final class AuditThread extends Thread implements CancelCallback { if (parentName != null) { parentName = Strings.cleanStringForXml(parentName, '_'); for (int i = 1; i < currentFile.getPathList().length; i++) { - String parent = (currentFile.getPathList().length > i + 1 - ? currentFile.getPathList()[i + 1] : null); + String parent = currentFile.getPathList().length > i + 1 ? + currentFile.getPathList()[i + 1] : null; parent = Strings.cleanStringForXml(parent, '_'); mim.createDirectory(currentFile.getPathList()[i], parent, coll); } @@ -725,8 +713,7 @@ public final class AuditThread extends Thread implements CancelCallback { private String[] createMailList() { String addrs = SettingsUtil.getString(coll, ConfigConstants.ATTR_EMAIL_RECIPIENTS); - String[] maillist = (addrs == null ? null : addrs.split("\\s*,\\s*")); - return maillist; + return addrs == null ? null : addrs.split("\\s*,\\s*"); } private void setCollectionState() { @@ -760,30 +747,41 @@ public final class AuditThread extends Thread implements CancelCallback { return; } + DataSource dataSource = PersistUtil.getDataSource(); EntityManager em = PersistUtil.getEntityManager(); - MonitoredItemManager mim = new MonitoredItemManager(em); EntityTransaction trans = em.getTransaction(); trans.begin(); - for (MonitoredItem mi : mim.listItemsBefore(coll, d)) { - LOG.trace("Updating missing item: " + mi.getPath()); - LOG.trace("Item information: LS= " + mi.getLastSeen() - + ", LV = " + mi.getLastVisited() - + ", SC = " + mi.getStateChange() - + "Audit started on: " + d); - if (mi.getState() != 'M' - && (mi.getStateChange() == null - || d.after(mi.getStateChange()))) { - mi.setState('M'); - mi.setStateChange(new Date()); - em.persist(logManager.createItemEvent(LogEnum.FILE_MISSING, - mi.getPath())); + try { + // Update the monitored item table + Query query = em.createNamedQuery("MonitoredItem.updateMissing") + .setParameter("coll", coll) + .setParameter("date", d); + + int i = query.executeUpdate(); + if (i > 0) { + LOG.info("Set " + i + " new missing items"); + + // Add log entries for the new missing items + Connection connection = dataSource.getConnection(); + PreparedStatement ps = connection.prepareStatement("INSERT INTO logevent(session, path, date, logtype, collection_id) " + + "SELECT ?, path, NOW(), ?, parentcollection_id FROM monitored_item m WHERE m.parentcollection_id = ? AND m.state = ? AND m.statechange = ?"); + ps.setLong(1, session); + ps.setInt(2, LogEnum.FILE_MISSING.getType()); + ps.setLong(3, coll.getId()); + ps.setString(4, String.valueOf('M')); + ps.setTimestamp(5, new Timestamp(d.getTime())); + ps.executeUpdate(); + ps.close(); + connection.close(); } - mi.setLastVisited(new Date()); - em.merge(mi); + trans.commit(); + } catch (SQLException e) { + trans.rollback(); + LOG.error("SQL error, rolling back missing items and log entries", e); + } finally { + em.close(); } - trans.commit(); - em.close(); } private void compareToPeers() { diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/core/MonitoredItem.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/core/MonitoredItem.java index a93edf36ced494f0aef50000fdddb82f17a95d89..6c0860bd3694269b8669c7b6b085adc376158a1b 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/core/MonitoredItem.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/core/MonitoredItem.java @@ -101,35 +101,48 @@ import javax.persistence.TemporalType; @NamedQuery(name = "MonitoredItem.listRemoteErrors", query = "SELECT m FROM MonitoredItem m WHERE (m.state = 'P' OR m.state = 'D') AND m.directory = false AND m.parentCollection = :coll"), @NamedQuery(name = "MonitoredItem.listLocalErrors", query = - "SELECT m FROM MonitoredItem m WHERE (m.state = 'C' OR m.state = 'M' OR m.state = 'T' OR m.state = 'I') AND m.directory = false AND m.parentCollection = :coll") + "SELECT m FROM MonitoredItem m WHERE (m.state = 'C' OR m.state = 'M' OR m.state = 'T' OR m.state = 'I') AND m.directory = false AND m.parentCollection = :coll"), + @NamedQuery(name = "MonitoredItem.updateMissing", query = + "UPDATE MonitoredItem SET state = 'M', stateChange = :date, lastVisited = :date WHERE parentCollection = :coll AND lastVisited < :date AND state != 'M'") }) public class MonitoredItem implements Serializable, Comparable { private static final long serialVersionUID = 1L; + @Id @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; + @Column(nullable = false, columnDefinition = "VARCHAR(255) COLLATE utf8") private String path; // path relative to base directory + @Column(columnDefinition = "VARCHAR(255) COLLATE utf8") private String parentPath; + @Column(nullable = false) private boolean directory; // true if directory + @Temporal(TemporalType.TIMESTAMP) private Date lastSeen; + @Temporal(TemporalType.TIMESTAMP) private Date stateChange; + @Temporal(TemporalType.TIMESTAMP) private Date lastVisited; + @ManyToOne @JoinColumn(nullable = false) private Collection parentCollection; - private String fileDigest; + @Column(nullable = false) private char state; + @ManyToOne(cascade = CascadeType.ALL) private Token token; + private long size; + private String fileDigest; public void setId( Long id ) { this.id = id; diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/register/IngestThread.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/register/IngestThread.java index 72b57e60f080f1d7f1c619285193ba335253df6a..83e54bf3c64d8dbca21eac1542ec7ba76425dcb4 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/register/IngestThread.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/register/IngestThread.java @@ -62,7 +62,7 @@ import java.util.concurrent.RecursiveAction; public class IngestThread extends RecursiveAction { private static final Logger LOG = Logger.getLogger(IngestThread.class); - // These only gets read from, never written to + // These only get read from, never written to private Map tokens; private Collection coll; private List identifiers; @@ -293,4 +293,4 @@ public class IngestThread extends RecursiveAction { } -} +} \ No newline at end of file diff --git a/ace-am/src/main/webapp/status.jsp b/ace-am/src/main/webapp/status.jsp index 330671338bc3ca4a5f4ebc33afb5bd90e409c2dd..4d16aa2057998885d8451e45c53b77ba9a8af9bd 100644 --- a/ace-am/src/main/webapp/status.jsp +++ b/ace-am/src/main/webapp/status.jsp @@ -133,56 +133,7 @@ background-color: #e8e8e8; } - #searchtable { - margin-top: 10px; - margin-bottom: -10px; - margin-left: auto; - margin-right: auto; - width: 650px; - } - - .input { - padding: 2px; - display: flex; - width: 650px; - } - - .input-group-addon { - border: 1px solid #ccc; - font-size: 12px; - background-color: #e8e8e8; - text-align: center; - width: 75px; - height: 20px; - line-height: 20px; - padding: 3px 10px; - } - - .form-input { - border: 1px solid #ccc; - width: 100%; - height: 20px; - padding: 3px 8px; - margin-left: -1px; - margin-right: 5px; - } - .form-select { - border: 1px solid #ccc; - width: 100%; - padding: 3px 8px; - margin-left: -3px; - margin-right: 5px; - } - - .btn { - padding: 2px; - width: 75px; - height: 25px; - border: 1px solid #e8e8e8; - margin-left: 2px; - margin-top: 2px; - } @@ -344,8 +295,6 @@ @@ -356,9 +305,6 @@ 1000 diff --git a/ace-am/src/main/webapp/style.css b/ace-am/src/main/webapp/style.css index 36fe6ca59abafe93820c4b8f788a63f987d586ff..3f4a43ac78a830f962da28a0d84bce681ee826a3 100644 --- a/ace-am/src/main/webapp/style.css +++ b/ace-am/src/main/webapp/style.css @@ -1,5 +1,5 @@ body { - font-family: Arial,Helvetica,sans-serif; + font-family: Arial, Helvetica, sans-serif; font-size: 10px; width: 750px; border: 1px solid #000000; @@ -13,10 +13,19 @@ h1 { margin-left: 10px; } +a:link, a:visited { + font-size: 12px; + color: #003388; + text-decoration: none; +} + +a:hover { + color: #0066CC; +} -a:link, a:visited {font-size: 12px; color: #003388; text-decoration: none;} -a:hover {color: #0066CC;} -a img {border:none;} +a img { + border: none; +} input { font-size: 12px; @@ -29,13 +38,16 @@ input { margin-left: 50px; margin-right: 50px; } + .menuheader { float: right; - + } + .menuheader table { - + } + /** .menubar { padding: 0px; @@ -79,7 +91,6 @@ width: 100%; text-align: center; } - .footer { /*padding-left: 50px;*/ text-align: center; @@ -101,7 +112,7 @@ vertical-align: top; .submitLink { color: #4444cc; - font-family: Arial,Helvetica,sans-serif; + font-family: Arial, Helvetica, sans-serif; background-color: transparent; text-decoration: underline; border: none; @@ -120,8 +131,6 @@ vertical-align: top; font-size: small; } - - #statustable thead { background-color: #e8e8e8; } @@ -130,7 +139,7 @@ vertical-align: top; border-bottom: 1px dotted #555; } -#statustable tr td{ +#statustable tr td { padding-left: 5px; } @@ -163,10 +172,10 @@ vertical-align: top; } #settingsTable { - font-size: small; - border: 1px solid #000000; - margin-left: 5%; - margin-right: 5%; + font-size: small; + border: 1px solid #000000; + margin-left: 5%; + margin-right: 5%; } .settingsRow { @@ -178,9 +187,9 @@ vertical-align: top; } .settingsName { - width: 200px; - float: left; - margin-left: 2px; + width: 200px; + float: left; + margin-left: 2px; } .settingsVal { @@ -193,7 +202,7 @@ vertical-align: top; float: left; } -.dropDownContainer{ +.dropDownContainer { margin: 0; padding: 0; width: 100%; @@ -201,15 +210,66 @@ vertical-align: top; } .collName { -margin: 1px; -color: #000; -padding: 3px 10px; -cursor: pointer; -position: relative; -background-color:#eee; + margin: 1px; + color: #000; + padding: 3px 10px; + cursor: pointer; + position: relative; + background-color: #eee; } .ingestContent { -padding: 5px 10px; -background-color:#fff; -} \ No newline at end of file + padding: 5px 10px; + background-color: #fff; +} + +#searchtable { + margin-top: 10px; + margin-bottom: -10px; + margin-left: auto; + margin-right: auto; + width: 650px; +} + +.input { + padding: 2px; + display: flex; + width: 650px; +} + +.input-group-addon { + border: 1px solid #ccc; + font-size: 12px; + background-color: #e8e8e8; + text-align: center; + width: 75px; + height: 20px; + line-height: 20px; + padding: 3px 10px; +} + +.form-input { + border: 1px solid #ccc; + width: 100%; + /* height: 20px; */ + padding: 3px 8px; + margin-left: -1px; + margin-right: 5px; +} + +.form-select { + border: 1px solid #ccc; + width: 100%; + padding: 3px 8px; + margin-left: -3px; + margin-right: 5px; +} + +.btn { + padding: 2px; + width: 75px; + height: 25px; + border: 1px solid #e8e8e8; + margin-left: 2px; + margin-top: 2px; +}
- <%-- |<    - << --%> |<    << - <%-- - >>    - >| --%> >>    >|