diff --git a/ace-am/pom.xml b/ace-am/pom.xml index f271039b43b56aa49c8fb416c5b7eccb2e11e6ed..f7c76f6dc7e2ab1991321f34e485ffbf9767899f 100644 --- a/ace-am/pom.xml +++ b/ace-am/pom.xml @@ -5,7 +5,6 @@ ace edu.umiacs.ace 1.14-SNAPSHOT - ../pom.xml ace-am ace-am @@ -105,7 +104,7 @@ provided - jstl + javax.servlet jstl 1.1.2 @@ -196,34 +195,8 @@ irods-api 1.6 - - - + + org.glassfish.jersey.containers jersey-container-servlet @@ -234,12 +207,6 @@ jersey-media-json-jackson 2.24 - - edu.umiacs.ace - ace-ims-ws - 1.14-SNAPSHOT - jar - org.apache.httpcomponents httpclient diff --git a/ace-am/src/main/java/edu/umiacs/ace/driver/localfile/LocalFileAccess.java b/ace-am/src/main/java/edu/umiacs/ace/driver/localfile/LocalFileAccess.java index 0c7cb5348b0b9e6aee4dce89c7733db8e5da1a85..cf325b39f86966a14262c9fc66ae825c00f4d3d6 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/driver/localfile/LocalFileAccess.java +++ b/ace-am/src/main/java/edu/umiacs/ace/driver/localfile/LocalFileAccess.java @@ -34,22 +34,23 @@ import edu.umiacs.ace.driver.AuditIterable; import edu.umiacs.ace.driver.DriverStateBean; import edu.umiacs.ace.driver.DriverStateBean.State; import edu.umiacs.ace.driver.FileBean; -import edu.umiacs.ace.driver.filter.PathFilter; -import edu.umiacs.ace.driver.StorageDriver; -import edu.umiacs.ace.monitor.core.MonitoredItem; -import edu.umiacs.ace.monitor.core.Collection; import edu.umiacs.ace.driver.QueryThrottle; import edu.umiacs.ace.driver.StateBeanDigestListener; +import edu.umiacs.ace.driver.StorageDriver; +import edu.umiacs.ace.driver.filter.PathFilter; +import edu.umiacs.ace.monitor.core.Collection; +import edu.umiacs.ace.monitor.core.MonitoredItem; import edu.umiacs.ace.util.HashValue; import edu.umiacs.ace.util.ThreadedDigestStream; import edu.umiacs.ace.util.ThrottledInputStream; -import edu.umiacs.io.IO; import edu.umiacs.util.Strings; +import org.apache.log4j.Logger; + +import javax.persistence.EntityManager; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; -import java.security.DigestInputStream; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; @@ -58,12 +59,10 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Queue; -import javax.persistence.EntityManager; -import org.apache.log4j.Logger; /** * Storage driver for accessing files stored on a local file system (ie, java.io.File) - * + * * @author toaster */ public class LocalFileAccess extends StorageDriver { @@ -138,18 +137,19 @@ public class LocalFileAccess extends StorageDriver { class MyIterator implements Iterator { private FileBean next; - private Queue dirsToProcess = new LinkedList(); - private Queue filesToProcess = new LinkedList(); + private Queue dirsToProcess = new LinkedList<>(); + private Queue filesToProcess = new LinkedList<>(); private MessageDigest digest; -// private byte[] buffer = new byte[BLOCK_SIZE]; private File rootFile; private PathFilter filter; private DriverStateBean statebean; private ThreadedDigestStream reader; private boolean cancel = false; - public MyIterator(MonitoredItem[] startPath, PathFilter filter, - String digestAlgorithm, DriverStateBean statebean) { + public MyIterator(MonitoredItem[] startPath, + PathFilter filter, + String digestAlgorithm, + DriverStateBean statebean) { this.statebean = statebean; this.filter = filter; try { @@ -163,8 +163,7 @@ public class LocalFileAccess extends StorageDriver { if (startPath != null) { for (MonitoredItem mi : startPath) { File startFile; - startFile = new File( - getCollection().getDirectory() + mi.getPath()); + startFile = new File(getCollection().getDirectory() + mi.getPath()); if (startFile.isDirectory()) { dirsToProcess.add(startFile); @@ -224,13 +223,13 @@ public class LocalFileAccess extends StorageDriver { if (fileList == null) { LOG.info("Could not read directory, skipping: " + directory); } else { - for (File f : directory.listFiles()) { + for (File f : fileList) { LOG.trace("Found item " + f); - if ( f.isDirectory() && + if ( f.isDirectory() && filter.process(extractPathList(f), true)) { LOG.trace("Adding matching directory: " + f); dirsToProcess.add(f); - } else if ( f.isFile() && + } else if ( f.isFile() && filter.process(extractPathList(f), false)) { LOG.trace("Adding matching file: " + f); filesToProcess.add(f); @@ -254,11 +253,9 @@ public class LocalFileAccess extends StorageDriver { int substrLength = rootFile.getPath().length(); // build directory path - List dirPathList = new ArrayList(); + List dirPathList = new ArrayList<>(); File currFile = file; while (!currFile.equals(rootFile)) { -// LOG.trace("Adding dir to path: " + currFile.getPath().substring( -// substrLength)); String pathToAdd = currFile.getPath().substring(substrLength); pathToAdd = pathToAdd.replace(File.separatorChar, '/'); dirPathList.add(pathToAdd); @@ -269,11 +266,7 @@ public class LocalFileAccess extends StorageDriver { @SuppressWarnings("empty-statement") private FileBean processFile(File file) { - - - DigestInputStream dis = null; FileBean fb = new FileBean(); - fb.setPathList(extractPathList(file)); LOG.trace("Processing file: " + file); @@ -307,7 +300,6 @@ public class LocalFileAccess extends StorageDriver { fb.setError(true); fb.setErrorMessage(Strings.exceptionAsString(ie)); } finally { - IO.release(dis); statebean.setStateAndReset(State.IDLE); if (cancel) { return null; diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditConfigurationContext.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditConfigurationContext.java index 94fa5fb4b0d64b8e37c2c5dddb2f6146879b9ba7..7b405d93d63afd1a98b9efc2b97075957d7bca3a 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditConfigurationContext.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditConfigurationContext.java @@ -55,6 +55,8 @@ import java.util.Timer; import java.util.TimerTask; import java.util.stream.Collectors; +import static edu.umiacs.ace.monitor.settings.SettingsConstants.*; + /** * Set the IMS for the AuditThread to use. Also, startup a background task * to handle firing off monitor tasks. @@ -65,13 +67,6 @@ import java.util.stream.Collectors; */ public final class AuditConfigurationContext implements ServletContextListener { - //private static final String PARAM_IMS = "ims"; - //private static final String PARAM_IMS_PORT = "ims.port"; - //private static final String PARAM_IMS_TOKEN_CLASS = "ims.tokenclass"; - //private static final String PARAM_IMS_SSL = "ims.ssl"; - //private static final String PARAM_DISABLE_AUTO_AUDIT = "auto.audit.disable"; - //private static final String PARAM_THROTTLE_MAXAUDIT = "throttle.maxaudit"; - //private static final String PARAM_AUDIT_ONLY = "audit.only"; public static final String ATTRIBUTE_PAUSE = "pause"; private static final long HOUR = 1000 * 60 * 60; private Timer checkTimer; @@ -98,68 +93,51 @@ public final class AuditConfigurationContext implements ServletContextListener { ctx.setAttribute(ATTRIBUTE_PAUSE, pb); // Invert the boolean because the PB checks if we're paused, not enabled - String enableAudits = resultMap.getOrDefault( - SettingsConstants.PARAM_AUTO_AUDIT_ENABLE, - SettingsConstants.autoAudit); + String enableAudits = resultMap.getOrDefault(PARAM_AUTO_AUDIT_ENABLE, autoAudit); pb.setPaused(!Boolean.valueOf(enableAudits)); checkTimer = new Timer("Audit Check Timer"); checkTimer.schedule(new MyTimerTask(pb), 0, HOUR); // set IMS for audit Thread from server parameter - AuditThreadFactory.setIMS(resultMap.getOrDefault( - SettingsConstants.PARAM_IMS, - SettingsConstants.ims)); + AuditThreadFactory.setIMS(resultMap.getOrDefault(PARAM_IMS, ims)); - String tokenClass = resultMap.getOrDefault( - SettingsConstants.PARAM_IMS_TOKEN_CLASS, - SettingsConstants.imsTokenClass); + String tokenClass = resultMap.getOrDefault(PARAM_IMS_TOKEN_CLASS, imsTokenClass); AuditThreadFactory.setTokenClass(tokenClass); - String port = resultMap.getOrDefault( - SettingsConstants.PARAM_IMS_PORT, - SettingsConstants.imsPort); + String port = resultMap.getOrDefault(PARAM_IMS_PORT, imsPort); if (Strings.isValidInt(port)) { AuditThreadFactory.setImsPort(Integer.parseInt(port)); } - String auditOnly = resultMap.getOrDefault( - SettingsConstants.PARAM_AUDIT_ONLY, - SettingsConstants.auditOnly); + String auditOnly = resultMap.getOrDefault(PARAM_AUDIT_ONLY, SettingsConstants.auditOnly); AuditThreadFactory.setAuditOnly(Boolean.valueOf(auditOnly)); - // keep this off for now even though it isn't used - AuditThreadFactory.setAuditSampling(false); - - String imsSsl = resultMap.getOrDefault( - SettingsConstants.PARAM_IMS_SSL, - SettingsConstants.imsSSL); + String imsSsl = resultMap.getOrDefault(PARAM_IMS_SSL, imsSSL); AuditThreadFactory.setSSL(Boolean.valueOf(imsSsl)); - String blocking = resultMap.getOrDefault( - SettingsConstants.PARAM_AUDIT_BLOCKING, - SettingsConstants.auditBlocking); + String blocking = resultMap.getOrDefault(PARAM_AUDIT_BLOCKING, auditBlocking); AuditThreadFactory.setBlocking(Boolean.valueOf(blocking)); - String blockTimeS = resultMap.getOrDefault( - SettingsConstants.PARAM_AUDIT_MAX_BLOCK_TIME, - SettingsConstants.auditMaxBlockTime); - int blockTime = 0; - if (Strings.isValidInt(blockTimeS)) { - blockTime = Integer.parseInt(blockTimeS); + int maxRetry = Integer.parseInt(imsMaxRetry); + String imsRetryString = resultMap.getOrDefault(PARAM_IMS_MAX_RETRY, imsMaxRetry); + if (Strings.isNonNegativeInt(imsRetryString)) { + maxRetry = Integer.parseInt(imsRetryString); } - - // Just in case... - if (blockTime < 0) { - blockTime = 0; + AuditThreadFactory.setImsRetryAttempts(maxRetry); + + int resetTimeout = Integer.parseInt(imsResetTimeout); + String imsResetTimeout = resultMap.getOrDefault( + PARAM_IMS_RESET_TIMEOUT, + SettingsConstants.imsResetTimeout); + if (Strings.isNonNegativeInt(imsResetTimeout)) { + resetTimeout = Integer.parseInt(imsResetTimeout); } - AuditThreadFactory.setMaxBlockTime(blockTime); + AuditThreadFactory.setImsResetTimeout(resetTimeout); - String maxAudit = resultMap.getOrDefault( - SettingsConstants.PARAM_THROTTLE_MAXAUDIT, - SettingsConstants.maxAudit); - if (Strings.isValidInt(maxAudit)) { - int audit = Integer.parseInt(maxAudit); + String maxAuditString = resultMap.getOrDefault(PARAM_THROTTLE_MAXAUDIT, maxAudit); + if (Strings.isValidInt(maxAuditString)) { + int audit = Integer.parseInt(maxAuditString); if (audit > 0) { AuditThreadFactory.setMaxAudits(audit); } diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java index cedbcef826a0838021e06f8ffdfa292f60a789a5..6c5429de6ce787af7c3016e75f3285c45150e54f 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java @@ -83,16 +83,14 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; /** - * - * * @author toaster */ public final class AuditThread extends Thread implements CancelCallback { private static final Logger LOG = Logger.getLogger(AuditThread.class); + private boolean fallback = false; - private Map itemMap = - new ConcurrentHashMap(); + private Map itemMap = new ConcurrentHashMap<>(); private String imsHost; private int imsPort; private Collection coll; @@ -101,7 +99,7 @@ public final class AuditThread extends Thread implements CancelCallback { private StorageDriver driver; private long session; private boolean auditOnly; - private boolean verbose = false; + private boolean verbose; // exposed for jsp display private long totalErrors = 0; private long newFilesFound = 0; @@ -116,17 +114,10 @@ public final class AuditThread extends Thread implements CancelCallback { private LogEventManager logManager; private AuditIterable iterableItems; - public AuditThread(Collection c, - StorageDriver driver, - boolean auditOnly, - MonitoredItem... startItem) { - this(c, driver, auditOnly, true, startItem); - } - - public AuditThread(Collection c, - StorageDriver driver, + public AuditThread(Collection c, + StorageDriver driver, boolean auditOnly, - boolean verbose, + boolean verbose, MonitoredItem... startItem) { this.auditOnly = auditOnly; this.verbose = verbose; @@ -222,18 +213,18 @@ public final class AuditThread extends Thread implements CancelCallback { callback = new FileAuditCallback(coll, session, this); boolean auditTokens = SettingsUtil.getBoolean(coll, - ConfigConstants.ATTR_AUDIT_TOKENS); + ConfigConstants.ATTR_AUDIT_TOKENS); // Audit only does not attempt to connect to the IMS, so we // only need these checks if we are not in it - if( !auditOnly ) { + if (!auditOnly) { // If we can open a connection to the IMS, check what mode // we're in, else fallback so we go to audit only mode - if ( openIms() ) { + if (openIms()) { boolean openValidator = openTokenValidator(MessageDigest.getInstance("SHA-256")); //short circuit - if ( auditTokens && !openValidator ) { + if (auditTokens && !openValidator) { return; } } else { @@ -242,8 +233,6 @@ public final class AuditThread extends Thread implements CancelCallback { } performAudit(); - // Let outstanding tokens finish, TODO, de-hackify this. - sleep(2000); } catch (Throwable e) { LOG.fatal("Uncaught exception in performAudit()", e); if (abortException != null) { @@ -264,11 +253,11 @@ public final class AuditThread extends Thread implements CancelCallback { validator.close(); validator = null; } - + lastFileSeen = "Setting collection state"; setCollectionState(); - if ( verbose ) { + if (verbose) { logAuditFinish(); generateAuditReport(); } @@ -283,16 +272,18 @@ public final class AuditThread extends Thread implements CancelCallback { private boolean openIms() { try { IMSService ims; - ims = IMSService.connect(imsHost, - imsPort, - AuditThreadFactory.useSSL(), - AuditThreadFactory.isBlocking(), - AuditThreadFactory.getMaxBlockTime()); - - batch = ims.createImmediateTokenRequestBatch(tokenClassName, - callback, - 1000, - 5000); + ims = IMSService.connect(imsHost, + imsPort, + AuditThreadFactory.useSSL(), + AuditThreadFactory.isBlocking(), + AuditThreadFactory.getImsRetryAttempts(), + AuditThreadFactory.getImsResetTimeout()); + + batch = ims.createImmediateTokenRequestBatch(coll.getId().toString(), + tokenClassName, + callback, + 1000, + 5000); return true; } catch (IMSException e) { EntityManager em; @@ -317,17 +308,20 @@ public final class AuditThread extends Thread implements CancelCallback { try { IMSService ims; ims = IMSService.connect(imsHost, - imsPort, - AuditThreadFactory.useSSL(), - AuditThreadFactory.isBlocking(), - AuditThreadFactory.getMaxBlockTime()); + imsPort, + AuditThreadFactory.useSSL(), + AuditThreadFactory.isBlocking(), + AuditThreadFactory.getImsRetryAttempts(), + AuditThreadFactory.getImsResetTimeout()); TokenAuditCallback tokenCallback = new TokenAuditCallback(itemMap, this, coll, session); - validator = ims.createTokenValidator(tokenCallback, 1000, - 5000, digest); - // "SHA-256-0", new BatchCallback(), 1000, 5000); + validator = ims.createTokenValidator(coll.getId().toString(), + tokenCallback, + 1000, + 5000, + digest); return true; } catch (IMSException e) { EntityManager em; @@ -346,7 +340,6 @@ public final class AuditThread extends Thread implements CancelCallback { } private void performAudit() { - // 1. Setup audit PathFilter filter = new SimpleFilter(coll); Date startDate = new Date(); @@ -354,7 +347,7 @@ public final class AuditThread extends Thread implements CancelCallback { // Sleep to ensure that we update a monitored item at a time strictly // greater than our start date try { - Thread.sleep(1500); + Thread.sleep(500); } catch (InterruptedException e) { abortException = e; return; @@ -363,9 +356,8 @@ public final class AuditThread extends Thread implements CancelCallback { // 2. Get file list try { - iterableItems = driver.getWorkList(coll.getDigestAlgorithm(), - filter, - baseItemPathList); + String digestAlgorithm = coll.getDigestAlgorithm(); + iterableItems = driver.getWorkList(digestAlgorithm, filter, baseItemPathList); } catch (Exception e) { abortException = e; return; @@ -407,15 +399,19 @@ public final class AuditThread extends Thread implements CancelCallback { return; } - // let token batch finish before processing or items waiting tokens - // will appear as errors. - // Thread.sleep(2000); + /* + * I'm pretty sure this isn't necessary, from my (brief) testing I haven't noticed any issues + * + * // let token batch finish before processing or items waiting tokens + * // will appear as errors. if (batch != null) { batch.close(); batch = null; } + */ // harvest remote collections + // it doesn't make sense for this to happen before the batch and validation threads close lastFileSeen = "comparing to peer sites"; compareToPeers(); } @@ -429,8 +425,7 @@ public final class AuditThread extends Thread implements CancelCallback { SchedulerContextListener.mailReport(rs, createMailList()); } catch (MessagingException e) { EntityManager em = PersistUtil.getEntityManager(); - logManager.persistCollectionEvent(LogEnum.SMTP_ERROR, - e.getMessage(), em); + logManager.persistCollectionEvent(LogEnum.SMTP_ERROR, e.getMessage(), em); em.close(); LOG.error("Could not send report summary", e); } @@ -441,7 +436,6 @@ public final class AuditThread extends Thread implements CancelCallback { * 1. we got an exception somewhere, abortException will be set * 2. user requested cancel * 3. we finished as expected - * */ private void logAuditFinish() { EntityManager em = PersistUtil.getEntityManager(); @@ -509,9 +503,9 @@ public final class AuditThread extends Thread implements CancelCallback { LOG.trace( "Driver returned {Item=" + currentFile.getPathList()[0] - + ";error=" + currentFile.isError() + ";error-msg=" - + currentFile.getErrorMessage() + ";hash=" - + currentFile.getHash() + "}"); + + ";error=" + currentFile.isError() + ";error-msg=" + + currentFile.getErrorMessage() + ";hash=" + + currentFile.getHash() + "}"); mim = new MonitoredItemManager(em); @@ -522,8 +516,8 @@ public final class AuditThread extends Thread implements CancelCallback { try { String parentName = extractAndRegisterParent(mim, currentFile); - MonitoredItem item = null; - if ((item = mim.getItemByPath(fileName, coll)) != null) { + MonitoredItem item = mim.getItemByPath(fileName, coll); + if (item != null) { LogEvent event = null; LOG.trace("Updating existing item " + fileName); item.setLastVisited(new Date()); @@ -548,8 +542,7 @@ public final class AuditThread extends Thread implements CancelCallback { trans.commit(); } else if (!auditOnly && !fallback) { // OK, no registered item, do the registration - LogEvent[] event = addNewFile(currentFile, fileName, - parentName, mim); + LogEvent[] event = addNewFile(currentFile, fileName, parentName, mim); // transaction inside loop to ensure order of commits // default, jpa ignores order of persist calls @@ -572,45 +565,44 @@ public final class AuditThread extends Thread implements CancelCallback { private LogEvent requestNewToken(FileBean currentFile, MonitoredItem item) { LogEvent event; if (item.getState() != 'T') { - event = logManager.createItemEvent(LogEnum.MISSING_TOKEN, - item.getPath()); + event = logManager.createItemEvent(LogEnum.MISSING_TOKEN, item.getPath()); item.setStateChange(new Date()); item.setState('T'); item.setSize(currentFile.getFileSize()); - LOG.trace( - "Toggling state to Missing Token for " + item.getPath()); + LOG.trace("Toggling state to Missing Token for " + item.getPath()); } else { event = null; } + String currentFileHash = currentFile.getHash(); TokenRequest request = new TokenRequest(); request.setName(item.getId().toString()); - request.setHashValue(currentFile.getHash()); - if (!Strings.isEmpty(item.getPath()) && - !Strings.isEmpty(currentFile.getHash()) && batch != null) { + request.setHashValue(currentFileHash); + if (!Strings.isEmpty(item.getPath()) && + !Strings.isEmpty(currentFileHash) && batch != null) { try { batch.add(request); } catch (InterruptedException e) { - abortException = e; + abortException = e; } } return event; } private LogEvent validateIntegrity(FileBean currentFile, MonitoredItem item) { - LOG.trace( - "Generated checksum: " + currentFile.getHash() - + " expected checksum: " + item.getFileDigest()); + String storedDigest = item.getFileDigest(); + String currentFileHash = currentFile.getHash(); + LOG.trace("Generated checksum: " + currentFileHash + " expected checksum: " + storedDigest); LogEvent event; // If we have a registered file, set the digested value - if (null == item.getFileDigest()) { + if (null == storedDigest) { LOG.trace("Setting digest for registered file " + item.getPath()); - item.setFileDigest(currentFile.getHash()); + item.setFileDigest(currentFileHash); item.setLastSeen(new Date()); } - if (currentFile.getHash().equals(item.getFileDigest())) { + if (currentFileHash.equals(storedDigest)) { LOG.trace("Digests match for " + item.getPath()); // File is active and intact // log the transition if it already isn't A @@ -618,8 +610,7 @@ public final class AuditThread extends Thread implements CancelCallback { // we handle those later if (item.getState() != 'A' && item.getState() != 'P' && item.getState() != 'D' && item.getState() != 'R') { - event = logManager.createItemEvent(LogEnum.FILE_ONLINE, - item.getPath()); + event = logManager.createItemEvent(LogEnum.FILE_ONLINE, item.getPath()); item.setState('A'); item.setStateChange(new Date()); LOG.trace("Toggling state to Active for " + item.getPath()); @@ -633,21 +624,18 @@ public final class AuditThread extends Thread implements CancelCallback { if (validator != null) { itemMap.put(token, item); try { - validator.add(item.getFileDigest(), token); + validator.add(storedDigest, token); } catch (InterruptedException e) { abortException = e; } } } else { if (item.getState() != 'C') { - String msg = "Expected digest: " + item.getFileDigest() - + " Saw: " + currentFile.getHash(); - event = logManager.createItemEvent(LogEnum.FILE_CORRUPT, - item.getPath(), msg); + String msg = "Expected digest: " + storedDigest + " Saw: " + currentFileHash; + event = logManager.createItemEvent(LogEnum.FILE_CORRUPT, item.getPath(), msg); item.setState('C'); item.setStateChange(new Date()); - LOG.trace( - "Toggling state to Corrupt for " + item.getPath()); + LOG.trace("Toggling state to Corrupt for " + item.getPath()); } else { event = null; } @@ -656,15 +644,17 @@ public final class AuditThread extends Thread implements CancelCallback { return event; } - private LogEvent[] addNewFile(FileBean currentFile, String fileName, - final String parentName, final MonitoredItemManager mim) { + private LogEvent[] addNewFile(FileBean currentFile, + String fileName, + final String parentName, + final MonitoredItemManager mim) { newFilesFound++; LogEvent[] event = new LogEvent[2]; LOG.trace("Registering new item " + fileName); - event[0] = logManager.createItemEvent(LogEnum.FILE_NEW, fileName, - coll.getDirectory() + fileName); + String fullName = coll.getDirectory() + fileName; + event[0] = logManager.createItemEvent(LogEnum.FILE_NEW, fileName, fullName); if (currentFile.isError()) { mim.addItem(fileName, parentName, false, coll, 'M', 0); event[1] = logManager.createItemEvent(LogEnum.ERROR_READING, fileName, @@ -678,8 +668,8 @@ public final class AuditThread extends Thread implements CancelCallback { TokenRequest request = new TokenRequest(); request.setName(mi.getId().toString()); request.setHashValue(currentFile.getHash()); - if (!Strings.isEmpty(fileName) && - !Strings.isEmpty(currentFile.getHash()) && batch != null) { + if (!Strings.isEmpty(fileName) && + !Strings.isEmpty(currentFile.getHash()) && batch != null) { try { batch.add(request); } catch (InterruptedException e) { @@ -690,12 +680,10 @@ public final class AuditThread extends Thread implements CancelCallback { return event; } - private String extractAndRegisterParent(MonitoredItemManager mim, - FileBean currentFile) { + private String extractAndRegisterParent(MonitoredItemManager mim, FileBean currentFile) { String parentName = (currentFile.getPathList().length > 1 ? currentFile.getPathList()[1] : null); - // 1. make sure directory path is registered if (parentName != null) { parentName = Strings.cleanStringForXml(parentName, '_'); @@ -757,8 +745,8 @@ public final class AuditThread extends Thread implements CancelCallback { try { // Update the monitored item table Query query = em.createNamedQuery("MonitoredItem.updateMissing") - .setParameter("coll", coll) - .setParameter("date", d); + .setParameter("coll", coll) + .setParameter("date", d); int i = query.executeUpdate(); if (i > 0) { @@ -787,12 +775,10 @@ public final class AuditThread extends Thread implements CancelCallback { } private void compareToPeers() { - EntityManager em = PersistUtil.getEntityManager(); MonitoredItemManager mim = new MonitoredItemManager(em); List currentErrors = mim.listRemoteErrors(coll); - mim = null; em.close(); em = null; @@ -803,12 +789,11 @@ public final class AuditThread extends Thread implements CancelCallback { if (digestStream == null) { em = PersistUtil.getEntityManager(); logManager.persistCollectionEvent(LogEnum.SYSTEM_ERROR, - "Cannot collect digests from remote site: " - + pc.getSite().getRemoteURL(), em); + "Cannot collect digests from remote site: " + pc.getSite().getRemoteURL(), + em); em.close(); em = null; - LOG.info("remote site returned null stream " + pc.getSite(). - getRemoteURL()); + LOG.info("remote site returned null stream " + pc.getSite().getRemoteURL()); continue; } CollectionCompare2 cc = new CollectionCompare2(digestStream, null); @@ -817,8 +802,6 @@ public final class AuditThread extends Thread implements CancelCallback { try { CompareResults cr = new CompareResults(cc); cc.compareTo(cr, coll, null); - // cc.loadCollectionTable(coll, null); - // cc.doCompare(); // For now, we don't care about files that only exist remotely // cc.getUnseenTargetFiles(); // State: P, only set if file is currently active as local errors @@ -827,8 +810,7 @@ public final class AuditThread extends Thread implements CancelCallback { mim = new MonitoredItemManager(em); for (String unseenFile : cr.getUnseenTargetFiles()) { - LOG.trace("Item missing at remote " - + unseenFile + " " + pc.getSite()); + LOG.trace("Item missing at remote " + unseenFile + " " + pc.getSite()); MonitoredItem mi = mim.getItemByPath(unseenFile, coll); currentErrors.remove(mi); if (mi.getState() == 'A') { @@ -850,12 +832,10 @@ public final class AuditThread extends Thread implements CancelCallback { MonitoredItem mi = mim.getItemByPath(dd.getName(), coll); currentErrors.remove(mi); if (mi.getState() == 'A') { - LOG.trace( - "Item corrupt at remote " + mi.getPath() + " " + pc.getSite()); + LOG.trace("Item corrupt at remote " + mi.getPath() + " " + pc.getSite()); mi.setState('D'); mi.setStateChange(new Date()); - EntityTransaction trans = em.getTransaction(); trans.begin(); em.merge(mi); @@ -882,12 +862,10 @@ public final class AuditThread extends Thread implements CancelCallback { trans.begin(); for (MonitoredItem mi : currentErrors) { - LOG.trace( - "Item online at remote " + mi.getPath()); + LOG.trace("Item online at remote " + mi.getPath()); mi.setState('A'); mi.setStateChange(currDate); - em.persist(logManager.createItemEvent(LogEnum.REMOTE_FILE_ONLINE, - mi.getPath())); + em.persist(logManager.createItemEvent(LogEnum.REMOTE_FILE_ONLINE, mi.getPath())); em.merge(mi); } diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThreadFactory.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThreadFactory.java index a2a506b109a494e36d423360fe6f594d05fc96f4..babbe70eddd4b56fd204fb6d33f51e4d0fd759ba 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThreadFactory.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThreadFactory.java @@ -65,10 +65,10 @@ public class AuditThreadFactory { private static int imsPort = 80; private static String tokenClass = "SHA-256"; private static boolean auditOnly = false; - private static boolean auditSample = false; private static boolean ssl = false; private static boolean blocking = false; - private static int maxBlockTime = 0; + private static int imsRetryAttempts = 3; + private static int imsResetTimeout = 3000; public static void setIMS( String ims ) { if (Strings.isEmpty(ims)) { @@ -95,8 +95,8 @@ public class AuditThreadFactory { } public static void setImsPort(int imsPort) { - if (imsPort < 1 && imsPort > 32768) { - LOG.error("ims port must be between 1 and 32768, setting default"); + if (imsPort < 1 || imsPort > 32768) { + LOG.warn("ims port must be between 1 and 32768, setting default"); imsPort = Integer.parseInt(SettingsConstants.imsPort); } AuditThreadFactory.imsPort = imsPort; @@ -106,27 +106,32 @@ public class AuditThreadFactory { AuditThreadFactory.auditOnly = auditOnlyMode; } - public static void setAuditSampling(boolean auditSampling ) { - AuditThreadFactory.auditSample = auditSampling; + public static int getImsRetryAttempts() { + return imsRetryAttempts; } - public static void setBlocking(boolean blocking) { - AuditThreadFactory.blocking = blocking; + public static void setImsRetryAttempts(int attempts) { + if (attempts >= 0) { + imsRetryAttempts = attempts; + } } - public static boolean isBlocking() { - return AuditThreadFactory.blocking; + public static int getImsResetTimeout() { + return imsResetTimeout; } - public static void setMaxBlockTime(int maxBlockTime) { - if ( maxBlockTime < 0 ) { - maxBlockTime = 0; + public static void setImsResetTimeout(int timeout) { + if (timeout >= 0) { + imsResetTimeout = timeout; } - AuditThreadFactory.maxBlockTime = maxBlockTime; } - public static int getMaxBlockTime() { - return maxBlockTime; + public static void setBlocking(boolean blocking) { + AuditThreadFactory.blocking = blocking; + } + + public static boolean isBlocking() { + return AuditThreadFactory.blocking; } public static boolean isAuditing() { diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditTokens.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditTokens.java index c1ccde351e4448837a30a1518b834b99030310aa..95670b4c7221868f95714b3d465cfa597d11a12d 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditTokens.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditTokens.java @@ -197,9 +197,13 @@ public final class AuditTokens extends Thread implements CancelCallback { imsPort, AuditThreadFactory.useSSL(), AuditThreadFactory.isBlocking(), - AuditThreadFactory.getMaxBlockTime()); + AuditThreadFactory.getImsRetryAttempts(), + AuditThreadFactory.getImsResetTimeout()); callback = new TokenAuditCallback(itemMap, this, collection, session); - validator = ims.createTokenValidator(callback, 1000, 5000, + validator = ims.createTokenValidator(collection.getId().toString(), + callback, + 1000, + 5000, digest); } catch (Throwable e) { EntityManager em; diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/TokenAuditCallback.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/TokenAuditCallback.java index acfa76077279ed0ae0dc299ab0e21cc58dc31899..4a2888e9a33f0277649cf76d082e83c27c5345b1 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/TokenAuditCallback.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/TokenAuditCallback.java @@ -57,14 +57,14 @@ public final class TokenAuditCallback implements ValidationCallback { private long totalErrors = 0; private long validTokens = 0; private CancelCallback cancel; - // private Collection collection; - LogEventManager logManager; + private LogEventManager logManager; - public TokenAuditCallback( Map itemMap, - CancelCallback callback, Collection collection, long session ) { + public TokenAuditCallback(Map itemMap, + CancelCallback callback, + Collection collection, + long session ) { this.itemMap = itemMap; this.cancel = callback; - // this.collection = collection; logManager = new LogEventManager(session, collection); } @@ -81,12 +81,8 @@ public final class TokenAuditCallback implements ValidationCallback { totalErrors++; LOG.error("Exception throw registering", throwable); EntityManager em = PersistUtil.getEntityManager(); - // EntityTransaction trans = em.getTransaction(); - // trans.begin(); String msg = "Exception in batch thread" + Strings.exceptionAsString(throwable); logManager.persistCollectionEvent(LogEnum.SYSTEM_ERROR, msg, em); - // lem.abortSite(collection, "Exception in batch thread", throwable); - // trans.commit(); em.close(); cancel.cancel(); } @@ -106,10 +102,7 @@ public final class TokenAuditCallback implements ValidationCallback { if ( !token.getValid() ) { token.setValid(true); - // LogEventManager lem = new LogEventManager(session, collection); - // String path = response.getName(); em.persist(logManager.createItemEvent(LogEnum.TOKEN_VALID, item.getPath())); - // em.persist(lem.validToken(path, collection)); } if ( item.getState() == 'I' || item.getState() == 'R' ) { diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/AddSettingServlet.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/AddSettingServlet.java index e8226c1dfa4d77d02ecd2906039072f8d69573c0..7f6f484380ddbc39dddc23dc3e7823a4ca1db5f9 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/AddSettingServlet.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/AddSettingServlet.java @@ -6,32 +6,36 @@ package edu.umiacs.ace.monitor.settings; import edu.umiacs.ace.util.EntityManagerServlet; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import javax.persistence.EntityManager; -import javax.servlet.RequestDispatcher; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import org.apache.commons.fileupload.FileItemIterator; import org.apache.commons.fileupload.FileItemStream; import org.apache.commons.fileupload.FileUploadException; import org.apache.commons.fileupload.servlet.ServletFileUpload; import org.apache.commons.fileupload.util.Streams; +import org.apache.log4j.Logger; + +import javax.persistence.EntityManager; +import javax.servlet.RequestDispatcher; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; /** * * @author shake */ -public class AddSettingServlet extends EntityManagerServlet{ +public class AddSettingServlet extends EntityManagerServlet { + + private static final Logger log = Logger.getLogger(AddSettingServlet.class); @Override - protected void processRequest(HttpServletRequest request, HttpServletResponse - response, EntityManager em) throws ServletException, IOException { - List customSettings = new ArrayList(); + protected void processRequest(HttpServletRequest request, + HttpServletResponse response, + EntityManager em) throws ServletException, IOException { + List customSettings = new ArrayList<>(); ServletFileUpload su = new ServletFileUpload(); try { @@ -58,7 +62,7 @@ public class AddSettingServlet extends EntityManagerServlet{ } } catch (FileUploadException ex) { - // Logger.getLogger(SettingsServlet.class.getName()).log(Level.SEVERE, null, ex); + log.warn("Exception in AddSettingServlet", ex); } SettingsUtil.updateSettings(customSettings); diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/AutoAuditMigrationContextListener.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/AutoAuditMigrationContextListener.java index 21d7584f0296ad8216523b2af9ce83e9ac84357d..882147e6a5bdf589ce2e8eca4e80f29ac0494148 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/AutoAuditMigrationContextListener.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/AutoAuditMigrationContextListener.java @@ -2,7 +2,9 @@ package edu.umiacs.ace.monitor.settings; import edu.umiacs.ace.util.PersistUtil; import edu.umiacs.sql.SQL; +import edu.umiacs.util.Check; import org.apache.log4j.Logger; +import org.apache.log4j.NDC; import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextListener; @@ -12,45 +14,77 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import static edu.umiacs.ace.monitor.settings.SettingsConstants.PARAM_IMS_MAX_RETRY; +import static edu.umiacs.ace.monitor.settings.SettingsConstants.PARAM_IMS_RESET_TIMEOUT; +import static edu.umiacs.ace.monitor.settings.SettingsConstants.imsMaxRetry; +import static edu.umiacs.ace.monitor.settings.SettingsConstants.imsResetTimeout; + /** - * Context listener to migrate from auto.audit.disable to auto.audit.enable - * + * Context listener to migrate audit/ims settings + * auto.audit.disable to auto.audit.enable + * audit.max.block.time to ims.max.retry + * add ims.reset.timeout + *

* Created by shake on 4/5/17. */ public class AutoAuditMigrationContextListener implements ServletContextListener { private static final Logger LOG = Logger.getLogger(AutoAuditMigrationContextListener.class); - private ResultSet set; - private Connection conn; - private PreparedStatement statement; - @Override public void contextInitialized(ServletContextEvent servletContextEvent) { - Connection conn = null; + NDC.push("[MIGRATION] "); DataSource ds = PersistUtil.getDataSource(); - try { - conn = ds.getConnection(); - migrate(conn); + try (Connection conn = ds.getConnection()) { + migrateAutoAudit(conn); + migrateAuditBlocking(conn); } catch (Exception e) { - LOG.error("[MIGRATION] Error migrating audo.audit.disable setting", e); + LOG.error("Error migrating audit settings", e); } finally { - release(); - + NDC.pop(); } } - private void migrate(Connection conn) throws SQLException { - PreparedStatement statement = conn.prepareStatement("SELECT id, value, custom FROM system_settings WHERE attr = 'auto.audit.disable'"); - ResultSet set = statement.executeQuery(); - while (set.next()) { - Long id = set.getLong(1); - Boolean value = Boolean.valueOf(set.getString(2)); - Boolean custom = set.getBoolean(3); - if (!custom) { - LOG.info("[MIGRATION] Found auto.audit setting to migrate from " + value + " -> " + !value); - createNew(conn, !value); + /** + * Migrate from auto.audit.disable to auto.audit.enable + * + * @param conn The database connection + * @throws SQLException if there's an exception communicating with the database + */ + private void migrateAutoAudit(Connection conn) throws SQLException { + String query = "SELECT id, value, custom FROM system_settings WHERE attr = 'auto.audit.disable'"; + try (PreparedStatement statement = conn.prepareStatement(query); + ResultSet set = statement.executeQuery()) { + while (set.next()) { + Long id = set.getLong(1); + Boolean value = Boolean.valueOf(set.getString(2)); + Boolean custom = set.getBoolean(3); + if (!custom) { + LOG.info("Found auto.audit setting to migrate from " + value + " -> " + !value); + createNew(conn, "auto.audit.enable", String.valueOf(!value)); + delete(conn, id); + } + } + } + } + + /** + * Check if audit.max.block.time exists. If true, add ims.max.retry and ims.reset.timeout and + * remove audit.max.block.time + * + * @param conn The database connection + * @throws SQLException if there's an exception communicating with the database + */ + private void migrateAuditBlocking(Connection conn) throws SQLException { + String query = "SELECT id FROM system_settings WHERE attr = 'audit.max.block.time'"; + try (PreparedStatement statement = conn.prepareStatement(query); + ResultSet set = statement.executeQuery()) { + while (set.next()) { + Long id = set.getLong(1); + LOG.info("Found audit.max.block.time setting to migrate to ims.max.retry"); + createNew(conn, PARAM_IMS_MAX_RETRY, imsMaxRetry); + createNew(conn, PARAM_IMS_RESET_TIMEOUT, imsResetTimeout); delete(conn, id); } } @@ -62,21 +96,14 @@ public class AutoAuditMigrationContextListener implements ServletContextListener SQL.release(statement); } - private void createNew(Connection conn, Boolean value) throws SQLException { - PreparedStatement statement = conn.prepareStatement("INSERT INTO system_settings VALUES (DEFAULT, 'auto.audit.enable','" + value.toString() + "',0)"); - statement.executeUpdate(); - SQL.release(statement); - } + private void createNew(Connection conn, String attr, String value) throws SQLException { + Check.notNull("attr", attr); + Check.notNull("value", value); - private void release() { - if (conn != null) { - SQL.release(conn); - } - if (statement != null) { - SQL.release(statement); - } - if (set != null) { - SQL.release(set); + String query = "INSERT INTO system_settings VALUES (DEFAULT, '%s', '%s', 0)"; + try (PreparedStatement statement = + conn.prepareStatement(String.format(query, attr, value))) { + statement.executeUpdate(); } } diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/DeleteSettingsServlet.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/DeleteSettingsServlet.java index aa07688fe56d956021a48c61e54b7dfe7cee27c6..f5b9c260b87015c2e48153ad9572731abdc03f66 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/DeleteSettingsServlet.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/DeleteSettingsServlet.java @@ -7,15 +7,16 @@ package edu.umiacs.ace.monitor.settings; import edu.umiacs.ace.util.EntityManagerServlet; import edu.umiacs.util.Strings; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; + import javax.persistence.EntityManager; import javax.servlet.RequestDispatcher; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; /** * @@ -25,10 +26,11 @@ public class DeleteSettingsServlet extends EntityManagerServlet{ public static final String PARAM_SETTING = "setting"; @Override - protected void processRequest(HttpServletRequest request, HttpServletResponse - response, EntityManager em) throws ServletException, IOException { + protected void processRequest(HttpServletRequest request, + HttpServletResponse response, + EntityManager em) throws ServletException, IOException { - List settingsToDelete = new ArrayList(); + List settingsToDelete = new ArrayList<>(); RequestDispatcher dispatcher = null; if ( !Strings.isEmpty(request.getParameter(PARAM_SETTING)) ) { diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsConstants.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsConstants.java index 6fd3e18b82db9899b2026a7e220b8e046678f5fa..438325c2a46f858850d4e48270ac45c4e132ac98 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsConstants.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsConstants.java @@ -1,18 +1,19 @@ package edu.umiacs.ace.monitor.settings; /** - * * @author shake */ public class SettingsConstants { + // Attributes public static final String PARAM_AUDIT_BLOCKING = "audit.blocking"; - public static final String PARAM_AUDIT_MAX_BLOCK_TIME = "audit.max.block.time"; - public static final String PARAM_AUDIT_ONLY="audit.only"; + public static final String PARAM_AUDIT_ONLY = "audit.only"; public static final String PARAM_AUDIT_SAMPLE = "audit.sample"; public static final String PARAM_IMS = "ims"; public static final String PARAM_IMS_PORT = "ims.port"; public static final String PARAM_IMS_TOKEN_CLASS = "ims.tokenclass"; + public static final String PARAM_IMS_MAX_RETRY = "ims.max.retry"; + public static final String PARAM_IMS_RESET_TIMEOUT = "ims.reset.timeout"; public static final String PARAM_IMS_SSL = "ims.ssl"; public static final String PARAM_AUTO_AUDIT_ENABLE = "auto.audit.enable"; public static final String PARAM_THROTTLE_MAXAUDIT = "throttle.maxaudit"; @@ -22,19 +23,17 @@ public class SettingsConstants { public static final String PARAM_FROM = "mail.from"; public static final String PARAM_USER_AUTH = "auth.management"; public static final String PARAM_4J_FILE = "log4j.appender.A1.File"; - public static final String PARAM_4J_APPENDER = "log4j.appender.A1"; + public static final String PARAM_4J_APPENDER = "log4j.appender.A1"; public static final String PARAM_4J_FILE_SIZE = "log4j.appender.A1.maxFileSize"; public static final String PARAM_4J_BACKUP_INDEX = "log4j.appender.A1.maxBackupIndex"; public static final String PARAM_4J_ROOT_LOGGER = "log4j.rootLogger"; - public static final String PARAM_4J_LAYOUT = "log4j.appender.A1.layout"; + public static final String PARAM_4J_LAYOUT = "log4j.appender.A1.layout"; public static final String PARAM_4J_CONV_PAT = "log4j.appender.A1.layout.ConversionPattern"; public static final String PARAM_4J_IRODS = "log4j.logger.edu.umiacs.irods"; - public static final String PARAM_4J_CLASS ="log4j.logger.edu.umiacs"; - public static final String PARAM_INGEST = "ingest.maxthreads"; + public static final String PARAM_4J_CLASS = "log4j.logger.edu.umiacs"; // Default Values public static final String auditBlocking = "false"; - public static final String auditMaxBlockTime = "0"; public static final String auditOnly = "false"; public static final String auditSample = "false"; public static final String mailServer = "localhost.localdomain"; @@ -44,22 +43,45 @@ public class SettingsConstants { public static final String ims = "ims.umiacs.umd.edu"; public static final String imsPort = "80"; public static final String imsTokenClass = "SHA-256"; + public static final String imsMaxRetry = "3"; + public static final String imsResetTimeout = "3000"; public static final String imsSSL = "false"; public static final String authManagement = "true"; - public static final String maxIngestThreads = "4"; public static final String throttleWait = "0"; - public static final String throttleBPS= "0"; + public static final String throttleBPS = "0"; // Yay log4j public static final String log4JA1File = "/tmp/aceam.log"; public static final String log4JA1 = "org.apache.log4j.RollingFileAppender"; public static final String log4JA1MaxFileSize = "100000KB"; public static final String log4JA1MaxBackupIndex = "5"; public static final String log4JRootLogger = "FATAL, A1"; - public static final String log4JA1Layout = "org.apache.log4j.PatternLayout"; + public static final String log4JA1Layout = "org.apache.log4j.PatternLayout"; public static final String log4JA1layoutConversationPattern = "%d{[dd/MMM/yyyy:HH:mm:ss]} %x%m%n"; public static final String log4JLoggerIrods = "ERROR"; public static final String log4JLoggerUMIACS = "TRACE"; + /** + * @deprecated migrated to PARAM_IMS_MAX_RETRY and PARAM_IMS_RESET_TIMEOUT + */ + @Deprecated + public static final String PARAM_AUDIT_MAX_BLOCK_TIME = "audit.max.block.time"; + /** + * @deprecated migrated to PARAM_IMS_MAX_RETRY and PARAM_IMS_RESET_TIMEOUT + */ + @Deprecated + public static final String auditMaxBlockTime = "0"; + + /** + * @deprecated no longer used + */ + @Deprecated + public static final String PARAM_INGEST = "ingest.maxthreads"; + + /** + * @deprecated no longer used + */ + @Deprecated + public static final String maxIngestThreads = "4"; } diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsParameter.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsParameter.java index 897a909db5a329aa523fa3b7649ff353acbede93..c5ba602458bb13ce450e6f31ab710b0cf6574db1 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsParameter.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsParameter.java @@ -49,6 +49,7 @@ public class SettingsParameter implements Serializable { public SettingsParameter(String attr, String value) { this.attr = attr; this.value = value; + this.custom = false; } public Long getId() { diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsServlet.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsServlet.java index e004e5cf8f3c33ee2320ec7790f83266c0f71b72..97055e9468326f4c370fe74004161b094e3b40ef 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsServlet.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsServlet.java @@ -63,10 +63,8 @@ public class SettingsServlet extends EntityManagerServlet { } } - Map settingsMap = - settingsToMap(SettingsUtil.getCurrentSettings()); - Map customMap = - settingsToMap(SettingsUtil.getCustomSettings()); + Map settingsMap = settingsToMap(SettingsUtil.getCurrentSettings()); + Map customMap = settingsToMap(SettingsUtil.getCustomSettings()); request.setAttribute("currSettings", settingsMap); request.setAttribute("customSettings", customMap); diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsUtil.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsUtil.java index 0868f39e9d09dfad2e26905b807da571205ef134..cadbb984c8fdee2d35c866e73e326ed4c5347ae9 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsUtil.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsUtil.java @@ -4,6 +4,7 @@ import edu.umiacs.ace.monitor.audit.AuditThreadFactory; import edu.umiacs.ace.monitor.core.Collection; import edu.umiacs.ace.monitor.reporting.SchedulerContextListener; import edu.umiacs.ace.util.PersistUtil; +import edu.umiacs.util.Strings; import javax.persistence.EntityManager; import javax.persistence.EntityTransaction; @@ -11,12 +12,11 @@ import javax.persistence.NoResultException; import javax.persistence.Query; import javax.persistence.TypedQuery; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; + +import static edu.umiacs.ace.monitor.settings.SettingsConstants.*; /** - * * @author shake */ public class SettingsUtil { @@ -27,7 +27,7 @@ public class SettingsUtil { * @param attr the attribute to query on * @return the Setting associated with the attribute */ - public static SettingsParameter getItemByAttr( String attr ) { + public static SettingsParameter getItemByAttr(String attr) { EntityManager em = PersistUtil.getEntityManager(); Query q = em.createNamedQuery("SettingsParameter.getAttr"); q.setParameter("attr", attr); @@ -35,7 +35,7 @@ public class SettingsUtil { try { return (SettingsParameter) q.getSingleResult(); } catch (NoResultException ex) { - // zzz + // ignore, return null } finally { em.close(); } @@ -87,17 +87,17 @@ public class SettingsUtil { EntityTransaction trans = em.getTransaction(); trans.begin(); - for ( SettingsParameter setting : settings ) { + for (SettingsParameter setting : settings) { // Skip any empty settings - if ( setting.getName().trim().isEmpty() || - setting.getValue().trim().isEmpty() ) { + if (setting.getName().trim().isEmpty() || + setting.getValue().trim().isEmpty()) { continue; } SettingsParameter old = getItemByAttr(setting.getName()); // If there is no item, persist the new setting - if ( old == null ) { + if (old == null) { em.persist(setting); } else { // Else update and merge the old item @@ -122,14 +122,14 @@ public class SettingsUtil { EntityTransaction trans = em.getTransaction(); trans.begin(); - for ( String name : settings ) { + for (String name : settings) { // TODO: Find a better way to do this // Could possibly wrap the ID into the settings list SettingsParameter setting = getItemByAttr(name); SettingsParameter managedSetting = em.find(SettingsParameter.class, setting.getId()); - if ( setting != null ) { + if (setting != null) { em.remove(managedSetting); } } @@ -138,16 +138,6 @@ public class SettingsUtil { em.close(); } - // Get the names of all current settings - public static Set getParamNames() { - List settings = getCurrentSettings(); - Set paramSet = new HashSet<>(); - for ( SettingsParameter s : settings ) { - paramSet.add(s.getName()); - } - return paramSet; - } - /** * Get the default values for all settings @@ -157,68 +147,43 @@ public class SettingsUtil { public static List getDefaultSettings() { List defaults = new ArrayList<>(); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_IMS, - SettingsConstants.ims,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_IMS_PORT, - SettingsConstants.imsPort,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_IMS_SSL, - SettingsConstants.imsSSL,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_IMS_TOKEN_CLASS, - SettingsConstants.imsTokenClass,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_AUTO_AUDIT_ENABLE, - SettingsConstants.autoAudit,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_THROTTLE_MAXAUDIT, - SettingsConstants.maxAudit,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_TIME, - SettingsConstants.throttleWait,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_BPS, - SettingsConstants.throttleBPS,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_SMTP_SERVER, - SettingsConstants.mailServer,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_FROM, - SettingsConstants.mailFrom,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_USER_AUTH, - SettingsConstants.authManagement,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_4J_APPENDER, - SettingsConstants.log4JA1,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_4J_BACKUP_INDEX, - SettingsConstants.log4JA1MaxBackupIndex,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_4J_CLASS, - SettingsConstants.log4JLoggerUMIACS,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_4J_CONV_PAT, - SettingsConstants.log4JA1layoutConversationPattern,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_4J_FILE, - SettingsConstants.log4JA1File,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_4J_FILE_SIZE, - SettingsConstants.log4JA1MaxFileSize,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_4J_IRODS, - SettingsConstants.log4JLoggerIrods,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_4J_LAYOUT, - SettingsConstants.log4JA1Layout,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_4J_ROOT_LOGGER, - SettingsConstants.log4JRootLogger,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_INGEST, - SettingsConstants.maxIngestThreads, false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_AUDIT_ONLY, - SettingsConstants.auditOnly, false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_AUDIT_SAMPLE, - SettingsConstants.auditSample, false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_AUDIT_BLOCKING, - SettingsConstants.auditBlocking, false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_AUDIT_MAX_BLOCK_TIME, - SettingsConstants.auditMaxBlockTime, false)); + defaults.add(new SettingsParameter(PARAM_IMS, ims)); + defaults.add(new SettingsParameter(PARAM_IMS_PORT, imsPort)); + defaults.add(new SettingsParameter(PARAM_IMS_SSL, imsSSL)); + defaults.add(new SettingsParameter(PARAM_IMS_TOKEN_CLASS, imsTokenClass)); + defaults.add(new SettingsParameter(PARAM_AUTO_AUDIT_ENABLE, autoAudit)); + defaults.add(new SettingsParameter(PARAM_THROTTLE_MAXAUDIT, maxAudit)); + defaults.add(new SettingsParameter(PARAM_TIME, throttleWait)); + defaults.add(new SettingsParameter(PARAM_BPS, throttleBPS)); + defaults.add(new SettingsParameter(PARAM_SMTP_SERVER, mailServer)); + defaults.add(new SettingsParameter(PARAM_FROM, mailFrom)); + defaults.add(new SettingsParameter(PARAM_USER_AUTH, authManagement)); + defaults.add(new SettingsParameter(PARAM_4J_APPENDER, log4JA1)); + defaults.add(new SettingsParameter(PARAM_4J_BACKUP_INDEX, log4JA1MaxBackupIndex)); + defaults.add(new SettingsParameter(PARAM_4J_CLASS, log4JLoggerUMIACS)); + defaults.add(new SettingsParameter(PARAM_4J_CONV_PAT, log4JA1layoutConversationPattern)); + defaults.add(new SettingsParameter(PARAM_4J_FILE, log4JA1File)); + defaults.add(new SettingsParameter(PARAM_4J_FILE_SIZE, log4JA1MaxFileSize)); + defaults.add(new SettingsParameter(PARAM_4J_IRODS, log4JLoggerIrods)); + defaults.add(new SettingsParameter(PARAM_4J_LAYOUT, log4JA1Layout)); + defaults.add(new SettingsParameter(PARAM_4J_ROOT_LOGGER, log4JRootLogger)); + // defaults.add(new SettingsParameter(PARAM_INGEST, maxIngestThreads)); + defaults.add(new SettingsParameter(PARAM_AUDIT_ONLY, auditOnly)); + defaults.add(new SettingsParameter(PARAM_AUDIT_SAMPLE, auditSample)); + defaults.add(new SettingsParameter(PARAM_AUDIT_BLOCKING, auditBlocking)); + defaults.add(new SettingsParameter(PARAM_IMS_MAX_RETRY, imsMaxRetry)); + defaults.add(new SettingsParameter(PARAM_IMS_RESET_TIMEOUT, imsResetTimeout)); return defaults; } /** - * - * @param c The collection to query + * @param c The collection to query * @param attr The attribute to query for * @return true if collection, settings are not null and parameter is "true" */ public static boolean getBoolean(Collection c, String attr) { - if (!containsKey(c, attr)) { + if (notContains(c, attr)) { return false; } @@ -226,7 +191,7 @@ public class SettingsUtil { } public static String getString(Collection c, String attr) { - if (!containsKey(c, attr)) { + if (notContains(c, attr)) { return null; } @@ -234,7 +199,7 @@ public class SettingsUtil { } public static int getInt(Collection c, String attr, int def) { - if (!containsKey(c, attr)) { + if (notContains(c, attr)) { return def; } try { @@ -244,40 +209,34 @@ public class SettingsUtil { } } - private static boolean containsKey(Collection c, String attr) { - return (c != null && c.getSettings() != null - && c.getSettings().containsKey(attr)); + private static boolean notContains(Collection c, String attr) { + return (c == null || c.getSettings() == null + || !c.getSettings().containsKey(attr)); } // Update the settings our context listeners would normally do private static void reloadSettings() { EntityManager em = PersistUtil.getEntityManager(); - Query q = em.createNamedQuery("SettingsParameter.getAttr"); - SettingsParameter s = null; + SettingsParameter s; // Host - q.setParameter("attr", SettingsConstants.PARAM_IMS); - s = (SettingsParameter) q.getSingleResult(); + s = getOrDefault(PARAM_IMS, ims, em); AuditThreadFactory.setIMS(s.getValue()); // Port - q.setParameter("attr", SettingsConstants.PARAM_IMS_PORT); - s = (SettingsParameter) q.getSingleResult(); + s = getOrDefault(PARAM_IMS_PORT, imsPort, em); AuditThreadFactory.setImsPort(Integer.parseInt(s.getValue())); // SSL - q.setParameter("attr", SettingsConstants.PARAM_IMS_SSL); - s = (SettingsParameter) q.getSingleResult(); + s = getOrDefault(PARAM_IMS_SSL, imsSSL, em); AuditThreadFactory.setSSL(Boolean.valueOf(s.getValue())); // Token Class - q.setParameter("attr", SettingsConstants.PARAM_IMS_TOKEN_CLASS); - s = (SettingsParameter) q.getSingleResult(); + s = getOrDefault(PARAM_IMS_TOKEN_CLASS, imsTokenClass, em); AuditThreadFactory.setTokenClass(s.getValue()); // Audit Only - q.setParameter("attr", SettingsConstants.PARAM_AUDIT_ONLY); - s = (SettingsParameter) q.getSingleResult(); + s = getOrDefault(PARAM_AUDIT_ONLY, auditOnly, em); AuditThreadFactory.setAuditOnly(Boolean.valueOf(s.getValue())); // Max Audits @@ -291,39 +250,42 @@ public class SettingsUtil { */ // Mail Server - q.setParameter("attr", SettingsConstants.PARAM_SMTP_SERVER); - s = (SettingsParameter) q.getSingleResult(); + s = getOrDefault(PARAM_SMTP_SERVER, mailServer, em); SchedulerContextListener.setMailServer(s.getValue()); // Mail From - q.setParameter("attr", SettingsConstants.PARAM_FROM); - s = (SettingsParameter) q.getSingleResult(); + s = getOrDefault(PARAM_FROM, mailFrom, em); SchedulerContextListener.setMailFrom(s.getValue()); // Audit Blocking - q.setParameter("attr", SettingsConstants.PARAM_AUDIT_BLOCKING); - s = (SettingsParameter) q.getSingleResult(); + s = getOrDefault(PARAM_AUDIT_BLOCKING, auditBlocking, em); AuditThreadFactory.setBlocking(Boolean.valueOf(s.getValue())); - // Max block time - q.setParameter("attr", SettingsConstants.PARAM_AUDIT_MAX_BLOCK_TIME); - s = (SettingsParameter) q.getSingleResult(); - AuditThreadFactory.setMaxBlockTime(Integer.parseInt(s.getValue())); + s = getOrDefault(PARAM_IMS_MAX_RETRY, imsMaxRetry, em); + if (Strings.isNonNegativeInt(s.getValue())) { + AuditThreadFactory.setImsRetryAttempts(Integer.parseInt(s.getValue())); + } + + s = getOrDefault(PARAM_IMS_RESET_TIMEOUT, imsResetTimeout, em); + if (Strings.isNonNegativeInt(s.getValue())) { + AuditThreadFactory.setImsResetTimeout(Integer.parseInt(s.getValue())); + } } /** * Get a SettingsParameter by its attribute name or set the default value and persist it * - * @param attr the Attribute to query for + * @param attr the Attribute to query for * @param defaultValue the default value of the attribute - * @param em the EntityManager to query with + * @param em the EntityManager to query with * @return the queried SettingsParameter */ public static SettingsParameter getOrDefault(String attr, String defaultValue, EntityManager em) { SettingsParameter result = new SettingsParameter(attr, defaultValue, false); - TypedQuery q = em.createNamedQuery("SettingsParameter.getAttr", + TypedQuery q = em.createNamedQuery( + "SettingsParameter.getAttr", SettingsParameter.class); q.setParameter("attr", attr); @@ -331,7 +293,10 @@ public class SettingsUtil { result = q.getSingleResult(); // just in case if (result.getValue() == null || result.getValue().isEmpty()) { + em.getTransaction().begin(); result.setValue(defaultValue); + em.merge(result); + em.getTransaction().commit(); } } catch (NoResultException ex) { em.getTransaction().begin(); diff --git a/ace-am/src/main/webapp/settings.jsp b/ace-am/src/main/webapp/settings.jsp index 8b0c36351513381daa8523b9126d89e746e00202..948f6d32fb998e7791c91834b40631e2afdcc061 100644 --- a/ace-am/src/main/webapp/settings.jsp +++ b/ace-am/src/main/webapp/settings.jsp @@ -1,185 +1,306 @@ -<%-- +<%-- Document : settings Created on : Jul 30, 2012, 4:07:16 PM Author : shake - TODO: Enable sampling when we have it properly implemented TODO: The Settings should come from a bean or smth so we can loop similar to the custom settings --%> -<%@page contentType="text/html" pageEncoding="UTF-8"%> -<%@taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"%> +<%@page contentType="text/html" pageEncoding="UTF-8" %> +<%@taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c" %> +"http://www.w3.org/TR/html4/loose.dtd"> + - - - System Settings - - - - -

-
- -

System Settings

- -
-
Mail Server:
-
-
-
-
-
Mail From:
-
-
-
-
-
Max Audit:
-
-
-
-
-
Audit IMS Blocking:
-
-
-
-
-
Audit Max Block Time:
-
-
-
-
-
Audit Wait Time:
-
-
-
-
-
Audit BPS:
-
-
-
-
-
Enable Auto Audit:
-
-
-
-
-
Audit Only Mode:
-
-
-
-
-
IMS Host:
-
-
-
-
-
IMS Port:
-
-
-
-
-
IMS SSL:
-
-
-
-
-
IMS Token Class:
-
-
-
-
-
User Management:
-
-
-
-
-
Max Ingest Threads:
-
-
-
-
-
Log Location:
-
-
-
- - -
-
Log Type:
-
-
-
- - -
-
Log Type:
-
-
-
-
-
Log File Size:
-
-
-
-
-
Log Backup Index:
-
-
-
- - -
-
-
Generic logging properties, - change only if you know what you are doing
-
-
-
Root Logger:
-
-
-
-
Pattern Layout:
-
-
-
-
Conversion Pattern:
-
-
-
-
IRods:
-
-
-
-
UMIACS Connection:
-
-
-
-
IRods Connection:
-
+ + + System Settings + + + + + +
+ System Settings +
+ +
+ + +
+
+ + + Set this to true to disable internal user management. This should only be used in + conjunction with changes to the Authentication realm listed above. + + +
+
+ + + Mail server to use when mailing reports. + You will need to set this if you want reports to be mailed properly. + + +
+
+ + + Set this e-mail address to an address e-mail should originate from + + +
+
+ +
+
+ + Maximum number of running audits + +
+
+ + Minimum time between file reads in milliseconds + +
+
+ + + Maximum bytes per second per running audit: default = 0 = unlimited + + +
+
+ + Block audits when connectivity to the IMS is lost + +
+
+ + Maximum amount of retries when IMS failure occur + +
+
+ + + Amount of time to wait in milliseconds before retrying after an IMS failure + + +
+
+ + Enable automated auditing + +
+
+ + + If true, do not attempt to register new items or contact the IMS during auditing. + Default value of false is recommended. + + +
+
+ +
+
+ + + IMS hostname to use. Unless you deployed your own IMS, + this should probably not be changed. + + +
+
+ + + Port the IMS Host runs on. + Typically 80 for http or 443 for https. + + +
+
+ + Use SSL when connecting to the IMS + +
+
+ + Token class for the IMS host to use + +
+
+ + +
+
+ + The location of your logfile + +
+ + +
+ + + Log everything to one file, will grow to infinity + + +
+ + +
+ + + Use a rolling log file + + +
+
+ + + Maximum file size allowed for your log + + +
+
+ + + Maximum backup index allowed for your log + + +
+ + +
+
+
Generic logging properties, change only if you know what you are doing
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ +
+ + +
+ +
-
-
-
Custom Settings:
+ +
+
+ Add setting
- -
-
${item.key}
-
-
- -
-
Add setting
-
Delete settings
+
+ Delete settings
-
-
- - - Cancel - +
+ + + +
+
+ + +
+ + + + + diff --git a/ace-am/src/main/webapp/style.css b/ace-am/src/main/webapp/style.css index 7c3af0d7861e8056929d392b071d61e9bae1e029..9b62941a0980f517218489de65b311193479f8ca 100644 --- a/ace-am/src/main/webapp/style.css +++ b/ace-am/src/main/webapp/style.css @@ -31,7 +31,6 @@ input { font-size: 12px; } - /* * Taken from bootstrap. If we import a lot of rules break and we end up * with broken pages. The ui in general needs an overhaul but that's a lot @@ -284,7 +283,6 @@ vertical-align: top; .form-input { border: 1px solid #ccc; width: 100%; - /* height: 20px; */ padding: 3px 8px; margin-left: -1px; margin-right: 5px; @@ -297,12 +295,129 @@ vertical-align: top; margin-right: 5px; } +.form-input-settings { + width: 100%; + max-width: 100%; + border: 1px solid #363636; + border-radius: 3px; + height: 1.5rem; + padding: 0.275rem; + box-sizing: border-box; + box-shadow: inset 0 1px 2px rgba(10, 10, 10, 0.1); +} + +.form-input-settings:hover { + border-color: #076475; +} + +.form-input-settings:focus { + border-color: #5883C9; + box-shadow: 0 0 0.125em rgb(77, 144, 255); +} + +.form-legend { + font-size: 1.25rem; + font-weight: bolder; +} + +.form-field:not(:last-child) { + margin-bottom: .75rem; +} + +.form-label { + color: #363636; + display: block; + font-size: 1rem; + font-weight: 700; + text-align: center; +} + +.form-label:not(:last-child) { + margin-bottom: .5em; +} + +.form-help { + display: block; + font-size: 0.7rem; + margin-bottom: .25rem; + margin-top: 0.25rem; + text-align: center; +} + +.form-group { + margin-top: .625rem; + margin-bottom: .5rem; +} + .btn { padding: 2px; width: 75px; height: 25px; - border: 1px solid #e8e8e8; + border: 1px solid #939393; border-radius: 0; margin-left: 2px; margin-top: 2px; + color: #363636; + background-color: hsl(217, 100%, 82%); + font-size: 0.75rem; +} + +.btn.is-secondary { + color: #363636; + background-color: #fff; +} + +.btn.btn-danger { + color: #fff; + background-color: #dc3545; +} + +.btn:hover { + border-color: #363636; +} + +/** + Tabs are pretty much the same as bulma.io with some small changes to match ACE styling + */ +.tabs { + display: flex; + font-size: 1rem; + justify-content: space-between; + overflow: hidden; + overflow-x: auto; + white-space: nowrap; +} + +.tabs a { + align-items: center; + border-bottom: 1px solid #939393; + color: #363636; + display: flex; + justify-content: center; + margin-bottom: -1px; + padding: 0.5em 1em; + vertical-align: top; + cursor: pointer; +} + +.tabs li { + display: block; +} + +.tabs li.is-active a { + border-bottom-color: #3273dc; + color: #3273dc +} + +.tab-block { + display: none; +} + +.tabs ul { + align-items: center; + border-bottom: solid 1px #939393; + display: flex; + flex-grow: 1; + flex-shrink: 0; + justify-content: flex-start; } diff --git a/ace-common/src/main/java/edu/umiacs/util/Strings.java b/ace-common/src/main/java/edu/umiacs/util/Strings.java index 5d3b5b41fdecce3aae7a7654874247ae3396ab02..0efa4085dcb04d8085e80883c6cfef024e8a031c 100644 --- a/ace-common/src/main/java/edu/umiacs/util/Strings.java +++ b/ace-common/src/main/java/edu/umiacs/util/Strings.java @@ -116,7 +116,7 @@ public final class Strings } } - /** + /** * Determines if a string is a valid integer. * * @param val string value to check. @@ -138,6 +138,20 @@ public final class Strings } } + /** + * Determine if a string is a valid integer which is greater than or equal to 0. + * + * @param val string value to check + * @return true if {@code val} can successfully be parsed and is positive + */ + public static boolean isNonNegativeInt(Object val) { + try { + return isValidInt(val) && Integer.parseInt(val.toString()) >= 0; + } catch (NumberFormatException e) { + return false; + } + } + /** * Return exception and contained stack trace as string * diff --git a/ace-ims-api/pom.xml b/ace-ims-api/pom.xml index c4f1eef0e2e9d578023b13f3dd03864793983583..5b3955488b53e237e28808726ec2185a1e9a59db 100644 --- a/ace-ims-api/pom.xml +++ b/ace-ims-api/pom.xml @@ -5,7 +5,6 @@ ace edu.umiacs.ace 1.14-SNAPSHOT - ../pom.xml ace-ims-api ace-ims-api @@ -16,8 +15,6 @@ - - edu.umiacs.ace ace-ims-ws @@ -28,6 +25,5 @@ ace-common ${project.version} - diff --git a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSResult.java b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSResult.java new file mode 100644 index 0000000000000000000000000000000000000000..e5a63056b6c01606d11d58860fe272b1e126a9d0 --- /dev/null +++ b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSResult.java @@ -0,0 +1,78 @@ +package edu.umiacs.ace.ims.api; + +import edu.umiacs.ace.exception.StatusCode; +import edu.umiacs.ace.ims.ws.IMSFault; +import edu.umiacs.ace.ims.ws.IMSFault_Exception; + +import javax.xml.ws.WebServiceException; +import java.util.Optional; + +/** + * Encapsulate the result of communicating with the IMS. If a response is successful, store it as + * a field 'e', and in the result of failure store the exception. + * + * @author shake + * @since 1.13.1 + */ +public class IMSResult { + + private final E e; + private final Exception exception; + + public IMSResult(E e) { + this.e = e; + this.exception = null; + } + + public IMSResult(Exception exception) { + this.e = null; + this.exception = exception; + } + + public Optional getResult() { + return Optional.ofNullable(e); + } + + public Optional getException() { + return Optional.ofNullable(exception); + } + + /** + * Get the result of the IMS request, or throw an exception if no result exists. + * + * @return the response from the IMS, if successful + * @throws IMSException on the event of failure + */ + public E getOrThrow() { + if (e != null) { + return e; + } + + throw createException(); + } + + private IMSException createException() { + if (exception == null) { + String message = "Invalid state; createException called with exception == null"; + throw new RuntimeException(message); + } + + if (exception instanceof WebServiceException) { + Throwable cause = exception.getCause(); + if (cause instanceof java.net.ConnectException || + cause instanceof java.io.FileNotFoundException) { + return new IMSConnectionException(cause.getMessage(), cause); + } + + } + + if (exception instanceof IMSFault_Exception) { + IMSFault fault = ((IMSFault_Exception) exception).getFaultInfo(); + return new IMSException(fault.getStatusCode(), fault.getMessage()); + } + + throw new IMSException(StatusCode.CLIENT_ERROR, "Client error: " + + exception.getMessage(), exception); + } + +} diff --git a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSService.java b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSService.java index 30f2e362cc922eafc5b279214b52bb55194fe243..8c5d6dc5b50b84e69c2ae2d244c356d89244439d 100644 --- a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSService.java +++ b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSService.java @@ -42,339 +42,308 @@ import edu.umiacs.ace.ims.ws.TokenReceipt; import edu.umiacs.ace.ims.ws.TokenRequest; import edu.umiacs.ace.ims.ws.TokenResponse; import edu.umiacs.util.Check; +import org.apache.log4j.Logger; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; +import javax.xml.namespace.QName; +import javax.xml.ws.BindingProvider; +import javax.xml.ws.WebServiceException; import java.net.MalformedURLException; import java.net.URL; import java.security.MessageDigest; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; -import javax.xml.namespace.QName; -import javax.xml.ws.BindingProvider; -import javax.xml.ws.WebServiceException; +import java.util.function.Supplier; + +import static java.util.Collections.emptyList; /** - * * @author mmcgann */ -public final class IMSService -{ +public final class IMSService { + + private static final Logger print = Logger.getLogger(IMSService.class); + + private static final int DEFAULT_RETRY = 3; private static final int DEFAULT_PORT = 8080; + private static final int DEFAULT_TIMEOUT = 5000; private static final String DEFAULT_PATH = "/ace-ims/IMSWebService?wsdl"; private static final boolean DEFAULT_SSL = false; private static final boolean DEFAULT_BLOCKING = false; + private boolean blocking; - private long maxBlockTime; + private final int imsRetryAttempts; + private final int imsResetTimeout; private IMSWebService port; - - private IMSService(URL url, boolean blocking, int maxBlockTimeMinutes) { + + private IMSService(URL url, boolean blocking, int imsRetryAttempts, int imsResetTimeout) { this.blocking = blocking; - this.maxBlockTime = TimeUnit.MILLISECONDS.convert(maxBlockTimeMinutes, - TimeUnit.MINUTES); + this.imsRetryAttempts = imsRetryAttempts; + this.imsResetTimeout = imsResetTimeout; try { IMSWebService_Service service = new IMSWebService_Service(url, - new QName("http://ws.ims.ace.umiacs.edu/", "IMSWebService")); + new QName("http://ws.ims.ace.umiacs.edu/", "IMSWebService")); port = service.getIMSWebServicePort(); - ((BindingProvider)port).getRequestContext() + ((BindingProvider) port).getRequestContext() .put(BindingProvider.SESSION_MAINTAIN_PROPERTY, true); - } catch ( Exception e ) { + } catch (Exception e) { throw handleException(e); } } /** * Open a connection to the IMS - * - * @param hostName hostname of the ims, usually ims.umiacs.umd.edu + * + * @param hostName hostname of the ims, usually ims.umiacs.umd.edu * @param portNumber port number for the ims, usually 8080 - * @param path url path to the ims web service wsdl, usually /ace-ims/IMSWebService?wsdl - * @param ssl flag for connecting with ssl + * @param path url path to the ims web service wsdl, usually /ace-ims/IMSWebService?wsdl + * @param ssl flag for connecting with ssl * @return ims connection */ - public static IMSService connect(String hostName, - int portNumber, - String path, + public static IMSService connect(String hostName, + int portNumber, + String path, boolean ssl, boolean blocking, - int maxBlockTimeMinutes) { + int imsRetryAttempts, + int imsResetTimeout) { Check.notEmpty("hostName", hostName); Check.isPositive("portNumber", portNumber); - Check.notNegative("maxBlockTime", maxBlockTimeMinutes); + Check.notNegative("imsRetryAttempts", imsRetryAttempts); + Check.notNegative("imsResetTimeout", imsResetTimeout); Check.notEmpty("path", path); - - if ( !path.startsWith("/") ) { + + if (!path.startsWith("/")) { path = "/" + path; } StringBuilder url = new StringBuilder(); - if ( ssl ) { + if (ssl) { url.append("https://").append(hostName).append(":") - .append(portNumber).append(path); + .append(portNumber).append(path); - }else { + } else { url.append("http://").append(hostName).append(":") - .append(portNumber).append(path); + .append(portNumber).append(path); } + try { - return new IMSService(new URL(url.toString()), blocking, maxBlockTimeMinutes); - } catch ( MalformedURLException mue ) { + return new IMSService(new URL(url.toString()), blocking, imsRetryAttempts, imsResetTimeout); + } catch (MalformedURLException mue) { throw new IllegalArgumentException("Invalid URL: " + url); } } - + /** * Open a connection to the IMS on the specified host using port 8080 * and the default wsdl path - * + * * @param hostName IMS server * @return ims connection */ - public static IMSService connect(String hostName) - { - return IMSService.connect(hostName, - DEFAULT_PORT, - DEFAULT_PATH, - DEFAULT_SSL, - DEFAULT_BLOCKING, - 0); + public static IMSService connect(String hostName) { + return IMSService.connect(hostName, + DEFAULT_PORT, + DEFAULT_PATH, + DEFAULT_SSL, + DEFAULT_BLOCKING, + DEFAULT_RETRY, + DEFAULT_TIMEOUT); } - + public static IMSService connect(String hostName, int port) { - return IMSService.connect(hostName, port, DEFAULT_PATH, DEFAULT_SSL, DEFAULT_BLOCKING, 0); + return IMSService.connect(hostName, port, DEFAULT_PATH, DEFAULT_SSL, DEFAULT_BLOCKING, DEFAULT_RETRY, DEFAULT_TIMEOUT); } public static IMSService connect(String hostName, int port, boolean ssl) { - return IMSService.connect(hostName, port, DEFAULT_PATH, ssl, DEFAULT_BLOCKING, 0); + return IMSService.connect(hostName, port, DEFAULT_PATH, ssl, DEFAULT_BLOCKING, DEFAULT_RETRY, DEFAULT_TIMEOUT); } - public static IMSService connect(String hostName, int port, boolean ssl, boolean blocking, int maxBlockTimeMinutes) { - return IMSService.connect(hostName, port, DEFAULT_PATH, ssl, blocking, maxBlockTimeMinutes); + public static IMSService connect(String hostName, + int port, + boolean ssl, + boolean blocking, + int imsRetryAttempts, + int imsResetTimeout) { + return IMSService.connect(hostName, port, DEFAULT_PATH, ssl, blocking, imsRetryAttempts, imsResetTimeout); } - -// public TokenClass createTokenClass(TokenClass tokenClass) -// { -// try -// { -// return port.createTokenClass(tokenClass); -// } -// catch ( Exception e ) -// { -// throw handleException(e); -// } -// } - - public List requestWitnessProofForRounds(List rounds) - { - try - { - if ( blocking) { - Class c = port.getClass(); - Method reqWitnessProof = c.getMethod("requestWitnessProofForRounds", List.class); - return blockUntil(reqWitnessProof, rounds); - } else { - return port.createWitnessProofForRound(rounds); + + public List requestWitnessProofForRounds(List rounds) { + Supplier>> supplier = () -> { + try { + List witness = port.createWitnessProofForRound(rounds); + return new IMSResult<>(witness); + } catch (Exception e) { + return new IMSResult<>(e); } - } - catch ( Exception e ) - { - throw handleException(e); - } + }; + + return invoke(supplier); } - + /** - * Blocking call requesting the IMS close current round and return with + * Blocking call requesting the IMS close current round and return with * responses to the requested tokens. - * + * * @param tokenClassName name of token service to use, usually 'SHA-256-0' - * @param requests list of token requests - * + * @param requests list of token requests * @return list of token responses. */ - public List requestTokensImmediate(String tokenClassName, - List requests) - { - try - { - if ( blocking) { - Class c = port.getClass(); - Method reqTokensImmediate = c.getMethod("requestTokensImmediate", - String.class, - List.class); - return blockUntil(reqTokensImmediate, tokenClassName, requests); - } else { - return port.requestTokensImmediate(tokenClassName, requests); - } - } - catch ( Exception e ) - { - throw handleException(e); - } - } - - public List getRoundSummaries(List rounds) - { - try - { - if ( blocking) { - Class c = port.getClass(); - Method roundSummaries = c.getMethod("getRoundSummaries", List.class); - return blockUntil(roundSummaries, rounds); - } else { - return port.getRoundSummaries(rounds); + public List requestTokensImmediate(String tokenClassName, + List requests) { + Supplier>> supplier = () -> { + try { + List responses = port.requestTokensImmediate(tokenClassName, requests); + return new IMSResult<>(responses); + } catch (Exception e) { + return new IMSResult<>(e); } - } - catch ( Exception e ) - { - throw handleException(e); - } - } + }; - // TODO: This should really be in a circuit breaker type class - private E blockUntil(Method m, Object... args) throws Exception { - boolean done = false; - E fin = null; + return invoke(supplier); + } - long start = System.currentTimeMillis(); - long elapsed = 0; - while ( !done && elapsed <= maxBlockTime ) { + public List getRoundSummaries(List rounds) { + Supplier>> supplier = () -> { try { - // avoid sleeping on the first pass - if ( elapsed != 0 ) { - TimeUnit.MINUTES.sleep(15); - } - - fin = (E) m.invoke(port, args); - - done = true; + List summaries = port.getRoundSummaries(rounds); + return new IMSResult<>(summaries); } catch (Exception e) { - long now = System.currentTimeMillis(); - elapsed = now - start; - - // We want to catch when we simply could not connect to the IMS - // server then continue to block, or when java.lang.reflect catches - // an exception on the invocation of the method - if ( e instanceof WebServiceException ) { - Throwable cause = e.getCause(); - if ( cause instanceof java.net.ConnectException ) { - continue; - } - } else if ( e instanceof InvocationTargetException) { - // TODO: Properly catch java.net.ConnectException - // it's kind of tricky as the invocation exception makes - // the stack trace murky - continue; - } - throw handleException(e); + return new IMSResult<>(e); } + } ; - } - - // TODO: Throw timeout exception - - return fin; + return invoke(supplier); } public TokenReceipt requestTokensAsync(String tokenClassName, - List requests) - { - try - { - if ( blocking) { - Class c = port.getClass(); - Method m = c.getMethod("requestTokensAsync", String.class, List.class); - return blockUntil(m, tokenClassName, requests); - } else { - return port.requestTokensAsync(tokenClassName, requests); + List requests) { + Supplier> supplier = () -> { + try { + TokenReceipt receipt = port.requestTokensAsync(tokenClassName, requests); + return new IMSResult<>(receipt); + } catch (Exception e) { + return new IMSResult<>(e); } - } - catch ( Exception e ) - { - throw handleException(e); - } + }; + + return invoke(supplier); } - public List retrieveAsyncTokens(List receiptList) - { - if (receiptList == null || receiptList.isEmpty()) - { - return Collections.emptyList(); + public List retrieveAsyncTokens(List receiptList) { + if (receiptList == null || receiptList.isEmpty()) { + return emptyList(); } - List responses = new ArrayList(); + List responses = new ArrayList<>(); - try - { - for (TokenReceipt receipt : receiptList) - { + try { + for (TokenReceipt receipt : receiptList) { long sessionKey = receipt.getSessionKey(); long requestNumber = receipt.getRequestNumber(); List rResp = port.retrieveTokens(requestNumber, sessionKey); responses.addAll(rResp); } return responses; - - } catch (Exception e) - { + + } catch (Exception e) { throw handleException(e); } } - + /** - * Create a request batch based on the requestTokensImmediate call. - * + * Create a request batch based on the requestTokensImmediate call. + * + * @param identifier a unique identifier for tracking what the request batch is working on * @param tokenClassName token class name to use 'SHA-256-0' - * @param callback calback to handle token responses and errors + * @param callback callback to handle token responses and errors * @param maxQueueLength maximum queue size before requests will be forced - * @param maxWaitTime time to wait between sending requests - * + * @param maxWaitTime time to wait between sending requests * @return batched request */ - public TokenRequestBatch createImmediateTokenRequestBatch( - String tokenClassName, RequestBatchCallback callback, - int maxQueueLength, int maxWaitTime) - { + public TokenRequestBatch createImmediateTokenRequestBatch(String identifier, + String tokenClassName, + RequestBatchCallback callback, + int maxQueueLength, + int maxWaitTime) { Check.notEmpty("tokenClassName", tokenClassName); Check.notNull("callback", callback); Check.isPositive("maxQueueLength", maxQueueLength); Check.isPositive("maxWaitTime", maxWaitTime); - - return new ImmediateTokenRequestBatch(this, tokenClassName, callback, - maxQueueLength, maxWaitTime); + + return new ImmediateTokenRequestBatch(this, identifier, tokenClassName, callback, maxQueueLength, maxWaitTime); } - - public TokenValidator createTokenValidator(ValidationCallback callback, - int maxQueueLength, int maxWaitTime, MessageDigest digest) - { + + /** + * Create a {@link TokenValidator} + * + * @param identifier a unique identifier for tracking what the request batch is working on + * @param callback callback to handle token validation and errors + * @param maxQueueLength maximum queue size before requests will be forced + * @param maxWaitTime time to wait between sending requests + * @param digest type of {@link MessageDigest} used + * @return the {@link TokenValidator} + */ + public TokenValidator createTokenValidator(String identifier, + ValidationCallback callback, + int maxQueueLength, + int maxWaitTime, + MessageDigest digest) { Check.notNull("callback", callback); Check.isPositive("maxQueueLength", maxQueueLength); Check.isPositive("maxWaitTime", maxWaitTime); - - return new TokenValidator(this, callback, - maxWaitTime, maxQueueLength, digest); + + return new TokenValidator(this, identifier, callback, maxWaitTime, maxQueueLength, digest); + } + + private E invoke(Supplier> supplier) { + boolean done = false; + IMSResult imsResult = new IMSResult<>(new RuntimeException("IMS not contacted")); + + long attempt = 0; + + // first attempt always goes through (attempt == 0) + // when blocking, make sure that attempt is less than maxAttempts + while (!done && (attempt == 0 || (attempt <= imsRetryAttempts && blocking))) { + if (attempt > 0) { + try { + TimeUnit.MILLISECONDS.sleep(imsResetTimeout); + } catch (InterruptedException exception) { + // we were probably cancelled, get out of here now + IMSConnectionException imsException = new IMSConnectionException( + exception.getMessage(), + exception.getCause()); + imsResult = new IMSResult<>(imsException); + break; + } + } + + print.trace("Supplier: attempt = " + attempt + + ", maxBlock = " + imsRetryAttempts + + ", blocking = " + blocking); + imsResult = supplier.get(); + done = imsResult.getResult().isPresent(); + attempt++; + } + + return imsResult.getOrThrow(); } - - private IMSException handleException(Exception e) - { - if ( e instanceof WebServiceException ) - { + + private IMSException handleException(Exception e) { + if (e instanceof WebServiceException) { Throwable cause = e.getCause(); - if ( cause instanceof java.net.ConnectException || - cause instanceof java.io.FileNotFoundException ) - { + if (cause instanceof java.net.ConnectException || + cause instanceof java.io.FileNotFoundException) { return new IMSConnectionException(cause.getMessage(), cause); } - + } - if ( e instanceof IMSFault_Exception ) - { - IMSFault fault = ((IMSFault_Exception)e).getFaultInfo(); + if (e instanceof IMSFault_Exception) { + IMSFault fault = ((IMSFault_Exception) e).getFaultInfo(); return new IMSException(fault.getStatusCode(), fault.getMessage()); } - throw new IMSException(StatusCode.CLIENT_ERROR, "Client error: " + + throw new IMSException(StatusCode.CLIENT_ERROR, "Client error: " + e.getMessage(), e); } } diff --git a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/ImmediateTokenRequestBatch.java b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/ImmediateTokenRequestBatch.java index 7cb3183586647ef61e3af8041434b6ab713e8f8c..8a744a10356baa2fe1a0c40015bceed497220ead 100644 --- a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/ImmediateTokenRequestBatch.java +++ b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/ImmediateTokenRequestBatch.java @@ -34,6 +34,9 @@ package edu.umiacs.ace.ims.api; import edu.umiacs.ace.ims.ws.TokenRequest; import edu.umiacs.ace.ims.ws.TokenResponse; import edu.umiacs.util.Check; +import org.apache.log4j.Logger; +import org.apache.log4j.NDC; + import java.util.ArrayList; import java.util.Date; import java.util.LinkedList; @@ -41,34 +44,39 @@ import java.util.List; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import org.apache.log4j.Logger; -import org.apache.log4j.NDC; /** + * Thread which uses the {@link IMSService} to call + * {@link IMSService#requestTokensImmediate(String, List)} when processing ACE Collections which + * need to receive ACE Tokens for files. * * @author mmcgann */ -class ImmediateTokenRequestBatch extends Thread implements TokenRequestBatch -{ - - //private static final int MAXQUEUESIZE = 10 - private IMSService service; - private LinkedList requests = new LinkedList(); - private String tokenClassName; - private int maxWaitTime; - private int maxQueueLength; +class ImmediateTokenRequestBatch extends Thread implements TokenRequestBatch { + + private static final Logger print = Logger.getLogger(ImmediateTokenRequestBatch.class); + private RequestBatchCallback callback; - private boolean shutdownRequested = false; private boolean processNow = false; - private Lock lock = new ReentrantLock(); - private Condition processCondition; - private static final Logger print = - Logger.getLogger(ImmediateTokenRequestBatch.class); - - ImmediateTokenRequestBatch(IMSService service, String tokenClassName, - RequestBatchCallback callback, int maxQueueLength, int maxWaitTime) - { - this.service = service; + private boolean shutdownRequested = false; + private LinkedList requests = new LinkedList<>(); + + private final IMSService service; + private final String identifier; + private final String tokenClassName; + private final int maxWaitTime; + private final int maxQueueLength; + private final Lock lock = new ReentrantLock(); + private final Condition processCondition; + + ImmediateTokenRequestBatch(IMSService imsService, + String identifier, + String tokenClassName, + RequestBatchCallback callback, + int maxQueueLength, + int maxWaitTime) { + this.service = imsService; + this.identifier = identifier; this.tokenClassName = tokenClassName; this.callback = callback; this.maxQueueLength = maxQueueLength; @@ -77,159 +85,158 @@ class ImmediateTokenRequestBatch extends Thread implements TokenRequestBatch this.start(); } - public void add(TokenRequest request) throws InterruptedException - { + /** + * Add a {@link TokenRequest} to the {@code requests} list so that an ACE Token can be + * requested from the ACE IMS + *

+ * This method will block when trying to acquire a lock from the main thread. This can only + * happen IFF a batch of requests is not already being processed. If the {@code requests} has a + * size greater than or equal to the {@code maxQueueLength} after adding the + * {@link TokenRequest}, this will wake up the main thread in order to process the current batch + * ({@code requests}). + * + * @param request the {@link TokenRequest} to create an ACE Token for + * @throws InterruptedException if the thread is interrupted + */ + public void add(TokenRequest request) throws InterruptedException { Check.notNull("request", request); lock.lockInterruptibly(); - try - { - if ( shutdownRequested ) - { + try { + if (shutdownRequested) { throw new IllegalStateException("Process shutdown"); } requests.offer(request); - if ( requests.size() >= maxQueueLength ) - { + if (requests.size() >= maxQueueLength) { processCondition.signal(); } - } - finally - { + } finally { lock.unlock(); } } - public void close() - { + /** + * Signal that the {@link ImmediateTokenRequestBatch} should be shut down. Blocks until the + * lock can be acquired at which point signal the {@link Condition}. Once the final batch has + * been run, block again on {@link #join()}. + */ + public void close() { lock.lock(); - try - { + try { shutdownRequested = true; - print.info("Shutdown requested"); + print.info("Shutdown requested on Token Request"); processCondition.signal(); - } - finally - { + } finally { lock.unlock(); } - try - { + try { this.join(); - } - catch ( InterruptedException ie ) - { + } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } } - private boolean isBatchReady() - { + /** + * Check if a batch is ready + * * requests is non empty AND + * * requests.size is greater than equal to maxQueueLength OR + * * processNow flag is set (the processCondition deadline elapsed) OR + * * shutdownRequested flag is set (close called) + * + * @return true if a batch should be processed, false otherwise + */ + private boolean isBatchReady() { lock.lock(); - try - { + try { return requests.size() > 0 && (requests.size() >= maxQueueLength || - processNow || - shutdownRequested); - } - finally - { + processNow || + shutdownRequested); + } finally { lock.unlock(); } } - private void processBatch() - { + /** + * Process the {@code requests} present and receive {@link TokenResponse}s from the IMS for + * each request. This will acquire a lock while filling up a local queue of items to process, + * so any call to {@link #add(TokenRequest)} on {@code requests} will block until + * {@code requests} has been drained. Once complete, the lock is released and + * {@link TokenRequest}s can continue to be added. + */ + private void processBatch() { List batch; lock.lock(); - try - { - batch = new ArrayList(maxQueueLength); + try { + batch = new ArrayList<>(maxQueueLength); int numAdded = 0; - while ( numAdded < maxQueueLength && !requests.isEmpty() ) - { + while (numAdded < maxQueueLength && !requests.isEmpty()) { batch.add(requests.poll()); } - } - finally - { + } finally { lock.unlock(); } - try - { + try { print.info("Sending batch: " + batch.size() + " requests"); List responses = service.requestTokensImmediate(tokenClassName, batch); callback.tokensReceived(batch, responses); - } - catch ( Exception e ) - { + } catch (Exception e) { print.error("Exception on send: " + e.getMessage(), e); callback.exceptionThrown(batch, e); } } + /** + * Entry point for the main thread. Runs until {@link #close()} is called or an + * {@link Exception} is thrown while running {@link #processBatch()} or + * {@link Condition#awaitUntil(Date)}. + * + * This will attempt to acquire the {@code lock} so that it can block for {@code minWaitTime}. + * While it is calling {@link Condition#awaitUntil(Date)}, the lock can be reacquired by other + * treads. This allows {@link #add(TokenRequest)} to be used in order to fill the + * {@code requests} list for processing. + */ @Override - public void run() - { - NDC.push("Request Thread: "); + public void run() { + NDC.push("Request Thread (" + identifier + "): "); print.info("Started"); -// lock.lock(); - try - { - while ( true ) - { + try { + while (true) { print.info("Checking batch"); - while ( isBatchReady() ) - { + while (isBatchReady()) { print.info("Processing batch"); processBatch(); } - if ( shutdownRequested ) - { + if (shutdownRequested) { print.info("Shutdown acknowledged"); break; } + lock.lock(); - try - { - Date deadline = new Date(System.currentTimeMillis() + - maxWaitTime); - try - { - print.info("Waiting until: " + deadline); - processNow = !processCondition.awaitUntil(deadline); - } - catch ( InterruptedException ie ) - { - Thread.currentThread().interrupt(); - print.info("Interrupted"); - break; - } - } - finally - { + Date deadline = new Date(System.currentTimeMillis() + maxWaitTime); + try { + print.info("Waiting until: " + deadline); + processNow = !processCondition.awaitUntil(deadline); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + print.info("Interrupted"); + break; + } finally { lock.unlock(); } } - } - catch ( Exception e ) - { + } catch (Exception e) { callback.unexpectedException(e); } finally { print.info("Stopped"); NDC.pop(); NDC.remove(); } -// finally -// { -// lock.unlock(); -// } } } diff --git a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/TokenValidator.java b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/TokenValidator.java index 66c815acc8c4c34b61a75d40f386cfff5f1fccbf..0958b6f346d4435bb4b5eb4dd207c67e81aa07a8 100644 --- a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/TokenValidator.java +++ b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/TokenValidator.java @@ -31,12 +31,16 @@ package edu.umiacs.ace.ims.api; +import edu.umiacs.ace.hashtree.Proof; import edu.umiacs.ace.hashtree.ProofValidator; import edu.umiacs.ace.ims.ws.ProofElement; import edu.umiacs.ace.ims.ws.RoundSummary; import edu.umiacs.ace.token.AceToken; import edu.umiacs.ace.util.HashValue; import edu.umiacs.util.Check; +import org.apache.log4j.Logger; +import org.apache.log4j.NDC; + import java.security.MessageDigest; import java.util.ArrayList; import java.util.Date; @@ -47,32 +51,39 @@ import java.util.Map; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import org.apache.log4j.Logger; -import org.apache.log4j.NDC; /** + * Validate ACE Tokens using {@link IMSService#getRoundSummaries(List)} for the {@link AceToken} + * rounds which we have. + * + * Note: This follows the same type of processing as {@link ImmediateTokenRequestBatch}, which + * can be referenced for documentation. * * @author toaster */ -public class TokenValidator extends Thread -{ +public class TokenValidator extends Thread { - private Map requests = new HashMap(); - private IMSService connection; + private static final Logger print = Logger.getLogger(TokenValidator.class); + + private Map requests = new HashMap<>(); private ValidationCallback callback; - private static final Logger print = - Logger.getLogger(TokenValidator.class); private boolean shutdownRequested = false; private boolean processNow = false; - private int maxWaitTime; - private int maxQueueLength; - private Lock lock = new ReentrantLock(); - private Condition processCondition; - private MessageDigest digest; - - TokenValidator(IMSService connection, ValidationCallback callback, - int maxWaitTime, int maxQueueLength, MessageDigest digest) - { + + private final IMSService connection; + private final String identifier; + private final int maxWaitTime; + private final int maxQueueLength; + private final Lock lock = new ReentrantLock(); + private final Condition processCondition; + private final MessageDigest digest; + + TokenValidator(IMSService connection, + String identifier, + ValidationCallback callback, + int maxWaitTime, + int maxQueueLength, + MessageDigest digest) { Check.notNull("connection", connection); Check.notNull("callback", callback); Check.notNull("digest", digest); @@ -81,91 +92,53 @@ public class TokenValidator extends Thread this.maxQueueLength = maxQueueLength; this.maxWaitTime = maxWaitTime; this.digest = digest; + this.identifier = identifier; processCondition = lock.newCondition(); this.start(); } -// public void add(String fileHash, AceToken token) throws InterruptedException -// { -// Check.notNull("token", token); -// Check.notNull("fileHash", fileHash); -// -// lock.lockInterruptibly(); -// try -// { -// if ( shutdownRequested ) -// { -// throw new IllegalStateException("Process shutdown"); -// } -// print.trace("Adding work: " + fileHash); -// requests.put(IMSUtil.convertToken(token), fileHash); -// if ( requests.size() >= maxQueueLength ) -// { -// processCondition.signal(); -// } -// } -// finally -// { -// lock.unlock(); -// } -// } - - public void add(String fileHash, AceToken token) throws InterruptedException - { + public void add(String fileHash, AceToken token) throws InterruptedException { Check.notNull("token", token); Check.notNull("fileHash", fileHash); lock.lockInterruptibly(); - try - { - if ( shutdownRequested ) - { + try { + if (shutdownRequested) { throw new IllegalStateException("Process shutdown"); } print.trace("Adding work: " + fileHash); requests.put(token, fileHash); - if ( requests.size() >= maxQueueLength ) - { + if (requests.size() >= maxQueueLength) { processCondition.signal(); } - } - finally - { + } finally { lock.unlock(); } } - public void close() - { + public void close() { lock.lock(); - try - { + try { shutdownRequested = true; - print.info("Shutdown requested"); + print.info("Shutdown requested for validator"); processCondition.signal(); - } - finally - { + } finally { lock.unlock(); } - try - { + try { this.join(); - } - catch ( InterruptedException ie ) - { + } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } } - private boolean isBatchReady() - { + private boolean isBatchReady() { return requests.size() > 0 && (requests.size() >= maxQueueLength || - processNow || - shutdownRequested); + processNow || + shutdownRequested); } /** @@ -174,26 +147,22 @@ public class TokenValidator extends Thread * 3. For each token, check to ensure it validates * 4. call appropriate callback for each token */ - private void processBatch() - { - List roundNumbers = new ArrayList(); + private void processBatch() { + List roundNumbers = new ArrayList<>(); ProofValidator pv = new ProofValidator(); - Map> proofMap = new HashMap>(); + Map> proofMap = new HashMap<>(); lock.lock(); - try - { + try { print.trace("Processing batch of size: " + requests.size()); - for ( AceToken token : requests.keySet() ) - { + for (AceToken token : requests.keySet()) { String hash = requests.get(token); WorkUnit unit = new WorkUnit(); unit.setHash(hash); unit.setTokenResponse(token); - if ( !roundNumbers.contains(token.getRound()) ) - { + if (!roundNumbers.contains(token.getRound())) { roundNumbers.add(token.getRound()); proofMap.put(token.getRound(), new LinkedList()); @@ -204,9 +173,7 @@ public class TokenValidator extends Thread print.trace("batch load finished, calling clear, unlocking."); requests.clear(); print.trace("requests size: " + requests.size()); - } - finally - { + } finally { lock.unlock(); } @@ -214,29 +181,24 @@ public class TokenValidator extends Thread List summaries = connection.getRoundSummaries(roundNumbers); print.trace("IMS returned " + summaries.size() + " rounds"); - for ( RoundSummary summary : summaries ) - { + for (RoundSummary summary : summaries) { long round = summary.getId(); - for ( WorkUnit unit : proofMap.get(round) ) - { + for (WorkUnit unit : proofMap.get(round)) { AceToken response = unit.getTokenResponse(); String localLeafHash = unit.getHash(); String imsSuppliedHash = summary.getHashValue(); -// String calculatedHash = calculateRoot(digest, -// localLeafHash, response.getProofElements()); - String calculatedHash = HashValue.asHexString(pv.rootHash(digest, response.getProof(), HashValue.asBytes(localLeafHash))); + Proof responseProof = response.getProof(); + byte[] localHash = HashValue.asBytes(localLeafHash); + byte[] rootHash = pv.rootHash(digest, responseProof, localHash); + String calculatedHash = HashValue.asHexString(rootHash); - if ( imsSuppliedHash.equals(calculatedHash) ) - { + if (imsSuppliedHash.equals(calculatedHash)) { callback.validToken(response); - } - else - { - callback.invalidToken(response, imsSuppliedHash, - calculatedHash); + } else { + callback.invalidToken(response, imsSuppliedHash, calculatedHash); } } @@ -248,51 +210,38 @@ public class TokenValidator extends Thread } @Override - public void run() - { + public void run() { - NDC.push("Validation Thread: "); + NDC.push("Validation Thread (" + identifier + "): "); print.info("Started"); lock.lock(); - try - { + try { - while ( true ) - { + while (true) { print.info("Checking batch"); - while ( isBatchReady() ) - { + while (isBatchReady()) { print.info("Processing batch, queued items: " + requests.size()); processBatch(); } - if ( shutdownRequested ) - { + if (shutdownRequested) { print.info("Shutdown acknowledged"); break; } - Date deadline = new Date(System.currentTimeMillis() - + maxWaitTime); - try - { + Date deadline = new Date(System.currentTimeMillis() + maxWaitTime); + try { print.info("Waiting until: " + deadline); processNow = !processCondition.awaitUntil(deadline); - } - catch ( InterruptedException ie ) - { + } catch (InterruptedException ie) { Thread.currentThread().interrupt(); print.info("Interrupted"); break; } } - } - catch ( Throwable e ) - { + } catch (Throwable e) { callback.unexpectedException(e); - } - finally - { + } finally { lock.unlock(); print.info("Stopped"); NDC.pop(); @@ -302,87 +251,71 @@ public class TokenValidator extends Thread /** * Calculate the root hash given the leaf, local hash and a list of proof elements - * proof calculation starts at elements[0] . The element.getIndex() position is + * proof calculation starts at elements[0] . The element.getIndex() position is * where the localHash, or hash from lower tree level is inserted. The * hashes for each element are run through the supplied digest. the elements[n] * value is what eventually gets returned. Please note digest will be reset prior * to computation start - * - * @param digest algorithm to use + * + * @param digest algorithm to use * @param localHash leaf hash to start tree bottom w/ - * @param elements levels proof tree w/ idx 0 the leaf and idx n root -1 + * @param elements levels proof tree w/ idx 0 the leaf and idx n root -1 * @return */ public static String calculateRoot(MessageDigest digest, String localHash, - List elements) - { + List elements) { // previous from lower level in tree is inserted at element.getIndex() position byte[] currentHash = HashValue.asBytes(localHash); digest.reset(); - for ( ProofElement element : elements ) - { + for (ProofElement element : elements) { digest.reset(); int index = element.getIndex(); int i = 0; -// System.out.println("index: " + element.getIndex()); - for ( String hash : element.getHashes() ) - { + for (String hash : element.getHashes()) { byte[] hashBytes = HashValue.asBytes(hash); // it element index is the current index, insert then continue on - if ( i == index ) - { -// System.out.println("prev hash: " + i + " " + HashValue.asHexString(currentHash)); + if (i == index) { digest.update(currentHash); i++; } -// System.out.println("elem hash: " + i + " " + hash); digest.update(hashBytes); i++; } // end case, if index is last element in list, then need to append - if ( index == i ) - { -// System.out.println("prev hash: " + i + " " + HashValue.asHexString(currentHash)); + if (index == i) { digest.update(currentHash); } currentHash = digest.digest(); -// System.out.println("Calc ihv: " + HashValue.asHexString(currentHash)); } return HashValue.asHexString(currentHash); } - private class WorkUnit - { + private class WorkUnit { private String hash; private AceToken tokenResponse; - public WorkUnit() - { + public WorkUnit() { } - public void setTokenResponse(AceToken tokenResponse) - { + public void setTokenResponse(AceToken tokenResponse) { this.tokenResponse = tokenResponse; } - public AceToken getTokenResponse() - { + public AceToken getTokenResponse() { return tokenResponse; } - public void setHash(String hash) - { + public void setHash(String hash) { this.hash = hash; } - public String getHash() - { + public String getHash() { return hash; } } diff --git a/pom.xml b/pom.xml index 66f4fbad06bad85073baefcc6f1ad63141287b1d..92001fbb9a2e13cf2adf765c3902ba5b04b4c366 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ 1.14-SNAPSHOT pom - scm:svn:https://subversion.umiacs.umd.edu/ace/tags/ace-1.9 + scm:git:https://gitlab.umiacs.umd.edu/adapt/ace.git ace @@ -19,10 +19,10 @@ org.apache.maven.plugins maven-compiler-plugin - 2.0.2 + 3.8.0 - 1.7 - 1.7 + 1.8 + 1.8