From dfe6647b2ebf2f6fd48567a9d5ad0f2941be4d2e Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Mon, 8 Apr 2019 12:04:50 -0400 Subject: [PATCH 01/49] P U R E L I N T --- .../edu/umiacs/ace/ims/api/IMSService.java | 236 ++++++++---------- .../ims/api/ImmediateTokenRequestBatch.java | 127 ++++------ 2 files changed, 145 insertions(+), 218 deletions(-) diff --git a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSService.java b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSService.java index 30f2e36..04d2600 100644 --- a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSService.java +++ b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSService.java @@ -43,6 +43,9 @@ import edu.umiacs.ace.ims.ws.TokenRequest; import edu.umiacs.ace.ims.ws.TokenResponse; import edu.umiacs.util.Check; +import javax.xml.namespace.QName; +import javax.xml.ws.BindingProvider; +import javax.xml.ws.WebServiceException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.MalformedURLException; @@ -52,16 +55,11 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; -import javax.xml.namespace.QName; -import javax.xml.ws.BindingProvider; -import javax.xml.ws.WebServiceException; /** - * * @author mmcgann */ -public final class IMSService -{ +public final class IMSService { private static final int DEFAULT_PORT = 8080; private static final String DEFAULT_PATH = "/ace-ims/IMSWebService?wsdl"; private static final boolean DEFAULT_SSL = false; @@ -70,35 +68,35 @@ public final class IMSService private long maxBlockTime; private IMSWebService port; - + private IMSService(URL url, boolean blocking, int maxBlockTimeMinutes) { this.blocking = blocking; this.maxBlockTime = TimeUnit.MILLISECONDS.convert(maxBlockTimeMinutes, - TimeUnit.MINUTES); + TimeUnit.MINUTES); try { IMSWebService_Service service = new IMSWebService_Service(url, - new QName("http://ws.ims.ace.umiacs.edu/", "IMSWebService")); + new QName("http://ws.ims.ace.umiacs.edu/", "IMSWebService")); port = service.getIMSWebServicePort(); - ((BindingProvider)port).getRequestContext() + ((BindingProvider) port).getRequestContext() .put(BindingProvider.SESSION_MAINTAIN_PROPERTY, true); - } catch ( Exception e ) { + } catch (Exception e) { throw handleException(e); } } /** * Open a connection to the IMS - * - * @param hostName hostname of the ims, usually ims.umiacs.umd.edu + * + * @param hostName hostname of the ims, usually ims.umiacs.umd.edu * @param portNumber port number for the ims, usually 8080 - * @param path url path to the ims web service wsdl, usually /ace-ims/IMSWebService?wsdl - * @param ssl flag for connecting with ssl + * @param path url path to the ims web service wsdl, usually /ace-ims/IMSWebService?wsdl + * @param ssl flag for connecting with ssl * @return ims connection */ - public static IMSService connect(String hostName, - int portNumber, - String path, + public static IMSService connect(String hostName, + int portNumber, + String path, boolean ssl, boolean blocking, int maxBlockTimeMinutes) { @@ -106,43 +104,42 @@ public final class IMSService Check.isPositive("portNumber", portNumber); Check.notNegative("maxBlockTime", maxBlockTimeMinutes); Check.notEmpty("path", path); - - if ( !path.startsWith("/") ) { + + if (!path.startsWith("/")) { path = "/" + path; } StringBuilder url = new StringBuilder(); - if ( ssl ) { + if (ssl) { url.append("https://").append(hostName).append(":") - .append(portNumber).append(path); + .append(portNumber).append(path); - }else { + } else { url.append("http://").append(hostName).append(":") - .append(portNumber).append(path); + .append(portNumber).append(path); } try { return new IMSService(new URL(url.toString()), blocking, maxBlockTimeMinutes); - } catch ( MalformedURLException mue ) { + } catch (MalformedURLException mue) { throw new IllegalArgumentException("Invalid URL: " + url); } } - + /** * Open a connection to the IMS on the specified host using port 8080 * and the default wsdl path - * + * * @param hostName IMS server * @return ims connection */ - public static IMSService connect(String hostName) - { - return IMSService.connect(hostName, - DEFAULT_PORT, - DEFAULT_PATH, - DEFAULT_SSL, - DEFAULT_BLOCKING, - 0); + public static IMSService connect(String hostName) { + return IMSService.connect(hostName, + DEFAULT_PORT, + DEFAULT_PATH, + DEFAULT_SSL, + DEFAULT_BLOCKING, + 0); } - + public static IMSService connect(String hostName, int port) { return IMSService.connect(hostName, port, DEFAULT_PATH, DEFAULT_SSL, DEFAULT_BLOCKING, 0); } @@ -154,97 +151,73 @@ public final class IMSService public static IMSService connect(String hostName, int port, boolean ssl, boolean blocking, int maxBlockTimeMinutes) { return IMSService.connect(hostName, port, DEFAULT_PATH, ssl, blocking, maxBlockTimeMinutes); } - -// public TokenClass createTokenClass(TokenClass tokenClass) -// { -// try -// { -// return port.createTokenClass(tokenClass); -// } -// catch ( Exception e ) -// { -// throw handleException(e); -// } -// } - - public List requestWitnessProofForRounds(List rounds) - { - try - { - if ( blocking) { + + public List requestWitnessProofForRounds(List rounds) { + try { + if (blocking) { Class c = port.getClass(); Method reqWitnessProof = c.getMethod("requestWitnessProofForRounds", List.class); return blockUntil(reqWitnessProof, rounds); } else { return port.createWitnessProofForRound(rounds); } - } - catch ( Exception e ) - { + } catch (Exception e) { throw handleException(e); } } - + /** - * Blocking call requesting the IMS close current round and return with + * Blocking call requesting the IMS close current round and return with * responses to the requested tokens. - * + * * @param tokenClassName name of token service to use, usually 'SHA-256-0' - * @param requests list of token requests - * + * @param requests list of token requests * @return list of token responses. */ - public List requestTokensImmediate(String tokenClassName, - List requests) - { - try - { - if ( blocking) { + public List requestTokensImmediate(String tokenClassName, + List requests) { + try { + if (blocking) { Class c = port.getClass(); Method reqTokensImmediate = c.getMethod("requestTokensImmediate", - String.class, - List.class); + String.class, + List.class); return blockUntil(reqTokensImmediate, tokenClassName, requests); } else { return port.requestTokensImmediate(tokenClassName, requests); } - } - catch ( Exception e ) - { + } catch (Exception e) { throw handleException(e); } } - - public List getRoundSummaries(List rounds) - { - try - { - if ( blocking) { + + public List getRoundSummaries(List rounds) { + try { + if (blocking) { Class c = port.getClass(); Method roundSummaries = c.getMethod("getRoundSummaries", List.class); return blockUntil(roundSummaries, rounds); } else { return port.getRoundSummaries(rounds); } - } - catch ( Exception e ) - { + } catch (Exception e) { throw handleException(e); } } // TODO: This should really be in a circuit breaker type class + // maybe switch from maxBlockTime to maxAttempts private E blockUntil(Method m, Object... args) throws Exception { boolean done = false; E fin = null; long start = System.currentTimeMillis(); long elapsed = 0; - while ( !done && elapsed <= maxBlockTime ) { + while (!done && elapsed <= maxBlockTime) { try { // avoid sleeping on the first pass - if ( elapsed != 0 ) { - TimeUnit.MINUTES.sleep(15); + if (elapsed != 0) { + TimeUnit.MINUTES.sleep(1); } fin = (E) m.invoke(port, args); @@ -257,12 +230,12 @@ public final class IMSService // We want to catch when we simply could not connect to the IMS // server then continue to block, or when java.lang.reflect catches // an exception on the invocation of the method - if ( e instanceof WebServiceException ) { + if (e instanceof WebServiceException) { Throwable cause = e.getCause(); - if ( cause instanceof java.net.ConnectException ) { + if (cause instanceof java.net.ConnectException) { continue; } - } else if ( e instanceof InvocationTargetException) { + } else if (e instanceof InvocationTargetException) { // TODO: Properly catch java.net.ConnectException // it's kind of tricky as the invocation exception makes // the stack trace murky @@ -279,102 +252,89 @@ public final class IMSService } public TokenReceipt requestTokensAsync(String tokenClassName, - List requests) - { - try - { - if ( blocking) { + List requests) { + try { + if (blocking) { Class c = port.getClass(); Method m = c.getMethod("requestTokensAsync", String.class, List.class); return blockUntil(m, tokenClassName, requests); } else { return port.requestTokensAsync(tokenClassName, requests); } - } - catch ( Exception e ) - { + } catch (Exception e) { throw handleException(e); } } - public List retrieveAsyncTokens(List receiptList) - { - if (receiptList == null || receiptList.isEmpty()) - { + public List retrieveAsyncTokens(List receiptList) { + if (receiptList == null || receiptList.isEmpty()) { return Collections.emptyList(); } List responses = new ArrayList(); - try - { - for (TokenReceipt receipt : receiptList) - { + try { + for (TokenReceipt receipt : receiptList) { long sessionKey = receipt.getSessionKey(); long requestNumber = receipt.getRequestNumber(); List rResp = port.retrieveTokens(requestNumber, sessionKey); responses.addAll(rResp); } return responses; - - } catch (Exception e) - { + + } catch (Exception e) { throw handleException(e); } } - + /** - * Create a request batch based on the requestTokensImmediate call. - * + * Create a request batch based on the requestTokensImmediate call. + * * @param tokenClassName token class name to use 'SHA-256-0' - * @param callback calback to handle token responses and errors + * @param callback calback to handle token responses and errors * @param maxQueueLength maximum queue size before requests will be forced - * @param maxWaitTime time to wait between sending requests - * + * @param maxWaitTime time to wait between sending requests * @return batched request */ - public TokenRequestBatch createImmediateTokenRequestBatch( - String tokenClassName, RequestBatchCallback callback, - int maxQueueLength, int maxWaitTime) - { + public TokenRequestBatch createImmediateTokenRequestBatch(String tokenClassName, + RequestBatchCallback callback, + int maxQueueLength, + int maxWaitTime) { Check.notEmpty("tokenClassName", tokenClassName); Check.notNull("callback", callback); Check.isPositive("maxQueueLength", maxQueueLength); Check.isPositive("maxWaitTime", maxWaitTime); - - return new ImmediateTokenRequestBatch(this, tokenClassName, callback, + + return new ImmediateTokenRequestBatch(this, tokenClassName, callback, maxQueueLength, maxWaitTime); } - - public TokenValidator createTokenValidator(ValidationCallback callback, - int maxQueueLength, int maxWaitTime, MessageDigest digest) - { + + public TokenValidator createTokenValidator(ValidationCallback callback, + int maxQueueLength, + int maxWaitTime, + MessageDigest digest) { Check.notNull("callback", callback); Check.isPositive("maxQueueLength", maxQueueLength); Check.isPositive("maxWaitTime", maxWaitTime); - - return new TokenValidator(this, callback, + + return new TokenValidator(this, callback, maxWaitTime, maxQueueLength, digest); } - - private IMSException handleException(Exception e) - { - if ( e instanceof WebServiceException ) - { + + private IMSException handleException(Exception e) { + if (e instanceof WebServiceException) { Throwable cause = e.getCause(); - if ( cause instanceof java.net.ConnectException || - cause instanceof java.io.FileNotFoundException ) - { + if (cause instanceof java.net.ConnectException || + cause instanceof java.io.FileNotFoundException) { return new IMSConnectionException(cause.getMessage(), cause); } - + } - if ( e instanceof IMSFault_Exception ) - { - IMSFault fault = ((IMSFault_Exception)e).getFaultInfo(); + if (e instanceof IMSFault_Exception) { + IMSFault fault = ((IMSFault_Exception) e).getFaultInfo(); return new IMSException(fault.getStatusCode(), fault.getMessage()); } - throw new IMSException(StatusCode.CLIENT_ERROR, "Client error: " + + throw new IMSException(StatusCode.CLIENT_ERROR, "Client error: " + e.getMessage(), e); } } diff --git a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/ImmediateTokenRequestBatch.java b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/ImmediateTokenRequestBatch.java index 7cb3183..54e9666 100644 --- a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/ImmediateTokenRequestBatch.java +++ b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/ImmediateTokenRequestBatch.java @@ -34,6 +34,9 @@ package edu.umiacs.ace.ims.api; import edu.umiacs.ace.ims.ws.TokenRequest; import edu.umiacs.ace.ims.ws.TokenResponse; import edu.umiacs.util.Check; +import org.apache.log4j.Logger; +import org.apache.log4j.NDC; + import java.util.ArrayList; import java.util.Date; import java.util.LinkedList; @@ -41,19 +44,21 @@ import java.util.List; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import org.apache.log4j.Logger; -import org.apache.log4j.NDC; /** + * Thread which uses the {@link IMSService} to call `requestTokensImmediate` when processing + * ACE Collections which need to receive ACE Tokens for files. * * @author mmcgann */ -class ImmediateTokenRequestBatch extends Thread implements TokenRequestBatch -{ +class ImmediateTokenRequestBatch extends Thread implements TokenRequestBatch { + + private static final Logger print = Logger.getLogger(ImmediateTokenRequestBatch.class); //private static final int MAXQUEUESIZE = 10 + private String identifier; private IMSService service; - private LinkedList requests = new LinkedList(); + private LinkedList requests = new LinkedList<>(); private String tokenClassName; private int maxWaitTime; private int maxQueueLength; @@ -62,12 +67,12 @@ class ImmediateTokenRequestBatch extends Thread implements TokenRequestBatch private boolean processNow = false; private Lock lock = new ReentrantLock(); private Condition processCondition; - private static final Logger print = - Logger.getLogger(ImmediateTokenRequestBatch.class); - ImmediateTokenRequestBatch(IMSService service, String tokenClassName, - RequestBatchCallback callback, int maxQueueLength, int maxWaitTime) - { + ImmediateTokenRequestBatch(IMSService service, + String tokenClassName, + RequestBatchCallback callback, + int maxQueueLength, + int maxWaitTime) { this.service = service; this.tokenClassName = tokenClassName; this.callback = callback; @@ -77,149 +82,111 @@ class ImmediateTokenRequestBatch extends Thread implements TokenRequestBatch this.start(); } - public void add(TokenRequest request) throws InterruptedException - { + public void add(TokenRequest request) throws InterruptedException { Check.notNull("request", request); lock.lockInterruptibly(); - try - { - if ( shutdownRequested ) - { + try { + if (shutdownRequested) { throw new IllegalStateException("Process shutdown"); } requests.offer(request); - if ( requests.size() >= maxQueueLength ) - { + if (requests.size() >= maxQueueLength) { processCondition.signal(); } - } - finally - { + } finally { lock.unlock(); } } - public void close() - { + public void close() { lock.lock(); - try - { + try { shutdownRequested = true; print.info("Shutdown requested"); processCondition.signal(); - } - finally - { + } finally { lock.unlock(); } - try - { + try { this.join(); - } - catch ( InterruptedException ie ) - { + } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } } - private boolean isBatchReady() - { + private boolean isBatchReady() { lock.lock(); - try - { + try { return requests.size() > 0 && (requests.size() >= maxQueueLength || - processNow || - shutdownRequested); - } - finally - { + processNow || + shutdownRequested); + } finally { lock.unlock(); } } - private void processBatch() - { + private void processBatch() { List batch; lock.lock(); - try - { - batch = new ArrayList(maxQueueLength); + try { + batch = new ArrayList<>(maxQueueLength); int numAdded = 0; - while ( numAdded < maxQueueLength && !requests.isEmpty() ) - { + while (numAdded < maxQueueLength && !requests.isEmpty()) { batch.add(requests.poll()); } - } - finally - { + } finally { lock.unlock(); } - try - { + try { print.info("Sending batch: " + batch.size() + " requests"); List responses = service.requestTokensImmediate(tokenClassName, batch); callback.tokensReceived(batch, responses); - } - catch ( Exception e ) - { + } catch (Exception e) { print.error("Exception on send: " + e.getMessage(), e); callback.exceptionThrown(batch, e); } } @Override - public void run() - { + public void run() { NDC.push("Request Thread: "); print.info("Started"); // lock.lock(); - try - { - while ( true ) - { + try { + while (true) { print.info("Checking batch"); - while ( isBatchReady() ) - { + while (isBatchReady()) { print.info("Processing batch"); processBatch(); } - if ( shutdownRequested ) - { + if (shutdownRequested) { print.info("Shutdown acknowledged"); break; } lock.lock(); - try - { + try { Date deadline = new Date(System.currentTimeMillis() + maxWaitTime); - try - { + try { print.info("Waiting until: " + deadline); processNow = !processCondition.awaitUntil(deadline); - } - catch ( InterruptedException ie ) - { + } catch (InterruptedException ie) { Thread.currentThread().interrupt(); print.info("Interrupted"); break; } - } - finally - { + } finally { lock.unlock(); } } - } - catch ( Exception e ) - { + } catch (Exception e) { callback.unexpectedException(e); } finally { print.info("Stopped"); -- GitLab From d97e7482a1aba253254b07cb20d2e4c35121bcae Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Mon, 8 Apr 2019 12:30:24 -0400 Subject: [PATCH 02/49] Add identifier to track operations when logging This is for the TokenRequestBatch and TokenValidator so that we can better understand what work is being done when problems arise in their threads. --- .../umiacs/ace/monitor/audit/AuditThread.java | 109 ++++----- .../umiacs/ace/monitor/audit/AuditTokens.java | 5 +- .../edu/umiacs/ace/ims/api/IMSService.java | 25 +- .../ims/api/ImmediateTokenRequestBatch.java | 51 ++-- .../umiacs/ace/ims/api/TokenValidator.java | 231 ++++++------------ 5 files changed, 176 insertions(+), 245 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java index cedbcef..f4afc75 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java @@ -83,8 +83,6 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; /** - * - * * @author toaster */ public final class AuditThread extends Thread implements CancelCallback { @@ -116,17 +114,10 @@ public final class AuditThread extends Thread implements CancelCallback { private LogEventManager logManager; private AuditIterable iterableItems; - public AuditThread(Collection c, - StorageDriver driver, - boolean auditOnly, - MonitoredItem... startItem) { - this(c, driver, auditOnly, true, startItem); - } - - public AuditThread(Collection c, - StorageDriver driver, + public AuditThread(Collection c, + StorageDriver driver, boolean auditOnly, - boolean verbose, + boolean verbose, MonitoredItem... startItem) { this.auditOnly = auditOnly; this.verbose = verbose; @@ -222,18 +213,18 @@ public final class AuditThread extends Thread implements CancelCallback { callback = new FileAuditCallback(coll, session, this); boolean auditTokens = SettingsUtil.getBoolean(coll, - ConfigConstants.ATTR_AUDIT_TOKENS); + ConfigConstants.ATTR_AUDIT_TOKENS); // Audit only does not attempt to connect to the IMS, so we // only need these checks if we are not in it - if( !auditOnly ) { + if (!auditOnly) { // If we can open a connection to the IMS, check what mode // we're in, else fallback so we go to audit only mode - if ( openIms() ) { + if (openIms()) { boolean openValidator = openTokenValidator(MessageDigest.getInstance("SHA-256")); //short circuit - if ( auditTokens && !openValidator ) { + if (auditTokens && !openValidator) { return; } } else { @@ -264,11 +255,11 @@ public final class AuditThread extends Thread implements CancelCallback { validator.close(); validator = null; } - + lastFileSeen = "Setting collection state"; setCollectionState(); - if ( verbose ) { + if (verbose) { logAuditFinish(); generateAuditReport(); } @@ -283,16 +274,17 @@ public final class AuditThread extends Thread implements CancelCallback { private boolean openIms() { try { IMSService ims; - ims = IMSService.connect(imsHost, - imsPort, - AuditThreadFactory.useSSL(), - AuditThreadFactory.isBlocking(), - AuditThreadFactory.getMaxBlockTime()); - - batch = ims.createImmediateTokenRequestBatch(tokenClassName, - callback, - 1000, - 5000); + ims = IMSService.connect(imsHost, + imsPort, + AuditThreadFactory.useSSL(), + AuditThreadFactory.isBlocking(), + AuditThreadFactory.getMaxBlockTime()); + + batch = ims.createImmediateTokenRequestBatch(coll.getId().toString(), + tokenClassName, + callback, + 1000, + 5000); return true; } catch (IMSException e) { EntityManager em; @@ -317,17 +309,19 @@ public final class AuditThread extends Thread implements CancelCallback { try { IMSService ims; ims = IMSService.connect(imsHost, - imsPort, - AuditThreadFactory.useSSL(), - AuditThreadFactory.isBlocking(), - AuditThreadFactory.getMaxBlockTime()); + imsPort, + AuditThreadFactory.useSSL(), + AuditThreadFactory.isBlocking(), + AuditThreadFactory.getMaxBlockTime()); TokenAuditCallback tokenCallback = new TokenAuditCallback(itemMap, this, coll, session); - validator = ims.createTokenValidator(tokenCallback, 1000, - 5000, digest); - // "SHA-256-0", new BatchCallback(), 1000, 5000); + validator = ims.createTokenValidator(coll.getId().toString(), + tokenCallback, + 1000, + 5000, + digest); return true; } catch (IMSException e) { EntityManager em; @@ -353,6 +347,7 @@ public final class AuditThread extends Thread implements CancelCallback { // Sleep to ensure that we update a monitored item at a time strictly // greater than our start date + // todo: do we need to sleep for such a long period of time? try { Thread.sleep(1500); } catch (InterruptedException e) { @@ -363,9 +358,8 @@ public final class AuditThread extends Thread implements CancelCallback { // 2. Get file list try { - iterableItems = driver.getWorkList(coll.getDigestAlgorithm(), - filter, - baseItemPathList); + String digestAlgorithm = coll.getDigestAlgorithm(); + iterableItems = driver.getWorkList(digestAlgorithm, filter, baseItemPathList); } catch (Exception e) { abortException = e; return; @@ -409,7 +403,7 @@ public final class AuditThread extends Thread implements CancelCallback { // let token batch finish before processing or items waiting tokens // will appear as errors. - // Thread.sleep(2000); + // todo: is this necessary? if (batch != null) { batch.close(); batch = null; @@ -429,8 +423,8 @@ public final class AuditThread extends Thread implements CancelCallback { SchedulerContextListener.mailReport(rs, createMailList()); } catch (MessagingException e) { EntityManager em = PersistUtil.getEntityManager(); - logManager.persistCollectionEvent(LogEnum.SMTP_ERROR, - e.getMessage(), em); + logManager.persistCollectionEvent(LogEnum.SMTP_ERROR, + e.getMessage(), em); em.close(); LOG.error("Could not send report summary", e); } @@ -441,7 +435,6 @@ public final class AuditThread extends Thread implements CancelCallback { * 1. we got an exception somewhere, abortException will be set * 2. user requested cancel * 3. we finished as expected - * */ private void logAuditFinish() { EntityManager em = PersistUtil.getEntityManager(); @@ -509,9 +502,9 @@ public final class AuditThread extends Thread implements CancelCallback { LOG.trace( "Driver returned {Item=" + currentFile.getPathList()[0] - + ";error=" + currentFile.isError() + ";error-msg=" - + currentFile.getErrorMessage() + ";hash=" - + currentFile.getHash() + "}"); + + ";error=" + currentFile.isError() + ";error-msg=" + + currentFile.getErrorMessage() + ";hash=" + + currentFile.getHash() + "}"); mim = new MonitoredItemManager(em); @@ -522,8 +515,8 @@ public final class AuditThread extends Thread implements CancelCallback { try { String parentName = extractAndRegisterParent(mim, currentFile); - MonitoredItem item = null; - if ((item = mim.getItemByPath(fileName, coll)) != null) { + MonitoredItem item = mim.getItemByPath(fileName, coll); + if (item != null) { LogEvent event = null; LOG.trace("Updating existing item " + fileName); item.setLastVisited(new Date()); @@ -586,12 +579,12 @@ public final class AuditThread extends Thread implements CancelCallback { TokenRequest request = new TokenRequest(); request.setName(item.getId().toString()); request.setHashValue(currentFile.getHash()); - if (!Strings.isEmpty(item.getPath()) && - !Strings.isEmpty(currentFile.getHash()) && batch != null) { + if (!Strings.isEmpty(item.getPath()) && + !Strings.isEmpty(currentFile.getHash()) && batch != null) { try { batch.add(request); } catch (InterruptedException e) { - abortException = e; + abortException = e; } } return event; @@ -600,7 +593,7 @@ public final class AuditThread extends Thread implements CancelCallback { private LogEvent validateIntegrity(FileBean currentFile, MonitoredItem item) { LOG.trace( "Generated checksum: " + currentFile.getHash() - + " expected checksum: " + item.getFileDigest()); + + " expected checksum: " + item.getFileDigest()); LogEvent event; // If we have a registered file, set the digested value @@ -657,7 +650,7 @@ public final class AuditThread extends Thread implements CancelCallback { } private LogEvent[] addNewFile(FileBean currentFile, String fileName, - final String parentName, final MonitoredItemManager mim) { + final String parentName, final MonitoredItemManager mim) { newFilesFound++; LogEvent[] event = new LogEvent[2]; @@ -678,8 +671,8 @@ public final class AuditThread extends Thread implements CancelCallback { TokenRequest request = new TokenRequest(); request.setName(mi.getId().toString()); request.setHashValue(currentFile.getHash()); - if (!Strings.isEmpty(fileName) && - !Strings.isEmpty(currentFile.getHash()) && batch != null) { + if (!Strings.isEmpty(fileName) && + !Strings.isEmpty(currentFile.getHash()) && batch != null) { try { batch.add(request); } catch (InterruptedException e) { @@ -691,7 +684,7 @@ public final class AuditThread extends Thread implements CancelCallback { } private String extractAndRegisterParent(MonitoredItemManager mim, - FileBean currentFile) { + FileBean currentFile) { String parentName = (currentFile.getPathList().length > 1 ? currentFile.getPathList()[1] : null); @@ -757,8 +750,8 @@ public final class AuditThread extends Thread implements CancelCallback { try { // Update the monitored item table Query query = em.createNamedQuery("MonitoredItem.updateMissing") - .setParameter("coll", coll) - .setParameter("date", d); + .setParameter("coll", coll) + .setParameter("date", d); int i = query.executeUpdate(); if (i > 0) { @@ -804,7 +797,7 @@ public final class AuditThread extends Thread implements CancelCallback { em = PersistUtil.getEntityManager(); logManager.persistCollectionEvent(LogEnum.SYSTEM_ERROR, "Cannot collect digests from remote site: " - + pc.getSite().getRemoteURL(), em); + + pc.getSite().getRemoteURL(), em); em.close(); em = null; LOG.info("remote site returned null stream " + pc.getSite(). diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditTokens.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditTokens.java index c1ccde3..4daffc5 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditTokens.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditTokens.java @@ -199,7 +199,10 @@ public final class AuditTokens extends Thread implements CancelCallback { AuditThreadFactory.isBlocking(), AuditThreadFactory.getMaxBlockTime()); callback = new TokenAuditCallback(itemMap, this, collection, session); - validator = ims.createTokenValidator(callback, 1000, 5000, + validator = ims.createTokenValidator(collection.getId().toString(), + callback, + 1000, + 5000, digest); } catch (Throwable e) { EntityManager em; diff --git a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSService.java b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSService.java index 04d2600..897e552 100644 --- a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSService.java +++ b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSService.java @@ -290,13 +290,15 @@ public final class IMSService { /** * Create a request batch based on the requestTokensImmediate call. * + * @param identifier a unique identifier for tracking what the request batch is working on * @param tokenClassName token class name to use 'SHA-256-0' - * @param callback calback to handle token responses and errors + * @param callback callback to handle token responses and errors * @param maxQueueLength maximum queue size before requests will be forced * @param maxWaitTime time to wait between sending requests * @return batched request */ - public TokenRequestBatch createImmediateTokenRequestBatch(String tokenClassName, + public TokenRequestBatch createImmediateTokenRequestBatch(String identifier, + String tokenClassName, RequestBatchCallback callback, int maxQueueLength, int maxWaitTime) { @@ -305,11 +307,21 @@ public final class IMSService { Check.isPositive("maxQueueLength", maxQueueLength); Check.isPositive("maxWaitTime", maxWaitTime); - return new ImmediateTokenRequestBatch(this, tokenClassName, callback, - maxQueueLength, maxWaitTime); + return new ImmediateTokenRequestBatch(this, identifier, tokenClassName, callback, maxQueueLength, maxWaitTime); } - public TokenValidator createTokenValidator(ValidationCallback callback, + /** + * Create a {@link TokenValidator} + * + * @param identifier a unique identifier for tracking what the request batch is working on + * @param callback callback to handle token validation and errors + * @param maxQueueLength maximum queue size before requests will be forced + * @param maxWaitTime time to wait between sending requests + * @param digest type of {@link MessageDigest} used + * @return the {@link TokenValidator} + */ + public TokenValidator createTokenValidator(String identifier, + ValidationCallback callback, int maxQueueLength, int maxWaitTime, MessageDigest digest) { @@ -317,8 +329,7 @@ public final class IMSService { Check.isPositive("maxQueueLength", maxQueueLength); Check.isPositive("maxWaitTime", maxWaitTime); - return new TokenValidator(this, callback, - maxWaitTime, maxQueueLength, digest); + return new TokenValidator(this, identifier, callback, maxWaitTime, maxQueueLength, digest); } private IMSException handleException(Exception e) { diff --git a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/ImmediateTokenRequestBatch.java b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/ImmediateTokenRequestBatch.java index 54e9666..e4dac97 100644 --- a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/ImmediateTokenRequestBatch.java +++ b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/ImmediateTokenRequestBatch.java @@ -55,25 +55,27 @@ class ImmediateTokenRequestBatch extends Thread implements TokenRequestBatch { private static final Logger print = Logger.getLogger(ImmediateTokenRequestBatch.class); - //private static final int MAXQUEUESIZE = 10 - private String identifier; - private IMSService service; - private LinkedList requests = new LinkedList<>(); - private String tokenClassName; - private int maxWaitTime; - private int maxQueueLength; private RequestBatchCallback callback; - private boolean shutdownRequested = false; private boolean processNow = false; - private Lock lock = new ReentrantLock(); - private Condition processCondition; + private boolean shutdownRequested = false; + private LinkedList requests = new LinkedList<>(); - ImmediateTokenRequestBatch(IMSService service, + private final IMSService service; + private final String identifier; + private final String tokenClassName; + private final int maxWaitTime; + private final int maxQueueLength; + private final Lock lock = new ReentrantLock(); + private final Condition processCondition; + + ImmediateTokenRequestBatch(IMSService imsService, + String identifier, String tokenClassName, RequestBatchCallback callback, int maxQueueLength, int maxWaitTime) { - this.service = service; + this.service = imsService; + this.identifier = identifier; this.tokenClassName = tokenClassName; this.callback = callback; this.maxQueueLength = maxQueueLength; @@ -155,10 +157,9 @@ class ImmediateTokenRequestBatch extends Thread implements TokenRequestBatch { @Override public void run() { - NDC.push("Request Thread: "); + NDC.push("Request Thread (" + identifier + "): "); print.info("Started"); -// lock.lock(); try { while (true) { print.info("Checking batch"); @@ -170,18 +171,16 @@ class ImmediateTokenRequestBatch extends Thread implements TokenRequestBatch { print.info("Shutdown acknowledged"); break; } + lock.lock(); + Date deadline = new Date(System.currentTimeMillis() + maxWaitTime); try { - Date deadline = new Date(System.currentTimeMillis() + - maxWaitTime); - try { - print.info("Waiting until: " + deadline); - processNow = !processCondition.awaitUntil(deadline); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - print.info("Interrupted"); - break; - } + print.info("Waiting until: " + deadline); + processNow = !processCondition.awaitUntil(deadline); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + print.info("Interrupted"); + break; } finally { lock.unlock(); } @@ -193,10 +192,6 @@ class ImmediateTokenRequestBatch extends Thread implements TokenRequestBatch { NDC.pop(); NDC.remove(); } -// finally -// { -// lock.unlock(); -// } } } diff --git a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/TokenValidator.java b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/TokenValidator.java index 66c815a..aa068ee 100644 --- a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/TokenValidator.java +++ b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/TokenValidator.java @@ -31,12 +31,16 @@ package edu.umiacs.ace.ims.api; +import edu.umiacs.ace.hashtree.Proof; import edu.umiacs.ace.hashtree.ProofValidator; import edu.umiacs.ace.ims.ws.ProofElement; import edu.umiacs.ace.ims.ws.RoundSummary; import edu.umiacs.ace.token.AceToken; import edu.umiacs.ace.util.HashValue; import edu.umiacs.util.Check; +import org.apache.log4j.Logger; +import org.apache.log4j.NDC; + import java.security.MessageDigest; import java.util.ArrayList; import java.util.Date; @@ -47,32 +51,35 @@ import java.util.Map; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import org.apache.log4j.Logger; -import org.apache.log4j.NDC; /** + * Validate ACE Tokens * * @author toaster */ -public class TokenValidator extends Thread -{ +public class TokenValidator extends Thread { + + private static final Logger print = Logger.getLogger(TokenValidator.class); - private Map requests = new HashMap(); - private IMSService connection; + private Map requests = new HashMap<>(); private ValidationCallback callback; - private static final Logger print = - Logger.getLogger(TokenValidator.class); private boolean shutdownRequested = false; private boolean processNow = false; - private int maxWaitTime; - private int maxQueueLength; - private Lock lock = new ReentrantLock(); - private Condition processCondition; - private MessageDigest digest; - - TokenValidator(IMSService connection, ValidationCallback callback, - int maxWaitTime, int maxQueueLength, MessageDigest digest) - { + + private final IMSService connection; + private final String identifier; + private final int maxWaitTime; + private final int maxQueueLength; + private final Lock lock = new ReentrantLock(); + private final Condition processCondition; + private final MessageDigest digest; + + TokenValidator(IMSService connection, + String identifier, + ValidationCallback callback, + int maxWaitTime, + int maxQueueLength, + MessageDigest digest) { Check.notNull("connection", connection); Check.notNull("callback", callback); Check.notNull("digest", digest); @@ -81,91 +88,53 @@ public class TokenValidator extends Thread this.maxQueueLength = maxQueueLength; this.maxWaitTime = maxWaitTime; this.digest = digest; + this.identifier = identifier; processCondition = lock.newCondition(); this.start(); } -// public void add(String fileHash, AceToken token) throws InterruptedException -// { -// Check.notNull("token", token); -// Check.notNull("fileHash", fileHash); -// -// lock.lockInterruptibly(); -// try -// { -// if ( shutdownRequested ) -// { -// throw new IllegalStateException("Process shutdown"); -// } -// print.trace("Adding work: " + fileHash); -// requests.put(IMSUtil.convertToken(token), fileHash); -// if ( requests.size() >= maxQueueLength ) -// { -// processCondition.signal(); -// } -// } -// finally -// { -// lock.unlock(); -// } -// } - - public void add(String fileHash, AceToken token) throws InterruptedException - { + public void add(String fileHash, AceToken token) throws InterruptedException { Check.notNull("token", token); Check.notNull("fileHash", fileHash); lock.lockInterruptibly(); - try - { - if ( shutdownRequested ) - { + try { + if (shutdownRequested) { throw new IllegalStateException("Process shutdown"); } print.trace("Adding work: " + fileHash); requests.put(token, fileHash); - if ( requests.size() >= maxQueueLength ) - { + if (requests.size() >= maxQueueLength) { processCondition.signal(); } - } - finally - { + } finally { lock.unlock(); } } - public void close() - { + public void close() { lock.lock(); - try - { + try { shutdownRequested = true; print.info("Shutdown requested"); processCondition.signal(); - } - finally - { + } finally { lock.unlock(); } - try - { + try { this.join(); - } - catch ( InterruptedException ie ) - { + } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } } - private boolean isBatchReady() - { + private boolean isBatchReady() { return requests.size() > 0 && (requests.size() >= maxQueueLength || - processNow || - shutdownRequested); + processNow || + shutdownRequested); } /** @@ -174,26 +143,22 @@ public class TokenValidator extends Thread * 3. For each token, check to ensure it validates * 4. call appropriate callback for each token */ - private void processBatch() - { - List roundNumbers = new ArrayList(); + private void processBatch() { + List roundNumbers = new ArrayList<>(); ProofValidator pv = new ProofValidator(); - Map> proofMap = new HashMap>(); + Map> proofMap = new HashMap<>(); lock.lock(); - try - { + try { print.trace("Processing batch of size: " + requests.size()); - for ( AceToken token : requests.keySet() ) - { + for (AceToken token : requests.keySet()) { String hash = requests.get(token); WorkUnit unit = new WorkUnit(); unit.setHash(hash); unit.setTokenResponse(token); - if ( !roundNumbers.contains(token.getRound()) ) - { + if (!roundNumbers.contains(token.getRound())) { roundNumbers.add(token.getRound()); proofMap.put(token.getRound(), new LinkedList()); @@ -204,9 +169,7 @@ public class TokenValidator extends Thread print.trace("batch load finished, calling clear, unlocking."); requests.clear(); print.trace("requests size: " + requests.size()); - } - finally - { + } finally { lock.unlock(); } @@ -214,29 +177,24 @@ public class TokenValidator extends Thread List summaries = connection.getRoundSummaries(roundNumbers); print.trace("IMS returned " + summaries.size() + " rounds"); - for ( RoundSummary summary : summaries ) - { + for (RoundSummary summary : summaries) { long round = summary.getId(); - for ( WorkUnit unit : proofMap.get(round) ) - { + for (WorkUnit unit : proofMap.get(round)) { AceToken response = unit.getTokenResponse(); String localLeafHash = unit.getHash(); String imsSuppliedHash = summary.getHashValue(); -// String calculatedHash = calculateRoot(digest, -// localLeafHash, response.getProofElements()); - String calculatedHash = HashValue.asHexString(pv.rootHash(digest, response.getProof(), HashValue.asBytes(localLeafHash))); + Proof responseProof = response.getProof(); + byte[] localHash = HashValue.asBytes(localLeafHash); + byte[] rootHash = pv.rootHash(digest, responseProof, localHash); + String calculatedHash = HashValue.asHexString(rootHash); - if ( imsSuppliedHash.equals(calculatedHash) ) - { + if (imsSuppliedHash.equals(calculatedHash)) { callback.validToken(response); - } - else - { - callback.invalidToken(response, imsSuppliedHash, - calculatedHash); + } else { + callback.invalidToken(response, imsSuppliedHash, calculatedHash); } } @@ -248,51 +206,38 @@ public class TokenValidator extends Thread } @Override - public void run() - { + public void run() { - NDC.push("Validation Thread: "); + NDC.push("Validation Thread (" + identifier + "): "); print.info("Started"); lock.lock(); - try - { + try { - while ( true ) - { + while (true) { print.info("Checking batch"); - while ( isBatchReady() ) - { + while (isBatchReady()) { print.info("Processing batch, queued items: " + requests.size()); processBatch(); } - if ( shutdownRequested ) - { + if (shutdownRequested) { print.info("Shutdown acknowledged"); break; } - Date deadline = new Date(System.currentTimeMillis() - + maxWaitTime); - try - { + Date deadline = new Date(System.currentTimeMillis() + maxWaitTime); + try { print.info("Waiting until: " + deadline); processNow = !processCondition.awaitUntil(deadline); - } - catch ( InterruptedException ie ) - { + } catch (InterruptedException ie) { Thread.currentThread().interrupt(); print.info("Interrupted"); break; } } - } - catch ( Throwable e ) - { + } catch (Throwable e) { callback.unexpectedException(e); - } - finally - { + } finally { lock.unlock(); print.info("Stopped"); NDC.pop(); @@ -302,87 +247,71 @@ public class TokenValidator extends Thread /** * Calculate the root hash given the leaf, local hash and a list of proof elements - * proof calculation starts at elements[0] . The element.getIndex() position is + * proof calculation starts at elements[0] . The element.getIndex() position is * where the localHash, or hash from lower tree level is inserted. The * hashes for each element are run through the supplied digest. the elements[n] * value is what eventually gets returned. Please note digest will be reset prior * to computation start - * - * @param digest algorithm to use + * + * @param digest algorithm to use * @param localHash leaf hash to start tree bottom w/ - * @param elements levels proof tree w/ idx 0 the leaf and idx n root -1 + * @param elements levels proof tree w/ idx 0 the leaf and idx n root -1 * @return */ public static String calculateRoot(MessageDigest digest, String localHash, - List elements) - { + List elements) { // previous from lower level in tree is inserted at element.getIndex() position byte[] currentHash = HashValue.asBytes(localHash); digest.reset(); - for ( ProofElement element : elements ) - { + for (ProofElement element : elements) { digest.reset(); int index = element.getIndex(); int i = 0; -// System.out.println("index: " + element.getIndex()); - for ( String hash : element.getHashes() ) - { + for (String hash : element.getHashes()) { byte[] hashBytes = HashValue.asBytes(hash); // it element index is the current index, insert then continue on - if ( i == index ) - { -// System.out.println("prev hash: " + i + " " + HashValue.asHexString(currentHash)); + if (i == index) { digest.update(currentHash); i++; } -// System.out.println("elem hash: " + i + " " + hash); digest.update(hashBytes); i++; } // end case, if index is last element in list, then need to append - if ( index == i ) - { -// System.out.println("prev hash: " + i + " " + HashValue.asHexString(currentHash)); + if (index == i) { digest.update(currentHash); } currentHash = digest.digest(); -// System.out.println("Calc ihv: " + HashValue.asHexString(currentHash)); } return HashValue.asHexString(currentHash); } - private class WorkUnit - { + private class WorkUnit { private String hash; private AceToken tokenResponse; - public WorkUnit() - { + public WorkUnit() { } - public void setTokenResponse(AceToken tokenResponse) - { + public void setTokenResponse(AceToken tokenResponse) { this.tokenResponse = tokenResponse; } - public AceToken getTokenResponse() - { + public AceToken getTokenResponse() { return tokenResponse; } - public void setHash(String hash) - { + public void setHash(String hash) { this.hash = hash; } - public String getHash() - { + public String getHash() { return hash; } } -- GitLab From 52855d89d61630f184e2c264a00259b91fa504cd Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Mon, 8 Apr 2019 13:31:27 -0400 Subject: [PATCH 03/49] Minor updates --- .../umiacs/ace/monitor/audit/AuditThread.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java index f4afc75..5e7181a 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java @@ -340,16 +340,14 @@ public final class AuditThread extends Thread implements CancelCallback { } private void performAudit() { - // 1. Setup audit PathFilter filter = new SimpleFilter(coll); Date startDate = new Date(); // Sleep to ensure that we update a monitored item at a time strictly // greater than our start date - // todo: do we need to sleep for such a long period of time? try { - Thread.sleep(1500); + Thread.sleep(500); } catch (InterruptedException e) { abortException = e; return; @@ -401,13 +399,16 @@ public final class AuditThread extends Thread implements CancelCallback { return; } - // let token batch finish before processing or items waiting tokens - // will appear as errors. - // todo: is this necessary? + /* + * I'm pretty sure this isn't necessary, from my (brief) testing I haven't noticed any issues + * + * // let token batch finish before processing or items waiting tokens + * // will appear as errors. if (batch != null) { batch.close(); batch = null; } + */ // harvest remote collections lastFileSeen = "comparing to peer sites"; @@ -780,12 +781,10 @@ public final class AuditThread extends Thread implements CancelCallback { } private void compareToPeers() { - EntityManager em = PersistUtil.getEntityManager(); MonitoredItemManager mim = new MonitoredItemManager(em); List currentErrors = mim.listRemoteErrors(coll); - mim = null; em.close(); em = null; @@ -810,8 +809,8 @@ public final class AuditThread extends Thread implements CancelCallback { try { CompareResults cr = new CompareResults(cc); cc.compareTo(cr, coll, null); - // cc.loadCollectionTable(coll, null); - // cc.doCompare(); + // cc.loadCollectionTable(coll, null); + // cc.doCompare(); // For now, we don't care about files that only exist remotely // cc.getUnseenTargetFiles(); // State: P, only set if file is currently active as local errors -- GitLab From 1c7565c005d7627787030671e946c4b74410881f Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Mon, 8 Apr 2019 13:32:14 -0400 Subject: [PATCH 04/49] Add comment --- .../src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java | 1 + 1 file changed, 1 insertion(+) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java index 5e7181a..3ad2b3f 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java @@ -411,6 +411,7 @@ public final class AuditThread extends Thread implements CancelCallback { */ // harvest remote collections + // it doesn't make sense for this to happen before the batch and validation threads close lastFileSeen = "comparing to peer sites"; compareToPeers(); } -- GitLab From e2b99969690ab6112de307aa6e764ccc7f9e1f2e Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Mon, 8 Apr 2019 17:00:43 -0400 Subject: [PATCH 05/49] Work in progress, trying to track down potential deadlock --- .../ace/driver/localfile/LocalFileAccess.java | 42 ++++++++----------- .../umiacs/ace/monitor/audit/AuditThread.java | 16 +++---- .../ace/monitor/audit/TokenAuditCallback.java | 17 +++----- 3 files changed, 30 insertions(+), 45 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/driver/localfile/LocalFileAccess.java b/ace-am/src/main/java/edu/umiacs/ace/driver/localfile/LocalFileAccess.java index 0c7cb53..115cbb0 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/driver/localfile/LocalFileAccess.java +++ b/ace-am/src/main/java/edu/umiacs/ace/driver/localfile/LocalFileAccess.java @@ -34,22 +34,23 @@ import edu.umiacs.ace.driver.AuditIterable; import edu.umiacs.ace.driver.DriverStateBean; import edu.umiacs.ace.driver.DriverStateBean.State; import edu.umiacs.ace.driver.FileBean; -import edu.umiacs.ace.driver.filter.PathFilter; -import edu.umiacs.ace.driver.StorageDriver; -import edu.umiacs.ace.monitor.core.MonitoredItem; -import edu.umiacs.ace.monitor.core.Collection; import edu.umiacs.ace.driver.QueryThrottle; import edu.umiacs.ace.driver.StateBeanDigestListener; +import edu.umiacs.ace.driver.StorageDriver; +import edu.umiacs.ace.driver.filter.PathFilter; +import edu.umiacs.ace.monitor.core.Collection; +import edu.umiacs.ace.monitor.core.MonitoredItem; import edu.umiacs.ace.util.HashValue; import edu.umiacs.ace.util.ThreadedDigestStream; import edu.umiacs.ace.util.ThrottledInputStream; -import edu.umiacs.io.IO; import edu.umiacs.util.Strings; +import org.apache.log4j.Logger; + +import javax.persistence.EntityManager; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; -import java.security.DigestInputStream; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; @@ -58,12 +59,10 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Queue; -import javax.persistence.EntityManager; -import org.apache.log4j.Logger; /** * Storage driver for accessing files stored on a local file system (ie, java.io.File) - * + * * @author toaster */ public class LocalFileAccess extends StorageDriver { @@ -138,18 +137,19 @@ public class LocalFileAccess extends StorageDriver { class MyIterator implements Iterator { private FileBean next; - private Queue dirsToProcess = new LinkedList(); - private Queue filesToProcess = new LinkedList(); + private Queue dirsToProcess = new LinkedList<>(); + private Queue filesToProcess = new LinkedList<>(); private MessageDigest digest; -// private byte[] buffer = new byte[BLOCK_SIZE]; private File rootFile; private PathFilter filter; private DriverStateBean statebean; private ThreadedDigestStream reader; private boolean cancel = false; - public MyIterator(MonitoredItem[] startPath, PathFilter filter, - String digestAlgorithm, DriverStateBean statebean) { + public MyIterator(MonitoredItem[] startPath, + PathFilter filter, + String digestAlgorithm, + DriverStateBean statebean) { this.statebean = statebean; this.filter = filter; try { @@ -163,8 +163,7 @@ public class LocalFileAccess extends StorageDriver { if (startPath != null) { for (MonitoredItem mi : startPath) { File startFile; - startFile = new File( - getCollection().getDirectory() + mi.getPath()); + startFile = new File(getCollection().getDirectory() + mi.getPath()); if (startFile.isDirectory()) { dirsToProcess.add(startFile); @@ -226,11 +225,11 @@ public class LocalFileAccess extends StorageDriver { } else { for (File f : directory.listFiles()) { LOG.trace("Found item " + f); - if ( f.isDirectory() && + if ( f.isDirectory() && filter.process(extractPathList(f), true)) { LOG.trace("Adding matching directory: " + f); dirsToProcess.add(f); - } else if ( f.isFile() && + } else if ( f.isFile() && filter.process(extractPathList(f), false)) { LOG.trace("Adding matching file: " + f); filesToProcess.add(f); @@ -257,8 +256,6 @@ public class LocalFileAccess extends StorageDriver { List dirPathList = new ArrayList(); File currFile = file; while (!currFile.equals(rootFile)) { -// LOG.trace("Adding dir to path: " + currFile.getPath().substring( -// substrLength)); String pathToAdd = currFile.getPath().substring(substrLength); pathToAdd = pathToAdd.replace(File.separatorChar, '/'); dirPathList.add(pathToAdd); @@ -269,11 +266,7 @@ public class LocalFileAccess extends StorageDriver { @SuppressWarnings("empty-statement") private FileBean processFile(File file) { - - - DigestInputStream dis = null; FileBean fb = new FileBean(); - fb.setPathList(extractPathList(file)); LOG.trace("Processing file: " + file); @@ -307,7 +300,6 @@ public class LocalFileAccess extends StorageDriver { fb.setError(true); fb.setErrorMessage(Strings.exceptionAsString(ie)); } finally { - IO.release(dis); statebean.setStateAndReset(State.IDLE); if (cancel) { return null; diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java index 3ad2b3f..5b423ec 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java @@ -88,9 +88,9 @@ import java.util.concurrent.ConcurrentHashMap; public final class AuditThread extends Thread implements CancelCallback { private static final Logger LOG = Logger.getLogger(AuditThread.class); + private boolean fallback = false; - private Map itemMap = - new ConcurrentHashMap(); + private Map itemMap = new ConcurrentHashMap<>(); private String imsHost; private int imsPort; private Collection coll; @@ -593,8 +593,7 @@ public final class AuditThread extends Thread implements CancelCallback { } private LogEvent validateIntegrity(FileBean currentFile, MonitoredItem item) { - LOG.trace( - "Generated checksum: " + currentFile.getHash() + LOG.trace("Generated checksum: " + currentFile.getHash() + " expected checksum: " + item.getFileDigest()); LogEvent event; @@ -641,8 +640,7 @@ public final class AuditThread extends Thread implements CancelCallback { item.getPath(), msg); item.setState('C'); item.setStateChange(new Date()); - LOG.trace( - "Toggling state to Corrupt for " + item.getPath()); + LOG.trace("Toggling state to Corrupt for " + item.getPath()); } else { event = null; } @@ -651,8 +649,10 @@ public final class AuditThread extends Thread implements CancelCallback { return event; } - private LogEvent[] addNewFile(FileBean currentFile, String fileName, - final String parentName, final MonitoredItemManager mim) { + private LogEvent[] addNewFile(FileBean currentFile, + String fileName, + final String parentName, + final MonitoredItemManager mim) { newFilesFound++; LogEvent[] event = new LogEvent[2]; diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/TokenAuditCallback.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/TokenAuditCallback.java index acfa760..4a2888e 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/TokenAuditCallback.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/TokenAuditCallback.java @@ -57,14 +57,14 @@ public final class TokenAuditCallback implements ValidationCallback { private long totalErrors = 0; private long validTokens = 0; private CancelCallback cancel; - // private Collection collection; - LogEventManager logManager; + private LogEventManager logManager; - public TokenAuditCallback( Map itemMap, - CancelCallback callback, Collection collection, long session ) { + public TokenAuditCallback(Map itemMap, + CancelCallback callback, + Collection collection, + long session ) { this.itemMap = itemMap; this.cancel = callback; - // this.collection = collection; logManager = new LogEventManager(session, collection); } @@ -81,12 +81,8 @@ public final class TokenAuditCallback implements ValidationCallback { totalErrors++; LOG.error("Exception throw registering", throwable); EntityManager em = PersistUtil.getEntityManager(); - // EntityTransaction trans = em.getTransaction(); - // trans.begin(); String msg = "Exception in batch thread" + Strings.exceptionAsString(throwable); logManager.persistCollectionEvent(LogEnum.SYSTEM_ERROR, msg, em); - // lem.abortSite(collection, "Exception in batch thread", throwable); - // trans.commit(); em.close(); cancel.cancel(); } @@ -106,10 +102,7 @@ public final class TokenAuditCallback implements ValidationCallback { if ( !token.getValid() ) { token.setValid(true); - // LogEventManager lem = new LogEventManager(session, collection); - // String path = response.getName(); em.persist(logManager.createItemEvent(LogEnum.TOKEN_VALID, item.getPath())); - // em.persist(lem.validToken(path, collection)); } if ( item.getState() == 'I' || item.getState() == 'R' ) { -- GitLab From 7a94748edf89ef80c944eaf0da41a3619ab812a9 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Tue, 9 Apr 2019 12:52:31 -0400 Subject: [PATCH 06/49] Add hints --- .../java/edu/umiacs/ace/ims/api/ImmediateTokenRequestBatch.java | 2 +- .../src/main/java/edu/umiacs/ace/ims/api/TokenValidator.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/ImmediateTokenRequestBatch.java b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/ImmediateTokenRequestBatch.java index e4dac97..2b4c8f5 100644 --- a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/ImmediateTokenRequestBatch.java +++ b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/ImmediateTokenRequestBatch.java @@ -106,7 +106,7 @@ class ImmediateTokenRequestBatch extends Thread implements TokenRequestBatch { lock.lock(); try { shutdownRequested = true; - print.info("Shutdown requested"); + print.info("Shutdown requested on Token Request"); processCondition.signal(); } finally { lock.unlock(); diff --git a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/TokenValidator.java b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/TokenValidator.java index aa068ee..29e39b9 100644 --- a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/TokenValidator.java +++ b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/TokenValidator.java @@ -117,7 +117,7 @@ public class TokenValidator extends Thread { lock.lock(); try { shutdownRequested = true; - print.info("Shutdown requested"); + print.info("Shutdown requested for validator"); processCondition.signal(); } finally { lock.unlock(); -- GitLab From f61b41c5da066d00b8f0f1b6062fa54c50a48488 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Thu, 11 Apr 2019 17:16:42 -0400 Subject: [PATCH 07/49] Rework blocking in IMSService to use a Supplier instead of reflection Include a IMSResult class which is used to extract the result of the ims communication or throw an exception, mimicking the behavior from before in a more type safe way. --- .../edu/umiacs/ace/ims/api/IMSResult.java | 78 ++++++++ .../edu/umiacs/ace/ims/api/IMSService.java | 172 ++++++++---------- 2 files changed, 157 insertions(+), 93 deletions(-) create mode 100644 ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSResult.java diff --git a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSResult.java b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSResult.java new file mode 100644 index 0000000..e5a6305 --- /dev/null +++ b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSResult.java @@ -0,0 +1,78 @@ +package edu.umiacs.ace.ims.api; + +import edu.umiacs.ace.exception.StatusCode; +import edu.umiacs.ace.ims.ws.IMSFault; +import edu.umiacs.ace.ims.ws.IMSFault_Exception; + +import javax.xml.ws.WebServiceException; +import java.util.Optional; + +/** + * Encapsulate the result of communicating with the IMS. If a response is successful, store it as + * a field 'e', and in the result of failure store the exception. + * + * @author shake + * @since 1.13.1 + */ +public class IMSResult { + + private final E e; + private final Exception exception; + + public IMSResult(E e) { + this.e = e; + this.exception = null; + } + + public IMSResult(Exception exception) { + this.e = null; + this.exception = exception; + } + + public Optional getResult() { + return Optional.ofNullable(e); + } + + public Optional getException() { + return Optional.ofNullable(exception); + } + + /** + * Get the result of the IMS request, or throw an exception if no result exists. + * + * @return the response from the IMS, if successful + * @throws IMSException on the event of failure + */ + public E getOrThrow() { + if (e != null) { + return e; + } + + throw createException(); + } + + private IMSException createException() { + if (exception == null) { + String message = "Invalid state; createException called with exception == null"; + throw new RuntimeException(message); + } + + if (exception instanceof WebServiceException) { + Throwable cause = exception.getCause(); + if (cause instanceof java.net.ConnectException || + cause instanceof java.io.FileNotFoundException) { + return new IMSConnectionException(cause.getMessage(), cause); + } + + } + + if (exception instanceof IMSFault_Exception) { + IMSFault fault = ((IMSFault_Exception) exception).getFaultInfo(); + return new IMSException(fault.getStatusCode(), fault.getMessage()); + } + + throw new IMSException(StatusCode.CLIENT_ERROR, "Client error: " + + exception.getMessage(), exception); + } + +} diff --git a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSService.java b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSService.java index 897e552..e7cfb6c 100644 --- a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSService.java +++ b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSService.java @@ -42,37 +42,41 @@ import edu.umiacs.ace.ims.ws.TokenReceipt; import edu.umiacs.ace.ims.ws.TokenRequest; import edu.umiacs.ace.ims.ws.TokenResponse; import edu.umiacs.util.Check; +import org.apache.log4j.Logger; import javax.xml.namespace.QName; import javax.xml.ws.BindingProvider; import javax.xml.ws.WebServiceException; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.net.MalformedURLException; import java.net.URL; import java.security.MessageDigest; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +import static java.util.Collections.emptyList; /** * @author mmcgann */ public final class IMSService { + + private static final Logger print = Logger.getLogger(IMSService.class); + private static final int DEFAULT_PORT = 8080; private static final String DEFAULT_PATH = "/ace-ims/IMSWebService?wsdl"; private static final boolean DEFAULT_SSL = false; private static final boolean DEFAULT_BLOCKING = false; private boolean blocking; - private long maxBlockTime; + private int maxBlockTime; private IMSWebService port; private IMSService(URL url, boolean blocking, int maxBlockTimeMinutes) { this.blocking = blocking; - this.maxBlockTime = TimeUnit.MILLISECONDS.convert(maxBlockTimeMinutes, - TimeUnit.MINUTES); + // todo: move from block time to attempts + this.maxBlockTime = (maxBlockTimeMinutes < 0) ? 0 : maxBlockTimeMinutes; try { IMSWebService_Service service = new IMSWebService_Service(url, @@ -117,6 +121,7 @@ public final class IMSService { url.append("http://").append(hostName).append(":") .append(portNumber).append(path); } + try { return new IMSService(new URL(url.toString()), blocking, maxBlockTimeMinutes); } catch (MalformedURLException mue) { @@ -153,17 +158,16 @@ public final class IMSService { } public List requestWitnessProofForRounds(List rounds) { - try { - if (blocking) { - Class c = port.getClass(); - Method reqWitnessProof = c.getMethod("requestWitnessProofForRounds", List.class); - return blockUntil(reqWitnessProof, rounds); - } else { - return port.createWitnessProofForRound(rounds); + Supplier>> supplier = () -> { + try { + List witness = port.createWitnessProofForRound(rounds); + return new IMSResult<>(witness); + } catch (Exception e) { + return new IMSResult<>(e); } - } catch (Exception e) { - throw handleException(e); - } + }; + + return invoke(supplier); } /** @@ -176,102 +180,51 @@ public final class IMSService { */ public List requestTokensImmediate(String tokenClassName, List requests) { - try { - if (blocking) { - Class c = port.getClass(); - Method reqTokensImmediate = c.getMethod("requestTokensImmediate", - String.class, - List.class); - return blockUntil(reqTokensImmediate, tokenClassName, requests); - } else { - return port.requestTokensImmediate(tokenClassName, requests); + Supplier>> supplier = () -> { + try { + List responses = port.requestTokensImmediate(tokenClassName, requests); + return new IMSResult<>(responses); + } catch (Exception e) { + return new IMSResult<>(e); } - } catch (Exception e) { - throw handleException(e); - } - } + }; - public List getRoundSummaries(List rounds) { - try { - if (blocking) { - Class c = port.getClass(); - Method roundSummaries = c.getMethod("getRoundSummaries", List.class); - return blockUntil(roundSummaries, rounds); - } else { - return port.getRoundSummaries(rounds); - } - } catch (Exception e) { - throw handleException(e); - } + return invoke(supplier); } - // TODO: This should really be in a circuit breaker type class - // maybe switch from maxBlockTime to maxAttempts - private E blockUntil(Method m, Object... args) throws Exception { - boolean done = false; - E fin = null; - - long start = System.currentTimeMillis(); - long elapsed = 0; - while (!done && elapsed <= maxBlockTime) { + public List getRoundSummaries(List rounds) { + Supplier>> supplier = () -> { try { - // avoid sleeping on the first pass - if (elapsed != 0) { - TimeUnit.MINUTES.sleep(1); - } - - fin = (E) m.invoke(port, args); - - done = true; + List summaries = port.getRoundSummaries(rounds); + return new IMSResult<>(summaries); } catch (Exception e) { - long now = System.currentTimeMillis(); - elapsed = now - start; - - // We want to catch when we simply could not connect to the IMS - // server then continue to block, or when java.lang.reflect catches - // an exception on the invocation of the method - if (e instanceof WebServiceException) { - Throwable cause = e.getCause(); - if (cause instanceof java.net.ConnectException) { - continue; - } - } else if (e instanceof InvocationTargetException) { - // TODO: Properly catch java.net.ConnectException - // it's kind of tricky as the invocation exception makes - // the stack trace murky - continue; - } - throw handleException(e); + return new IMSResult<>(e); } + } ; - } - - // TODO: Throw timeout exception - - return fin; + return invoke(supplier); } public TokenReceipt requestTokensAsync(String tokenClassName, List requests) { - try { - if (blocking) { - Class c = port.getClass(); - Method m = c.getMethod("requestTokensAsync", String.class, List.class); - return blockUntil(m, tokenClassName, requests); - } else { - return port.requestTokensAsync(tokenClassName, requests); + Supplier> supplier = () -> { + try { + TokenReceipt receipt = port.requestTokensAsync(tokenClassName, requests); + return new IMSResult<>(receipt); + } catch (Exception e) { + return new IMSResult<>(e); } - } catch (Exception e) { - throw handleException(e); - } + }; + + return invoke(supplier); } public List retrieveAsyncTokens(List receiptList) { if (receiptList == null || receiptList.isEmpty()) { - return Collections.emptyList(); + return emptyList(); } - List responses = new ArrayList(); + List responses = new ArrayList<>(); try { for (TokenReceipt receipt : receiptList) { @@ -332,6 +285,39 @@ public final class IMSService { return new TokenValidator(this, identifier, callback, maxWaitTime, maxQueueLength, digest); } + private E invoke(Supplier> supplier) { + boolean done = false; + IMSResult imsResult = new IMSResult<>(new RuntimeException("IMS not contacted")); + + long attempt = 0; + + // first attempt always goes through (attempt == 0) + // when blocking, make sure that attempt is less than maxAttempts + while (!done && (attempt == 0 || (attempt <= maxBlockTime && blocking))) { + if (attempt > 0) { + try { + TimeUnit.SECONDS.sleep(30); + } catch (InterruptedException exception) { + // we were probably cancelled, get out of here now + IMSConnectionException imsException = new IMSConnectionException( + exception.getMessage(), + exception.getCause()); + imsResult = new IMSResult<>(imsException); + break; + } + } + + print.info("calling supplier: attempt = " + attempt + + ", maxBlock = " + maxBlockTime + + ", blocking = " + blocking); + imsResult = supplier.get(); + done = imsResult.getResult().isPresent(); + attempt++; + } + + return imsResult.getOrThrow(); + } + private IMSException handleException(Exception e) { if (e instanceof WebServiceException) { Throwable cause = e.getCause(); -- GitLab From fe0f5c3cf04a4806ec1eb3b1a9b31c3644450c30 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Thu, 11 Apr 2019 17:17:46 -0400 Subject: [PATCH 08/49] Target java8 because it's 2019 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 66f4fba..52ba53c 100644 --- a/pom.xml +++ b/pom.xml @@ -21,8 +21,8 @@ maven-compiler-plugin 2.0.2 - 1.7 - 1.7 + 1.8 + 1.8 -- GitLab From a27de3a6cd4b90ed1e0feb2a24dd1b6b5187631b Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Thu, 11 Apr 2019 17:18:07 -0400 Subject: [PATCH 09/49] Update compiler-plugin version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 52ba53c..2fa908b 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ org.apache.maven.plugins maven-compiler-plugin - 2.0.2 + 3.8.0 1.8 1.8 -- GitLab From 20fe776c5f409146d25597d3c32f1d295a2ca1ad Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Thu, 11 Apr 2019 17:19:12 -0400 Subject: [PATCH 10/49] Update scm connection --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2fa908b..92001fb 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ 1.14-SNAPSHOT pom - scm:svn:https://subversion.umiacs.umd.edu/ace/tags/ace-1.9 + scm:git:https://gitlab.umiacs.umd.edu/adapt/ace.git ace -- GitLab From 4bc1aecc8e44e77f51ad231c3714a4e731584c64 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Thu, 11 Apr 2019 17:21:02 -0400 Subject: [PATCH 11/49] Remove unused/transient dependencies --- ace-am/pom.xml | 36 ++---------------------------------- 1 file changed, 2 insertions(+), 34 deletions(-) diff --git a/ace-am/pom.xml b/ace-am/pom.xml index f271039..07cbd36 100644 --- a/ace-am/pom.xml +++ b/ace-am/pom.xml @@ -196,34 +196,8 @@ irods-api 1.6 - - - + + org.glassfish.jersey.containers jersey-container-servlet @@ -234,12 +208,6 @@ jersey-media-json-jackson 2.24 - - edu.umiacs.ace - ace-ims-ws - 1.14-SNAPSHOT - jar - org.apache.httpcomponents httpclient -- GitLab From 2a29722078d3cee7d0f45500eb70cee8c80433c2 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Thu, 11 Apr 2019 17:27:01 -0400 Subject: [PATCH 12/49] Update jstl group --- ace-am/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ace-am/pom.xml b/ace-am/pom.xml index 07cbd36..461a3c3 100644 --- a/ace-am/pom.xml +++ b/ace-am/pom.xml @@ -105,7 +105,7 @@ provided - jstl + javax.servlet jstl 1.1.2 -- GitLab From e330c1f0a4d9c563d221eece30a09392fcae8c6e Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Thu, 11 Apr 2019 17:30:51 -0400 Subject: [PATCH 13/49] Remove redundant information --- ace-am/pom.xml | 1 - ace-ims-api/pom.xml | 4 ---- 2 files changed, 5 deletions(-) diff --git a/ace-am/pom.xml b/ace-am/pom.xml index 461a3c3..f7c76f6 100644 --- a/ace-am/pom.xml +++ b/ace-am/pom.xml @@ -5,7 +5,6 @@ ace edu.umiacs.ace 1.14-SNAPSHOT - ../pom.xml ace-am ace-am diff --git a/ace-ims-api/pom.xml b/ace-ims-api/pom.xml index c4f1eef..5b39554 100644 --- a/ace-ims-api/pom.xml +++ b/ace-ims-api/pom.xml @@ -5,7 +5,6 @@ ace edu.umiacs.ace 1.14-SNAPSHOT - ../pom.xml ace-ims-api ace-ims-api @@ -16,8 +15,6 @@ - - edu.umiacs.ace ace-ims-ws @@ -28,6 +25,5 @@ ace-common ${project.version} - -- GitLab From bffde603f6a06be6fbdb73df55cd189d76c0d5b5 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Thu, 11 Apr 2019 17:59:28 -0400 Subject: [PATCH 14/49] Remove unnecessary sleep --- .../src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java index 5b423ec..a37d076 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java @@ -233,8 +233,6 @@ public final class AuditThread extends Thread implements CancelCallback { } performAudit(); - // Let outstanding tokens finish, TODO, de-hackify this. - sleep(2000); } catch (Throwable e) { LOG.fatal("Uncaught exception in performAudit()", e); if (abortException != null) { -- GitLab From dec8c17188a5bba9e169f3285505ed1375af20ba Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Thu, 11 Apr 2019 18:22:17 -0400 Subject: [PATCH 15/49] Some more linting while we have the opportunity --- .../java/edu/umiacs/ace/monitor/audit/AuditThread.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java index a37d076..c0ebcf0 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java @@ -99,7 +99,7 @@ public final class AuditThread extends Thread implements CancelCallback { private StorageDriver driver; private long session; private boolean auditOnly; - private boolean verbose = false; + private boolean verbose; // exposed for jsp display private long totalErrors = 0; private long newFilesFound = 0; @@ -423,8 +423,7 @@ public final class AuditThread extends Thread implements CancelCallback { SchedulerContextListener.mailReport(rs, createMailList()); } catch (MessagingException e) { EntityManager em = PersistUtil.getEntityManager(); - logManager.persistCollectionEvent(LogEnum.SMTP_ERROR, - e.getMessage(), em); + logManager.persistCollectionEvent(LogEnum.SMTP_ERROR, e.getMessage(), em); em.close(); LOG.error("Could not send report summary", e); } @@ -610,8 +609,7 @@ public final class AuditThread extends Thread implements CancelCallback { // we handle those later if (item.getState() != 'A' && item.getState() != 'P' && item.getState() != 'D' && item.getState() != 'R') { - event = logManager.createItemEvent(LogEnum.FILE_ONLINE, - item.getPath()); + event = logManager.createItemEvent(LogEnum.FILE_ONLINE, item.getPath()); item.setState('A'); item.setStateChange(new Date()); LOG.trace("Toggling state to Active for " + item.getPath()); -- GitLab From a64e9c34ce026fca40e993d2ebb3c09d17bd59f2 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Thu, 18 Apr 2019 11:24:20 -0400 Subject: [PATCH 16/49] Remove unused code --- .../umiacs/ace/monitor/audit/AuditThread.java | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java index c0ebcf0..f737d07 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java @@ -564,8 +564,7 @@ public final class AuditThread extends Thread implements CancelCallback { private LogEvent requestNewToken(FileBean currentFile, MonitoredItem item) { LogEvent event; if (item.getState() != 'T') { - event = logManager.createItemEvent(LogEnum.MISSING_TOKEN, - item.getPath()); + event = logManager.createItemEvent(LogEnum.MISSING_TOKEN, item.getPath()); item.setStateChange(new Date()); item.setState('T'); item.setSize(currentFile.getFileSize()); @@ -632,8 +631,7 @@ public final class AuditThread extends Thread implements CancelCallback { if (item.getState() != 'C') { String msg = "Expected digest: " + item.getFileDigest() + " Saw: " + currentFile.getHash(); - event = logManager.createItemEvent(LogEnum.FILE_CORRUPT, - item.getPath(), msg); + event = logManager.createItemEvent(LogEnum.FILE_CORRUPT, item.getPath(), msg); item.setState('C'); item.setStateChange(new Date()); LOG.trace("Toggling state to Corrupt for " + item.getPath()); @@ -806,8 +804,6 @@ public final class AuditThread extends Thread implements CancelCallback { try { CompareResults cr = new CompareResults(cc); cc.compareTo(cr, coll, null); - // cc.loadCollectionTable(coll, null); - // cc.doCompare(); // For now, we don't care about files that only exist remotely // cc.getUnseenTargetFiles(); // State: P, only set if file is currently active as local errors @@ -839,12 +835,10 @@ public final class AuditThread extends Thread implements CancelCallback { MonitoredItem mi = mim.getItemByPath(dd.getName(), coll); currentErrors.remove(mi); if (mi.getState() == 'A') { - LOG.trace( - "Item corrupt at remote " + mi.getPath() + " " + pc.getSite()); + LOG.trace("Item corrupt at remote " + mi.getPath() + " " + pc.getSite()); mi.setState('D'); mi.setStateChange(new Date()); - EntityTransaction trans = em.getTransaction(); trans.begin(); em.merge(mi); @@ -871,12 +865,10 @@ public final class AuditThread extends Thread implements CancelCallback { trans.begin(); for (MonitoredItem mi : currentErrors) { - LOG.trace( - "Item online at remote " + mi.getPath()); + LOG.trace("Item online at remote " + mi.getPath()); mi.setState('A'); mi.setStateChange(currDate); - em.persist(logManager.createItemEvent(LogEnum.REMOTE_FILE_ONLINE, - mi.getPath())); + em.persist(logManager.createItemEvent(LogEnum.REMOTE_FILE_ONLINE, mi.getPath())); em.merge(mi); } -- GitLab From 9b829695b6cc0d764858789ce96dfac077b501d6 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Thu, 18 Apr 2019 12:36:44 -0400 Subject: [PATCH 17/49] Lint while we have the opportunity --- .../ace/driver/localfile/LocalFileAccess.java | 4 +-- .../umiacs/ace/monitor/audit/AuditThread.java | 35 +++++++++---------- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/driver/localfile/LocalFileAccess.java b/ace-am/src/main/java/edu/umiacs/ace/driver/localfile/LocalFileAccess.java index 115cbb0..cf325b3 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/driver/localfile/LocalFileAccess.java +++ b/ace-am/src/main/java/edu/umiacs/ace/driver/localfile/LocalFileAccess.java @@ -223,7 +223,7 @@ public class LocalFileAccess extends StorageDriver { if (fileList == null) { LOG.info("Could not read directory, skipping: " + directory); } else { - for (File f : directory.listFiles()) { + for (File f : fileList) { LOG.trace("Found item " + f); if ( f.isDirectory() && filter.process(extractPathList(f), true)) { @@ -253,7 +253,7 @@ public class LocalFileAccess extends StorageDriver { int substrLength = rootFile.getPath().length(); // build directory path - List dirPathList = new ArrayList(); + List dirPathList = new ArrayList<>(); File currFile = file; while (!currFile.equals(rootFile)) { String pathToAdd = currFile.getPath().substring(substrLength); diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java index f737d07..a8852d3 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java @@ -540,8 +540,7 @@ public final class AuditThread extends Thread implements CancelCallback { trans.commit(); } else if (!auditOnly && !fallback) { // OK, no registered item, do the registration - LogEvent[] event = addNewFile(currentFile, fileName, - parentName, mim); + LogEvent[] event = addNewFile(currentFile, fileName, parentName, mim); // transaction inside loop to ensure order of commits // default, jpa ignores order of persist calls @@ -568,17 +567,17 @@ public final class AuditThread extends Thread implements CancelCallback { item.setStateChange(new Date()); item.setState('T'); item.setSize(currentFile.getFileSize()); - LOG.trace( - "Toggling state to Missing Token for " + item.getPath()); + LOG.trace("Toggling state to Missing Token for " + item.getPath()); } else { event = null; } + String currentFileHash = currentFile.getHash(); TokenRequest request = new TokenRequest(); request.setName(item.getId().toString()); - request.setHashValue(currentFile.getHash()); + request.setHashValue(currentFileHash); if (!Strings.isEmpty(item.getPath()) && - !Strings.isEmpty(currentFile.getHash()) && batch != null) { + !Strings.isEmpty(currentFileHash) && batch != null) { try { batch.add(request); } catch (InterruptedException e) { @@ -589,18 +588,19 @@ public final class AuditThread extends Thread implements CancelCallback { } private LogEvent validateIntegrity(FileBean currentFile, MonitoredItem item) { - LOG.trace("Generated checksum: " + currentFile.getHash() - + " expected checksum: " + item.getFileDigest()); + String storedDigest = item.getFileDigest(); + String currentFileHash = currentFile.getHash(); + LOG.trace("Generated checksum: " + currentFileHash + " expected checksum: " + storedDigest); LogEvent event; // If we have a registered file, set the digested value - if (null == item.getFileDigest()) { + if (null == storedDigest) { LOG.trace("Setting digest for registered file " + item.getPath()); - item.setFileDigest(currentFile.getHash()); + item.setFileDigest(currentFileHash); item.setLastSeen(new Date()); } - if (currentFile.getHash().equals(item.getFileDigest())) { + if (currentFileHash.equals(storedDigest)) { LOG.trace("Digests match for " + item.getPath()); // File is active and intact // log the transition if it already isn't A @@ -622,15 +622,14 @@ public final class AuditThread extends Thread implements CancelCallback { if (validator != null) { itemMap.put(token, item); try { - validator.add(item.getFileDigest(), token); + validator.add(storedDigest, token); } catch (InterruptedException e) { abortException = e; } } } else { if (item.getState() != 'C') { - String msg = "Expected digest: " + item.getFileDigest() - + " Saw: " + currentFile.getHash(); + String msg = "Expected digest: " + storedDigest + " Saw: " + currentFileHash; event = logManager.createItemEvent(LogEnum.FILE_CORRUPT, item.getPath(), msg); item.setState('C'); item.setStateChange(new Date()); @@ -652,8 +651,8 @@ public final class AuditThread extends Thread implements CancelCallback { LogEvent[] event = new LogEvent[2]; LOG.trace("Registering new item " + fileName); - event[0] = logManager.createItemEvent(LogEnum.FILE_NEW, fileName, - coll.getDirectory() + fileName); + String fullName = coll.getDirectory() + fileName; + event[0] = logManager.createItemEvent(LogEnum.FILE_NEW, fileName, fullName); if (currentFile.isError()) { mim.addItem(fileName, parentName, false, coll, 'M', 0); event[1] = logManager.createItemEvent(LogEnum.ERROR_READING, fileName, @@ -679,12 +678,10 @@ public final class AuditThread extends Thread implements CancelCallback { return event; } - private String extractAndRegisterParent(MonitoredItemManager mim, - FileBean currentFile) { + private String extractAndRegisterParent(MonitoredItemManager mim, FileBean currentFile) { String parentName = (currentFile.getPathList().length > 1 ? currentFile.getPathList()[1] : null); - // 1. make sure directory path is registered if (parentName != null) { parentName = Strings.cleanStringForXml(parentName, '_'); -- GitLab From 1f8070b07b1eb0731c32d0d70b06f1b0d56010b1 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Thu, 18 Apr 2019 15:54:58 -0400 Subject: [PATCH 18/49] Add parameters for ims retry and timeout --- .../ace/monitor/settings/SettingsConstants.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsConstants.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsConstants.java index 6fd3e18..312259f 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsConstants.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsConstants.java @@ -6,13 +6,22 @@ package edu.umiacs.ace.monitor.settings; */ public class SettingsConstants { // Attributes + + /** + * @deprecated + * migrated to PARAM_IMS_MAX_RETRY and PARAM_IMS_RESET_TIMEOUT + */ + @Deprecated public static final String PARAM_AUDIT_BLOCKING = "audit.blocking"; + public static final String PARAM_AUDIT_MAX_BLOCK_TIME = "audit.max.block.time"; public static final String PARAM_AUDIT_ONLY="audit.only"; public static final String PARAM_AUDIT_SAMPLE = "audit.sample"; public static final String PARAM_IMS = "ims"; public static final String PARAM_IMS_PORT = "ims.port"; public static final String PARAM_IMS_TOKEN_CLASS = "ims.tokenclass"; + public static final String PARAM_IMS_MAX_RETRY = "ims.max.retry"; + public static final String PARAM_IMS_RESET_TIMEOUT = "ims.reset.timeout"; public static final String PARAM_IMS_SSL = "ims.ssl"; public static final String PARAM_AUTO_AUDIT_ENABLE = "auto.audit.enable"; public static final String PARAM_THROTTLE_MAXAUDIT = "throttle.maxaudit"; @@ -44,6 +53,8 @@ public class SettingsConstants { public static final String ims = "ims.umiacs.umd.edu"; public static final String imsPort = "80"; public static final String imsTokenClass = "SHA-256"; + public static final String imsMaxRetry = "3"; + public static final String imsResetTimeout = "3000"; public static final String imsSSL = "false"; public static final String authManagement = "true"; public static final String maxIngestThreads = "4"; -- GitLab From c6159482e61dde2cc09fa0be57d11c968110d238 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Thu, 18 Apr 2019 15:56:53 -0400 Subject: [PATCH 19/49] Clean up AddSettingServlet --- .../monitor/settings/AddSettingServlet.java | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/AddSettingServlet.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/AddSettingServlet.java index e8226c1..7f6f484 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/AddSettingServlet.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/AddSettingServlet.java @@ -6,32 +6,36 @@ package edu.umiacs.ace.monitor.settings; import edu.umiacs.ace.util.EntityManagerServlet; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import javax.persistence.EntityManager; -import javax.servlet.RequestDispatcher; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import org.apache.commons.fileupload.FileItemIterator; import org.apache.commons.fileupload.FileItemStream; import org.apache.commons.fileupload.FileUploadException; import org.apache.commons.fileupload.servlet.ServletFileUpload; import org.apache.commons.fileupload.util.Streams; +import org.apache.log4j.Logger; + +import javax.persistence.EntityManager; +import javax.servlet.RequestDispatcher; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; /** * * @author shake */ -public class AddSettingServlet extends EntityManagerServlet{ +public class AddSettingServlet extends EntityManagerServlet { + + private static final Logger log = Logger.getLogger(AddSettingServlet.class); @Override - protected void processRequest(HttpServletRequest request, HttpServletResponse - response, EntityManager em) throws ServletException, IOException { - List customSettings = new ArrayList(); + protected void processRequest(HttpServletRequest request, + HttpServletResponse response, + EntityManager em) throws ServletException, IOException { + List customSettings = new ArrayList<>(); ServletFileUpload su = new ServletFileUpload(); try { @@ -58,7 +62,7 @@ public class AddSettingServlet extends EntityManagerServlet{ } } catch (FileUploadException ex) { - // Logger.getLogger(SettingsServlet.class.getName()).log(Level.SEVERE, null, ex); + log.warn("Exception in AddSettingServlet", ex); } SettingsUtil.updateSettings(customSettings); -- GitLab From 1b8b123df8880cbd5d801980202ea9e8b6f3d593 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Thu, 18 Apr 2019 16:04:34 -0400 Subject: [PATCH 20/49] Deprecate the correct settings --- .../monitor/settings/SettingsConstants.java | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsConstants.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsConstants.java index 312259f..e796163 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsConstants.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsConstants.java @@ -1,21 +1,13 @@ package edu.umiacs.ace.monitor.settings; /** - * * @author shake */ public class SettingsConstants { - // Attributes - /** - * @deprecated - * migrated to PARAM_IMS_MAX_RETRY and PARAM_IMS_RESET_TIMEOUT - */ - @Deprecated + // Attributes public static final String PARAM_AUDIT_BLOCKING = "audit.blocking"; - - public static final String PARAM_AUDIT_MAX_BLOCK_TIME = "audit.max.block.time"; - public static final String PARAM_AUDIT_ONLY="audit.only"; + public static final String PARAM_AUDIT_ONLY = "audit.only"; public static final String PARAM_AUDIT_SAMPLE = "audit.sample"; public static final String PARAM_IMS = "ims"; public static final String PARAM_IMS_PORT = "ims.port"; @@ -31,19 +23,18 @@ public class SettingsConstants { public static final String PARAM_FROM = "mail.from"; public static final String PARAM_USER_AUTH = "auth.management"; public static final String PARAM_4J_FILE = "log4j.appender.A1.File"; - public static final String PARAM_4J_APPENDER = "log4j.appender.A1"; + public static final String PARAM_4J_APPENDER = "log4j.appender.A1"; public static final String PARAM_4J_FILE_SIZE = "log4j.appender.A1.maxFileSize"; public static final String PARAM_4J_BACKUP_INDEX = "log4j.appender.A1.maxBackupIndex"; public static final String PARAM_4J_ROOT_LOGGER = "log4j.rootLogger"; - public static final String PARAM_4J_LAYOUT = "log4j.appender.A1.layout"; + public static final String PARAM_4J_LAYOUT = "log4j.appender.A1.layout"; public static final String PARAM_4J_CONV_PAT = "log4j.appender.A1.layout.ConversionPattern"; public static final String PARAM_4J_IRODS = "log4j.logger.edu.umiacs.irods"; - public static final String PARAM_4J_CLASS ="log4j.logger.edu.umiacs"; + public static final String PARAM_4J_CLASS = "log4j.logger.edu.umiacs"; public static final String PARAM_INGEST = "ingest.maxthreads"; // Default Values public static final String auditBlocking = "false"; - public static final String auditMaxBlockTime = "0"; public static final String auditOnly = "false"; public static final String auditSample = "false"; public static final String mailServer = "localhost.localdomain"; @@ -59,18 +50,28 @@ public class SettingsConstants { public static final String authManagement = "true"; public static final String maxIngestThreads = "4"; public static final String throttleWait = "0"; - public static final String throttleBPS= "0"; + public static final String throttleBPS = "0"; // Yay log4j public static final String log4JA1File = "/tmp/aceam.log"; public static final String log4JA1 = "org.apache.log4j.RollingFileAppender"; public static final String log4JA1MaxFileSize = "100000KB"; public static final String log4JA1MaxBackupIndex = "5"; public static final String log4JRootLogger = "FATAL, A1"; - public static final String log4JA1Layout = "org.apache.log4j.PatternLayout"; + public static final String log4JA1Layout = "org.apache.log4j.PatternLayout"; public static final String log4JA1layoutConversationPattern = "%d{[dd/MMM/yyyy:HH:mm:ss]} %x%m%n"; public static final String log4JLoggerIrods = "ERROR"; public static final String log4JLoggerUMIACS = "TRACE"; + /** + * @deprecated migrated to PARAM_IMS_MAX_RETRY and PARAM_IMS_RESET_TIMEOUT + */ + @Deprecated + public static final String PARAM_AUDIT_MAX_BLOCK_TIME = "audit.max.block.time"; + /** + * @deprecated migrated to PARAM_IMS_MAX_RETRY and PARAM_IMS_RESET_TIMEOUT + */ + @Deprecated + public static final String auditMaxBlockTime = "0"; } -- GitLab From 6a4d1ca3f6df7e2f55b8b274faadac710fcc87fc Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Thu, 18 Apr 2019 16:05:00 -0400 Subject: [PATCH 21/49] Set custom to false --- .../java/edu/umiacs/ace/monitor/settings/SettingsParameter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsParameter.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsParameter.java index 897a909..c5ba602 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsParameter.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsParameter.java @@ -49,6 +49,7 @@ public class SettingsParameter implements Serializable { public SettingsParameter(String attr, String value) { this.attr = attr; this.value = value; + this.custom = false; } public Long getId() { -- GitLab From 87f7dcb3e9840268c0e4955af56ce21ed6af6dd5 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Thu, 18 Apr 2019 16:23:29 -0400 Subject: [PATCH 22/49] Additional function for checking if an int is non-negative --- .../src/main/java/edu/umiacs/util/Strings.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/ace-common/src/main/java/edu/umiacs/util/Strings.java b/ace-common/src/main/java/edu/umiacs/util/Strings.java index 5d3b5b4..0efa408 100644 --- a/ace-common/src/main/java/edu/umiacs/util/Strings.java +++ b/ace-common/src/main/java/edu/umiacs/util/Strings.java @@ -116,7 +116,7 @@ public final class Strings } } - /** + /** * Determines if a string is a valid integer. * * @param val string value to check. @@ -138,6 +138,20 @@ public final class Strings } } + /** + * Determine if a string is a valid integer which is greater than or equal to 0. + * + * @param val string value to check + * @return true if {@code val} can successfully be parsed and is positive + */ + public static boolean isNonNegativeInt(Object val) { + try { + return isValidInt(val) && Integer.parseInt(val.toString()) >= 0; + } catch (NumberFormatException e) { + return false; + } + } + /** * Return exception and contained stack trace as string * -- GitLab From 0b44833500314a99ae6e14f5b1c7e4ec122200d6 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Thu, 18 Apr 2019 17:23:12 -0400 Subject: [PATCH 23/49] Add imsMaxRetry and imsResetTimeout --- .../ace/monitor/audit/AuditThreadFactory.java | 31 +++++++++---------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThreadFactory.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThreadFactory.java index a2a506b..266d94e 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThreadFactory.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThreadFactory.java @@ -65,10 +65,10 @@ public class AuditThreadFactory { private static int imsPort = 80; private static String tokenClass = "SHA-256"; private static boolean auditOnly = false; - private static boolean auditSample = false; private static boolean ssl = false; private static boolean blocking = false; - private static int maxBlockTime = 0; + private static int imsRetryAttempts = 3; + private static int imsResetTimeout = 3000; public static void setIMS( String ims ) { if (Strings.isEmpty(ims)) { @@ -95,8 +95,8 @@ public class AuditThreadFactory { } public static void setImsPort(int imsPort) { - if (imsPort < 1 && imsPort > 32768) { - LOG.error("ims port must be between 1 and 32768, setting default"); + if (imsPort < 1 || imsPort > 32768) { + LOG.warn("ims port must be between 1 and 32768, setting default"); imsPort = Integer.parseInt(SettingsConstants.imsPort); } AuditThreadFactory.imsPort = imsPort; @@ -106,8 +106,16 @@ public class AuditThreadFactory { AuditThreadFactory.auditOnly = auditOnlyMode; } - public static void setAuditSampling(boolean auditSampling ) { - AuditThreadFactory.auditSample = auditSampling; + public static void setImsRetryAttempts(int attempts) { + if (attempts >= 0) { + imsRetryAttempts = attempts; + } + } + + public static void setImsResetTimeout(int timeout) { + if (timeout >= 0) { + imsResetTimeout = timeout; + } } public static void setBlocking(boolean blocking) { @@ -118,17 +126,6 @@ public class AuditThreadFactory { return AuditThreadFactory.blocking; } - public static void setMaxBlockTime(int maxBlockTime) { - if ( maxBlockTime < 0 ) { - maxBlockTime = 0; - } - AuditThreadFactory.maxBlockTime = maxBlockTime; - } - - public static int getMaxBlockTime() { - return maxBlockTime; - } - public static boolean isAuditing() { return !audits.isEmpty(); } -- GitLab From 0524a0b046d05eb6ef8649cfb0cf534bdda09bb0 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Thu, 18 Apr 2019 17:34:34 -0400 Subject: [PATCH 24/49] Set imsMaxRetry and imsResetTimeout from database values --- .../audit/AuditConfigurationContext.java | 70 +++++++------------ 1 file changed, 24 insertions(+), 46 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditConfigurationContext.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditConfigurationContext.java index 94fa5fb..3bdfd44 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditConfigurationContext.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditConfigurationContext.java @@ -55,6 +55,10 @@ import java.util.Timer; import java.util.TimerTask; import java.util.stream.Collectors; +import static edu.umiacs.ace.monitor.settings.SettingsConstants.*; +import static edu.umiacs.ace.monitor.settings.SettingsConstants.PARAM_IMS_MAX_RETRY; +import static edu.umiacs.ace.monitor.settings.SettingsConstants.imsMaxRetry; + /** * Set the IMS for the AuditThread to use. Also, startup a background task * to handle firing off monitor tasks. @@ -65,13 +69,6 @@ import java.util.stream.Collectors; */ public final class AuditConfigurationContext implements ServletContextListener { - //private static final String PARAM_IMS = "ims"; - //private static final String PARAM_IMS_PORT = "ims.port"; - //private static final String PARAM_IMS_TOKEN_CLASS = "ims.tokenclass"; - //private static final String PARAM_IMS_SSL = "ims.ssl"; - //private static final String PARAM_DISABLE_AUTO_AUDIT = "auto.audit.disable"; - //private static final String PARAM_THROTTLE_MAXAUDIT = "throttle.maxaudit"; - //private static final String PARAM_AUDIT_ONLY = "audit.only"; public static final String ATTRIBUTE_PAUSE = "pause"; private static final long HOUR = 1000 * 60 * 60; private Timer checkTimer; @@ -98,68 +95,49 @@ public final class AuditConfigurationContext implements ServletContextListener { ctx.setAttribute(ATTRIBUTE_PAUSE, pb); // Invert the boolean because the PB checks if we're paused, not enabled - String enableAudits = resultMap.getOrDefault( - SettingsConstants.PARAM_AUTO_AUDIT_ENABLE, - SettingsConstants.autoAudit); + String enableAudits = resultMap.getOrDefault(PARAM_AUTO_AUDIT_ENABLE, autoAudit); pb.setPaused(!Boolean.valueOf(enableAudits)); checkTimer = new Timer("Audit Check Timer"); checkTimer.schedule(new MyTimerTask(pb), 0, HOUR); // set IMS for audit Thread from server parameter - AuditThreadFactory.setIMS(resultMap.getOrDefault( - SettingsConstants.PARAM_IMS, - SettingsConstants.ims)); + AuditThreadFactory.setIMS(resultMap.getOrDefault(PARAM_IMS, ims)); - String tokenClass = resultMap.getOrDefault( - SettingsConstants.PARAM_IMS_TOKEN_CLASS, - SettingsConstants.imsTokenClass); + String tokenClass = resultMap.getOrDefault(PARAM_IMS_TOKEN_CLASS, imsTokenClass); AuditThreadFactory.setTokenClass(tokenClass); - String port = resultMap.getOrDefault( - SettingsConstants.PARAM_IMS_PORT, - SettingsConstants.imsPort); + String port = resultMap.getOrDefault(PARAM_IMS_PORT, imsPort); if (Strings.isValidInt(port)) { AuditThreadFactory.setImsPort(Integer.parseInt(port)); } - String auditOnly = resultMap.getOrDefault( - SettingsConstants.PARAM_AUDIT_ONLY, - SettingsConstants.auditOnly); + String auditOnly = resultMap.getOrDefault(PARAM_AUDIT_ONLY, SettingsConstants.auditOnly); AuditThreadFactory.setAuditOnly(Boolean.valueOf(auditOnly)); - // keep this off for now even though it isn't used - AuditThreadFactory.setAuditSampling(false); - - String imsSsl = resultMap.getOrDefault( - SettingsConstants.PARAM_IMS_SSL, - SettingsConstants.imsSSL); + String imsSsl = resultMap.getOrDefault(PARAM_IMS_SSL, imsSSL); AuditThreadFactory.setSSL(Boolean.valueOf(imsSsl)); - String blocking = resultMap.getOrDefault( - SettingsConstants.PARAM_AUDIT_BLOCKING, - SettingsConstants.auditBlocking); + String blocking = resultMap.getOrDefault(PARAM_AUDIT_BLOCKING, auditBlocking); AuditThreadFactory.setBlocking(Boolean.valueOf(blocking)); - String blockTimeS = resultMap.getOrDefault( - SettingsConstants.PARAM_AUDIT_MAX_BLOCK_TIME, - SettingsConstants.auditMaxBlockTime); - int blockTime = 0; - if (Strings.isValidInt(blockTimeS)) { - blockTime = Integer.parseInt(blockTimeS); + int maxRetry = 0; + String imsRetryString = resultMap.getOrDefault(PARAM_IMS_MAX_RETRY, imsMaxRetry); + if (Strings.isNonNegativeInt(imsRetryString)) { + maxRetry = Integer.parseInt(imsRetryString); } + AuditThreadFactory.setImsRetryAttempts(maxRetry); - // Just in case... - if (blockTime < 0) { - blockTime = 0; + int resetTimeout = 0; + String imsResetTimeout = resultMap.getOrDefault(PARAM_IMS_RESET_TIMEOUT, SettingsConstants.imsResetTimeout); + if (Strings.isNonNegativeInt(imsResetTimeout)) { + resetTimeout = Integer.parseInt(imsResetTimeout); } - AuditThreadFactory.setMaxBlockTime(blockTime); + AuditThreadFactory.setImsResetTimeout(resetTimeout); - String maxAudit = resultMap.getOrDefault( - SettingsConstants.PARAM_THROTTLE_MAXAUDIT, - SettingsConstants.maxAudit); - if (Strings.isValidInt(maxAudit)) { - int audit = Integer.parseInt(maxAudit); + String maxAuditString = resultMap.getOrDefault(PARAM_THROTTLE_MAXAUDIT, maxAudit); + if (Strings.isValidInt(maxAuditString)) { + int audit = Integer.parseInt(maxAuditString); if (audit > 0) { AuditThreadFactory.setMaxAudits(audit); } -- GitLab From 642077be3c9c9ea70fba7a1435fd45962218fe97 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Thu, 18 Apr 2019 17:37:39 -0400 Subject: [PATCH 25/49] Getters for retry and timeout --- .../java/edu/umiacs/ace/monitor/audit/AuditThread.java | 6 ++++-- .../edu/umiacs/ace/monitor/audit/AuditThreadFactory.java | 8 ++++++++ .../java/edu/umiacs/ace/monitor/audit/AuditTokens.java | 3 ++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java index a8852d3..dbfd3bf 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java @@ -276,7 +276,8 @@ public final class AuditThread extends Thread implements CancelCallback { imsPort, AuditThreadFactory.useSSL(), AuditThreadFactory.isBlocking(), - AuditThreadFactory.getMaxBlockTime()); + AuditThreadFactory.getImsRetryAttempts(), + AuditThreadFactory.getImsResetTimeout()); batch = ims.createImmediateTokenRequestBatch(coll.getId().toString(), tokenClassName, @@ -310,7 +311,8 @@ public final class AuditThread extends Thread implements CancelCallback { imsPort, AuditThreadFactory.useSSL(), AuditThreadFactory.isBlocking(), - AuditThreadFactory.getMaxBlockTime()); + AuditThreadFactory.getImsRetryAttempts(), + AuditThreadFactory.getImsResetTimeout()); TokenAuditCallback tokenCallback = new TokenAuditCallback(itemMap, this, coll, diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThreadFactory.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThreadFactory.java index 266d94e..babbe70 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThreadFactory.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThreadFactory.java @@ -106,12 +106,20 @@ public class AuditThreadFactory { AuditThreadFactory.auditOnly = auditOnlyMode; } + public static int getImsRetryAttempts() { + return imsRetryAttempts; + } + public static void setImsRetryAttempts(int attempts) { if (attempts >= 0) { imsRetryAttempts = attempts; } } + public static int getImsResetTimeout() { + return imsResetTimeout; + } + public static void setImsResetTimeout(int timeout) { if (timeout >= 0) { imsResetTimeout = timeout; diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditTokens.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditTokens.java index 4daffc5..95670b4 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditTokens.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditTokens.java @@ -197,7 +197,8 @@ public final class AuditTokens extends Thread implements CancelCallback { imsPort, AuditThreadFactory.useSSL(), AuditThreadFactory.isBlocking(), - AuditThreadFactory.getMaxBlockTime()); + AuditThreadFactory.getImsRetryAttempts(), + AuditThreadFactory.getImsResetTimeout()); callback = new TokenAuditCallback(itemMap, this, collection, session); validator = ims.createTokenValidator(collection.getId().toString(), callback, -- GitLab From bcc2b8cf98fe4d2a0e7dc36bc28f2073b7e47a53 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Thu, 18 Apr 2019 17:40:49 -0400 Subject: [PATCH 26/49] Set imsMaxRetry and imsResetTimeout when reloading settings --- .../ace/monitor/settings/SettingsUtil.java | 132 ++++++++---------- 1 file changed, 58 insertions(+), 74 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsUtil.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsUtil.java index 0868f39..c9d885a 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsUtil.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsUtil.java @@ -4,6 +4,7 @@ import edu.umiacs.ace.monitor.audit.AuditThreadFactory; import edu.umiacs.ace.monitor.core.Collection; import edu.umiacs.ace.monitor.reporting.SchedulerContextListener; import edu.umiacs.ace.util.PersistUtil; +import edu.umiacs.util.Strings; import javax.persistence.EntityManager; import javax.persistence.EntityTransaction; @@ -15,8 +16,9 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import static edu.umiacs.ace.monitor.settings.SettingsConstants.*; + /** - * * @author shake */ public class SettingsUtil { @@ -27,7 +29,7 @@ public class SettingsUtil { * @param attr the attribute to query on * @return the Setting associated with the attribute */ - public static SettingsParameter getItemByAttr( String attr ) { + public static SettingsParameter getItemByAttr(String attr) { EntityManager em = PersistUtil.getEntityManager(); Query q = em.createNamedQuery("SettingsParameter.getAttr"); q.setParameter("attr", attr); @@ -87,17 +89,17 @@ public class SettingsUtil { EntityTransaction trans = em.getTransaction(); trans.begin(); - for ( SettingsParameter setting : settings ) { + for (SettingsParameter setting : settings) { // Skip any empty settings - if ( setting.getName().trim().isEmpty() || - setting.getValue().trim().isEmpty() ) { + if (setting.getName().trim().isEmpty() || + setting.getValue().trim().isEmpty()) { continue; } SettingsParameter old = getItemByAttr(setting.getName()); // If there is no item, persist the new setting - if ( old == null ) { + if (old == null) { em.persist(setting); } else { // Else update and merge the old item @@ -122,14 +124,14 @@ public class SettingsUtil { EntityTransaction trans = em.getTransaction(); trans.begin(); - for ( String name : settings ) { + for (String name : settings) { // TODO: Find a better way to do this // Could possibly wrap the ID into the settings list SettingsParameter setting = getItemByAttr(name); SettingsParameter managedSetting = em.find(SettingsParameter.class, setting.getId()); - if ( setting != null ) { + if (setting != null) { em.remove(managedSetting); } } @@ -142,7 +144,7 @@ public class SettingsUtil { public static Set getParamNames() { List settings = getCurrentSettings(); Set paramSet = new HashSet<>(); - for ( SettingsParameter s : settings ) { + for (SettingsParameter s : settings) { paramSet.add(s.getName()); } return paramSet; @@ -157,63 +159,38 @@ public class SettingsUtil { public static List getDefaultSettings() { List defaults = new ArrayList<>(); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_IMS, - SettingsConstants.ims,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_IMS_PORT, - SettingsConstants.imsPort,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_IMS_SSL, - SettingsConstants.imsSSL,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_IMS_TOKEN_CLASS, - SettingsConstants.imsTokenClass,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_AUTO_AUDIT_ENABLE, - SettingsConstants.autoAudit,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_THROTTLE_MAXAUDIT, - SettingsConstants.maxAudit,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_TIME, - SettingsConstants.throttleWait,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_BPS, - SettingsConstants.throttleBPS,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_SMTP_SERVER, - SettingsConstants.mailServer,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_FROM, - SettingsConstants.mailFrom,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_USER_AUTH, - SettingsConstants.authManagement,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_4J_APPENDER, - SettingsConstants.log4JA1,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_4J_BACKUP_INDEX, - SettingsConstants.log4JA1MaxBackupIndex,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_4J_CLASS, - SettingsConstants.log4JLoggerUMIACS,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_4J_CONV_PAT, - SettingsConstants.log4JA1layoutConversationPattern,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_4J_FILE, - SettingsConstants.log4JA1File,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_4J_FILE_SIZE, - SettingsConstants.log4JA1MaxFileSize,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_4J_IRODS, - SettingsConstants.log4JLoggerIrods,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_4J_LAYOUT, - SettingsConstants.log4JA1Layout,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_4J_ROOT_LOGGER, - SettingsConstants.log4JRootLogger,false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_INGEST, - SettingsConstants.maxIngestThreads, false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_AUDIT_ONLY, - SettingsConstants.auditOnly, false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_AUDIT_SAMPLE, - SettingsConstants.auditSample, false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_AUDIT_BLOCKING, - SettingsConstants.auditBlocking, false)); - defaults.add(new SettingsParameter(SettingsConstants.PARAM_AUDIT_MAX_BLOCK_TIME, - SettingsConstants.auditMaxBlockTime, false)); + defaults.add(new SettingsParameter(PARAM_IMS, ims)); + defaults.add(new SettingsParameter(PARAM_IMS_PORT, imsPort)); + defaults.add(new SettingsParameter(PARAM_IMS_SSL, imsSSL)); + defaults.add(new SettingsParameter(PARAM_IMS_TOKEN_CLASS, imsTokenClass)); + defaults.add(new SettingsParameter(PARAM_AUTO_AUDIT_ENABLE, autoAudit)); + defaults.add(new SettingsParameter(PARAM_THROTTLE_MAXAUDIT, maxAudit)); + defaults.add(new SettingsParameter(PARAM_TIME, throttleWait)); + defaults.add(new SettingsParameter(PARAM_BPS, throttleBPS)); + defaults.add(new SettingsParameter(PARAM_SMTP_SERVER, mailServer)); + defaults.add(new SettingsParameter(PARAM_FROM, mailFrom)); + defaults.add(new SettingsParameter(PARAM_USER_AUTH, authManagement)); + defaults.add(new SettingsParameter(PARAM_4J_APPENDER, log4JA1)); + defaults.add(new SettingsParameter(PARAM_4J_BACKUP_INDEX, log4JA1MaxBackupIndex)); + defaults.add(new SettingsParameter(PARAM_4J_CLASS, log4JLoggerUMIACS)); + defaults.add(new SettingsParameter(PARAM_4J_CONV_PAT, log4JA1layoutConversationPattern)); + defaults.add(new SettingsParameter(PARAM_4J_FILE, log4JA1File)); + defaults.add(new SettingsParameter(PARAM_4J_FILE_SIZE, log4JA1MaxFileSize)); + defaults.add(new SettingsParameter(PARAM_4J_IRODS, log4JLoggerIrods)); + defaults.add(new SettingsParameter(PARAM_4J_LAYOUT, log4JA1Layout)); + defaults.add(new SettingsParameter(PARAM_4J_ROOT_LOGGER, log4JRootLogger)); + defaults.add(new SettingsParameter(PARAM_INGEST, maxIngestThreads)); + defaults.add(new SettingsParameter(PARAM_AUDIT_ONLY, auditOnly)); + defaults.add(new SettingsParameter(PARAM_AUDIT_SAMPLE, auditSample)); + defaults.add(new SettingsParameter(PARAM_AUDIT_BLOCKING, auditBlocking)); + defaults.add(new SettingsParameter(PARAM_IMS_MAX_RETRY, imsMaxRetry)); + defaults.add(new SettingsParameter(PARAM_IMS_RESET_TIMEOUT, imsResetTimeout)); return defaults; } /** - * - * @param c The collection to query + * @param c The collection to query * @param attr The attribute to query for * @return true if collection, settings are not null and parameter is "true" */ @@ -256,27 +233,27 @@ public class SettingsUtil { SettingsParameter s = null; // Host - q.setParameter("attr", SettingsConstants.PARAM_IMS); + q.setParameter("attr", PARAM_IMS); s = (SettingsParameter) q.getSingleResult(); AuditThreadFactory.setIMS(s.getValue()); // Port - q.setParameter("attr", SettingsConstants.PARAM_IMS_PORT); + q.setParameter("attr", PARAM_IMS_PORT); s = (SettingsParameter) q.getSingleResult(); AuditThreadFactory.setImsPort(Integer.parseInt(s.getValue())); // SSL - q.setParameter("attr", SettingsConstants.PARAM_IMS_SSL); + q.setParameter("attr", PARAM_IMS_SSL); s = (SettingsParameter) q.getSingleResult(); AuditThreadFactory.setSSL(Boolean.valueOf(s.getValue())); // Token Class - q.setParameter("attr", SettingsConstants.PARAM_IMS_TOKEN_CLASS); + q.setParameter("attr", PARAM_IMS_TOKEN_CLASS); s = (SettingsParameter) q.getSingleResult(); AuditThreadFactory.setTokenClass(s.getValue()); // Audit Only - q.setParameter("attr", SettingsConstants.PARAM_AUDIT_ONLY); + q.setParameter("attr", PARAM_AUDIT_ONLY); s = (SettingsParameter) q.getSingleResult(); AuditThreadFactory.setAuditOnly(Boolean.valueOf(s.getValue())); @@ -291,32 +268,39 @@ public class SettingsUtil { */ // Mail Server - q.setParameter("attr", SettingsConstants.PARAM_SMTP_SERVER); + q.setParameter("attr", PARAM_SMTP_SERVER); s = (SettingsParameter) q.getSingleResult(); SchedulerContextListener.setMailServer(s.getValue()); // Mail From - q.setParameter("attr", SettingsConstants.PARAM_FROM); + q.setParameter("attr", PARAM_FROM); s = (SettingsParameter) q.getSingleResult(); SchedulerContextListener.setMailFrom(s.getValue()); // Audit Blocking - q.setParameter("attr", SettingsConstants.PARAM_AUDIT_BLOCKING); + q.setParameter("attr", PARAM_AUDIT_BLOCKING); s = (SettingsParameter) q.getSingleResult(); AuditThreadFactory.setBlocking(Boolean.valueOf(s.getValue())); - // Max block time - q.setParameter("attr", SettingsConstants.PARAM_AUDIT_MAX_BLOCK_TIME); + q.setParameter("attr", PARAM_IMS_MAX_RETRY); + s = (SettingsParameter) q.getSingleResult(); + if (Strings.isNonNegativeInt(s.getValue())) { + AuditThreadFactory.setImsRetryAttempts(Integer.parseInt(s.getValue())); + } + + q.setParameter("attr", PARAM_IMS_RESET_TIMEOUT); s = (SettingsParameter) q.getSingleResult(); - AuditThreadFactory.setMaxBlockTime(Integer.parseInt(s.getValue())); + if (Strings.isNonNegativeInt(s.getValue())) { + AuditThreadFactory.setImsResetTimeout(Integer.parseInt(s.getValue())); + } } /** * Get a SettingsParameter by its attribute name or set the default value and persist it * - * @param attr the Attribute to query for + * @param attr the Attribute to query for * @param defaultValue the default value of the attribute - * @param em the EntityManager to query with + * @param em the EntityManager to query with * @return the queried SettingsParameter */ public static SettingsParameter getOrDefault(String attr, -- GitLab From 8255ab65fd586361e20eaf8d3f59f8b4086cd403 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Thu, 18 Apr 2019 17:46:21 -0400 Subject: [PATCH 27/49] Add imsMaxRetry and imsResetTimeout --- .../edu/umiacs/ace/ims/api/IMSService.java | 44 ++++++++++++------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSService.java b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSService.java index e7cfb6c..d5ee408 100644 --- a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSService.java +++ b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSService.java @@ -64,19 +64,23 @@ public final class IMSService { private static final Logger print = Logger.getLogger(IMSService.class); + private static final int DEFAULT_RETRY = 3; private static final int DEFAULT_PORT = 8080; + private static final int DEFAULT_TIMEOUT = 5000; private static final String DEFAULT_PATH = "/ace-ims/IMSWebService?wsdl"; private static final boolean DEFAULT_SSL = false; private static final boolean DEFAULT_BLOCKING = false; + private boolean blocking; - private int maxBlockTime; + private final int imsRetryAttempts; + private final int imsResetTimeout; private IMSWebService port; - private IMSService(URL url, boolean blocking, int maxBlockTimeMinutes) { + private IMSService(URL url, boolean blocking, int imsRetryAttempts, int imsResetTimeout) { this.blocking = blocking; - // todo: move from block time to attempts - this.maxBlockTime = (maxBlockTimeMinutes < 0) ? 0 : maxBlockTimeMinutes; + this.imsRetryAttempts = imsRetryAttempts; + this.imsResetTimeout = imsResetTimeout; try { IMSWebService_Service service = new IMSWebService_Service(url, @@ -103,10 +107,12 @@ public final class IMSService { String path, boolean ssl, boolean blocking, - int maxBlockTimeMinutes) { + int imsRetryAttempts, + int imsResetTimeout) { Check.notEmpty("hostName", hostName); Check.isPositive("portNumber", portNumber); - Check.notNegative("maxBlockTime", maxBlockTimeMinutes); + Check.notNegative("imsRetryAttempts", imsRetryAttempts); + Check.notNegative("imsResetTimeout", imsResetTimeout); Check.notEmpty("path", path); if (!path.startsWith("/")) { @@ -123,7 +129,7 @@ public final class IMSService { } try { - return new IMSService(new URL(url.toString()), blocking, maxBlockTimeMinutes); + return new IMSService(new URL(url.toString()), blocking, imsRetryAttempts, imsResetTimeout); } catch (MalformedURLException mue) { throw new IllegalArgumentException("Invalid URL: " + url); } @@ -142,19 +148,25 @@ public final class IMSService { DEFAULT_PATH, DEFAULT_SSL, DEFAULT_BLOCKING, - 0); + DEFAULT_RETRY, + DEFAULT_TIMEOUT); } public static IMSService connect(String hostName, int port) { - return IMSService.connect(hostName, port, DEFAULT_PATH, DEFAULT_SSL, DEFAULT_BLOCKING, 0); + return IMSService.connect(hostName, port, DEFAULT_PATH, DEFAULT_SSL, DEFAULT_BLOCKING, DEFAULT_RETRY, DEFAULT_TIMEOUT); } public static IMSService connect(String hostName, int port, boolean ssl) { - return IMSService.connect(hostName, port, DEFAULT_PATH, ssl, DEFAULT_BLOCKING, 0); + return IMSService.connect(hostName, port, DEFAULT_PATH, ssl, DEFAULT_BLOCKING, DEFAULT_RETRY, DEFAULT_TIMEOUT); } - public static IMSService connect(String hostName, int port, boolean ssl, boolean blocking, int maxBlockTimeMinutes) { - return IMSService.connect(hostName, port, DEFAULT_PATH, ssl, blocking, maxBlockTimeMinutes); + public static IMSService connect(String hostName, + int port, + boolean ssl, + boolean blocking, + int imsRetryAttempts, + int imsResetTimeout) { + return IMSService.connect(hostName, port, DEFAULT_PATH, ssl, blocking, imsRetryAttempts, imsResetTimeout); } public List requestWitnessProofForRounds(List rounds) { @@ -293,10 +305,10 @@ public final class IMSService { // first attempt always goes through (attempt == 0) // when blocking, make sure that attempt is less than maxAttempts - while (!done && (attempt == 0 || (attempt <= maxBlockTime && blocking))) { + while (!done && (attempt == 0 || (attempt <= imsRetryAttempts && blocking))) { if (attempt > 0) { try { - TimeUnit.SECONDS.sleep(30); + TimeUnit.MILLISECONDS.sleep(imsResetTimeout); } catch (InterruptedException exception) { // we were probably cancelled, get out of here now IMSConnectionException imsException = new IMSConnectionException( @@ -307,8 +319,8 @@ public final class IMSService { } } - print.info("calling supplier: attempt = " + attempt - + ", maxBlock = " + maxBlockTime + print.trace("calling supplier: attempt = " + attempt + + ", maxBlock = " + imsRetryAttempts + ", blocking = " + blocking); imsResult = supplier.get(); done = imsResult.getResult().isPresent(); -- GitLab From b989cfc6a28ff600e7f39683d0600304cdfe5a0a Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Fri, 19 Apr 2019 14:49:23 -0400 Subject: [PATCH 28/49] Lint --- .../monitor/settings/DeleteSettingsServlet.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/DeleteSettingsServlet.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/DeleteSettingsServlet.java index aa07688..f5b9c26 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/DeleteSettingsServlet.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/DeleteSettingsServlet.java @@ -7,15 +7,16 @@ package edu.umiacs.ace.monitor.settings; import edu.umiacs.ace.util.EntityManagerServlet; import edu.umiacs.util.Strings; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; + import javax.persistence.EntityManager; import javax.servlet.RequestDispatcher; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; /** * @@ -25,10 +26,11 @@ public class DeleteSettingsServlet extends EntityManagerServlet{ public static final String PARAM_SETTING = "setting"; @Override - protected void processRequest(HttpServletRequest request, HttpServletResponse - response, EntityManager em) throws ServletException, IOException { + protected void processRequest(HttpServletRequest request, + HttpServletResponse response, + EntityManager em) throws ServletException, IOException { - List settingsToDelete = new ArrayList(); + List settingsToDelete = new ArrayList<>(); RequestDispatcher dispatcher = null; if ( !Strings.isEmpty(request.getParameter(PARAM_SETTING)) ) { -- GitLab From 2f6ecc46de09bce8e5abacaa50c4f45e4a5ca1f6 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Fri, 19 Apr 2019 14:49:57 -0400 Subject: [PATCH 29/49] Additional settings migration for audit.max.block.time --- .../AutoAuditMigrationContextListener.java | 101 +++++++++++------- 1 file changed, 64 insertions(+), 37 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/AutoAuditMigrationContextListener.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/AutoAuditMigrationContextListener.java index 21d7584..882147e 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/AutoAuditMigrationContextListener.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/AutoAuditMigrationContextListener.java @@ -2,7 +2,9 @@ package edu.umiacs.ace.monitor.settings; import edu.umiacs.ace.util.PersistUtil; import edu.umiacs.sql.SQL; +import edu.umiacs.util.Check; import org.apache.log4j.Logger; +import org.apache.log4j.NDC; import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextListener; @@ -12,45 +14,77 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import static edu.umiacs.ace.monitor.settings.SettingsConstants.PARAM_IMS_MAX_RETRY; +import static edu.umiacs.ace.monitor.settings.SettingsConstants.PARAM_IMS_RESET_TIMEOUT; +import static edu.umiacs.ace.monitor.settings.SettingsConstants.imsMaxRetry; +import static edu.umiacs.ace.monitor.settings.SettingsConstants.imsResetTimeout; + /** - * Context listener to migrate from auto.audit.disable to auto.audit.enable - * + * Context listener to migrate audit/ims settings + * auto.audit.disable to auto.audit.enable + * audit.max.block.time to ims.max.retry + * add ims.reset.timeout + *

* Created by shake on 4/5/17. */ public class AutoAuditMigrationContextListener implements ServletContextListener { private static final Logger LOG = Logger.getLogger(AutoAuditMigrationContextListener.class); - private ResultSet set; - private Connection conn; - private PreparedStatement statement; - @Override public void contextInitialized(ServletContextEvent servletContextEvent) { - Connection conn = null; + NDC.push("[MIGRATION] "); DataSource ds = PersistUtil.getDataSource(); - try { - conn = ds.getConnection(); - migrate(conn); + try (Connection conn = ds.getConnection()) { + migrateAutoAudit(conn); + migrateAuditBlocking(conn); } catch (Exception e) { - LOG.error("[MIGRATION] Error migrating audo.audit.disable setting", e); + LOG.error("Error migrating audit settings", e); } finally { - release(); - + NDC.pop(); } } - private void migrate(Connection conn) throws SQLException { - PreparedStatement statement = conn.prepareStatement("SELECT id, value, custom FROM system_settings WHERE attr = 'auto.audit.disable'"); - ResultSet set = statement.executeQuery(); - while (set.next()) { - Long id = set.getLong(1); - Boolean value = Boolean.valueOf(set.getString(2)); - Boolean custom = set.getBoolean(3); - if (!custom) { - LOG.info("[MIGRATION] Found auto.audit setting to migrate from " + value + " -> " + !value); - createNew(conn, !value); + /** + * Migrate from auto.audit.disable to auto.audit.enable + * + * @param conn The database connection + * @throws SQLException if there's an exception communicating with the database + */ + private void migrateAutoAudit(Connection conn) throws SQLException { + String query = "SELECT id, value, custom FROM system_settings WHERE attr = 'auto.audit.disable'"; + try (PreparedStatement statement = conn.prepareStatement(query); + ResultSet set = statement.executeQuery()) { + while (set.next()) { + Long id = set.getLong(1); + Boolean value = Boolean.valueOf(set.getString(2)); + Boolean custom = set.getBoolean(3); + if (!custom) { + LOG.info("Found auto.audit setting to migrate from " + value + " -> " + !value); + createNew(conn, "auto.audit.enable", String.valueOf(!value)); + delete(conn, id); + } + } + } + } + + /** + * Check if audit.max.block.time exists. If true, add ims.max.retry and ims.reset.timeout and + * remove audit.max.block.time + * + * @param conn The database connection + * @throws SQLException if there's an exception communicating with the database + */ + private void migrateAuditBlocking(Connection conn) throws SQLException { + String query = "SELECT id FROM system_settings WHERE attr = 'audit.max.block.time'"; + try (PreparedStatement statement = conn.prepareStatement(query); + ResultSet set = statement.executeQuery()) { + while (set.next()) { + Long id = set.getLong(1); + LOG.info("Found audit.max.block.time setting to migrate to ims.max.retry"); + createNew(conn, PARAM_IMS_MAX_RETRY, imsMaxRetry); + createNew(conn, PARAM_IMS_RESET_TIMEOUT, imsResetTimeout); delete(conn, id); } } @@ -62,21 +96,14 @@ public class AutoAuditMigrationContextListener implements ServletContextListener SQL.release(statement); } - private void createNew(Connection conn, Boolean value) throws SQLException { - PreparedStatement statement = conn.prepareStatement("INSERT INTO system_settings VALUES (DEFAULT, 'auto.audit.enable','" + value.toString() + "',0)"); - statement.executeUpdate(); - SQL.release(statement); - } + private void createNew(Connection conn, String attr, String value) throws SQLException { + Check.notNull("attr", attr); + Check.notNull("value", value); - private void release() { - if (conn != null) { - SQL.release(conn); - } - if (statement != null) { - SQL.release(statement); - } - if (set != null) { - SQL.release(set); + String query = "INSERT INTO system_settings VALUES (DEFAULT, '%s', '%s', 0)"; + try (PreparedStatement statement = + conn.prepareStatement(String.format(query, attr, value))) { + statement.executeUpdate(); } } -- GitLab From 7977bebf4e354911405d041ae4274918c82620b1 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Fri, 19 Apr 2019 15:08:20 -0400 Subject: [PATCH 30/49] Display ims retry settings --- ace-am/src/main/webapp/settings.jsp | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/ace-am/src/main/webapp/settings.jsp b/ace-am/src/main/webapp/settings.jsp index 8b0c363..0556c98 100644 --- a/ace-am/src/main/webapp/settings.jsp +++ b/ace-am/src/main/webapp/settings.jsp @@ -39,16 +39,6 @@

-
-
Audit IMS Blocking:
-
-
-
-
-
Audit Max Block Time:
-
-
-
Audit Wait Time:
@@ -89,6 +79,21 @@
+
+
IMS Audit Blocking:
+
+
+
+
+
IMS Audit Max Retry Attempts:
+
+
+
+
+
IMS Audit Reset Timeout:
+
+
+
User Management:
-- GitLab From 3e67638933a50362f2cf77e239832720c2b6cecc Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Fri, 19 Apr 2019 17:16:00 -0400 Subject: [PATCH 31/49] New form classes --- ace-am/src/main/webapp/style.css | 51 +++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/ace-am/src/main/webapp/style.css b/ace-am/src/main/webapp/style.css index 7c3af0d..2dbd690 100644 --- a/ace-am/src/main/webapp/style.css +++ b/ace-am/src/main/webapp/style.css @@ -284,7 +284,6 @@ vertical-align: top; .form-input { border: 1px solid #ccc; width: 100%; - /* height: 20px; */ padding: 3px 8px; margin-left: -1px; margin-right: 5px; @@ -297,6 +296,56 @@ vertical-align: top; margin-right: 5px; } +.form-input-settings { + width: 100%; + max-width: 100%; + border: 1px solid #363636; + border-radius: 1px; + height: 1.5rem; + padding: 0.275rem; + box-sizing: border-box; +} + +.form-legend { + font-size: 1.25rem; + font-weight: bolder; +} + +.form-field:not(:last-child) { + margin-bottom: .75rem; +} + +.form-label { + color: #363636; + display: block; + font-size: 1rem; + font-weight: 700; + text-align: center; +} + +.form-label:not(:last-child) { + margin-bottom: .5em; +} + +.form-help { + display: block; + font-size: 0.7rem; + margin-bottom: .25rem; + margin-top: 0.25rem; + text-align: center; +} + +/* +from bulma - not sure if we'll need this +.control { + box-sizing: border-box; + clear: both; + font-size: 1rem; + position: relative; + text-align: left; +} +*/ + .btn { padding: 2px; width: 75px; -- GitLab From 934dc3a731b4270d1e36bd96d21d79a1caa6066c Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Fri, 19 Apr 2019 17:23:19 -0400 Subject: [PATCH 32/49] Update settings layout --- ace-am/src/main/webapp/settings.jsp | 422 ++++++++++++++++------------ 1 file changed, 247 insertions(+), 175 deletions(-) diff --git a/ace-am/src/main/webapp/settings.jsp b/ace-am/src/main/webapp/settings.jsp index 0556c98..bd61f05 100644 --- a/ace-am/src/main/webapp/settings.jsp +++ b/ace-am/src/main/webapp/settings.jsp @@ -1,190 +1,262 @@ -<%-- +<%-- Document : settings Created on : Jul 30, 2012, 4:07:16 PM Author : shake - TODO: Enable sampling when we have it properly implemented TODO: The Settings should come from a bean or smth so we can loop similar to the custom settings --%> -<%@page contentType="text/html" pageEncoding="UTF-8"%> -<%@taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"%> +<%@page contentType="text/html" pageEncoding="UTF-8" %> +<%@taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c" %> +"http://www.w3.org/TR/html4/loose.dtd"> + - - - System Settings - - - - -
-
- -

System Settings

- -
-
Mail Server:
-
-
-
-
-
Mail From:
-
-
-
-
-
Max Audit:
-
-
-
-
-
Audit Wait Time:
-
-
-
-
-
Audit BPS:
-
-
-
-
-
Enable Auto Audit:
-
-
-
-
-
Audit Only Mode:
-
-
-
-
-
IMS Host:
-
-
-
-
-
IMS Port:
-
-
-
-
-
IMS SSL:
-
-
-
-
-
IMS Token Class:
-
-
-
-
-
IMS Audit Blocking:
-
-
-
-
-
IMS Audit Max Retry Attempts:
-
-
-
-
-
IMS Audit Reset Timeout:
-
-
-
-
-
User Management:
-
-
-
-
-
Max Ingest Threads:
-
-
-
-
-
Log Location:
-
-
-
- - -
-
Log Type:
-
-
-
- - -
-
Log Type:
-
-
-
-
-
Log File Size:
-
-
-
-
-
Log Backup Index:
-
-
-
- - -
-
-
Generic logging properties, - change only if you know what you are doing
-
-
-
Root Logger:
-
-
-
-
Pattern Layout:
-
-
-
-
Conversion Pattern:
-
-
-
-
IRods:
-
-
-
-
UMIACS Connection:
-
+ + + System Settings + + + + + +
+ System Settings +
+ + + Mail server to use when mailing reports. + You will need to set this if you want reports to be mailed properly. + + +
+
+ + + Set this e-mail address to an address e-mail should originate from + + +
+
+ + Maximum number of running audits + + +
+
+ + Minimum time between file reads in milliseconds + +
+
+ + + Maximum bytes per second per running audit: default = 0 = unlimited + + +
+
+ + Enable automated auditing + +
+
+ + + If true, do not attempt to register new items or contact the IMS during auditing. + Default value of false is recommended. + + +
+
+ + + IMS hostname to use. Unless you deployed your own IMS, + this should probably not be changed. + + +
+
+ + + Port the IMS Host runs on. + Typically 80 for http or 443 for https. + + +
+
+ + Use SSL when connecting to the IMS + +
+
+ + Token class for the IMS host to use + +
+
+ + Block audits when connectivity to the IMS is lost + +
+
+ + Maximum amount of retries when IMS failure occur + +
+
+ + + Amount of time to wait before retrying after an IMS failure + + +
+
+ + + Set this to true to disable internal user management. This should only be used in + conjunction with changes to the Authentication realm listed above. + + +
+
+ + + The maximum number of threads allowed for ingesting token stores. + + +
+
+ + The location of your logfile + +
+ + +
+ + + Log everything to one file, will grow to infinity + +
-
-
IRods Connection:
-
+ + +
+ + + Use a rolling log file + +
-
-
-
Custom Settings:
+
+ + + Maximum file size allowed for your log + +
- -
-
${item.key}
-
-
- -
-
Add setting
-
Delete settings
+
+ + + Maximum backup index allowed for your log + +
-
-
- - - Cancel - + + +
+
+
Generic logging properties, change only if you know what you are doing
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+
Custom Settings:
+
+ + +
+ + +
+ +
+
+ Add setting +
+
+ Delete settings +
+
+
+
+ + + Cancel + - - + + -- GitLab From af74cc9ef52527c90339f6772bfa80efaf7cfa45 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Fri, 19 Apr 2019 17:54:06 -0400 Subject: [PATCH 33/49] Update submission buttons --- ace-am/src/main/webapp/settings.jsp | 8 ++++---- ace-am/src/main/webapp/style.css | 29 +++++++++++++++++++---------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/ace-am/src/main/webapp/settings.jsp b/ace-am/src/main/webapp/settings.jsp index bd61f05..d21b5a7 100644 --- a/ace-am/src/main/webapp/settings.jsp +++ b/ace-am/src/main/webapp/settings.jsp @@ -251,10 +251,10 @@
-
- - - Cancel +
+ + +
diff --git a/ace-am/src/main/webapp/style.css b/ace-am/src/main/webapp/style.css index 2dbd690..24bc5f5 100644 --- a/ace-am/src/main/webapp/style.css +++ b/ace-am/src/main/webapp/style.css @@ -335,23 +335,32 @@ vertical-align: top; text-align: center; } -/* -from bulma - not sure if we'll need this -.control { - box-sizing: border-box; - clear: both; - font-size: 1rem; - position: relative; - text-align: left; +.form-group { + margin-top: .625rem; + margin-bottom: .5rem; } -*/ .btn { padding: 2px; width: 75px; height: 25px; - border: 1px solid #e8e8e8; + border: 1px solid #939393; border-radius: 0; margin-left: 2px; margin-top: 2px; + color: #363636; + background-color: hsl(217, 100%, 82%); +} + +.is-right { + right: 0 +} + +.btn.is-secondary { + color: #363636; + background-color: #fff; +} + +.btn:hover { + border-color: #363636; } -- GitLab From 1d47423d5be985207ca501bdfeb56f3c8aea506b Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Fri, 19 Apr 2019 18:00:13 -0400 Subject: [PATCH 34/49] Remove is-right --- ace-am/src/main/webapp/settings.jsp | 2 +- ace-am/src/main/webapp/style.css | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/ace-am/src/main/webapp/settings.jsp b/ace-am/src/main/webapp/settings.jsp index d21b5a7..0cd17f6 100644 --- a/ace-am/src/main/webapp/settings.jsp +++ b/ace-am/src/main/webapp/settings.jsp @@ -253,7 +253,7 @@
- +
diff --git a/ace-am/src/main/webapp/style.css b/ace-am/src/main/webapp/style.css index 24bc5f5..ac6ddbb 100644 --- a/ace-am/src/main/webapp/style.css +++ b/ace-am/src/main/webapp/style.css @@ -350,10 +350,7 @@ vertical-align: top; margin-top: 2px; color: #363636; background-color: hsl(217, 100%, 82%); -} - -.is-right { - right: 0 + font-size: 0.75rem; } .btn.is-secondary { -- GitLab From e850cf0825c23fb6b227b4f862944540d22fc8af Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Fri, 19 Apr 2019 18:07:30 -0400 Subject: [PATCH 35/49] Add btn-danger --- ace-am/src/main/webapp/style.css | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ace-am/src/main/webapp/style.css b/ace-am/src/main/webapp/style.css index ac6ddbb..c49d282 100644 --- a/ace-am/src/main/webapp/style.css +++ b/ace-am/src/main/webapp/style.css @@ -358,6 +358,11 @@ vertical-align: top; background-color: #fff; } +.btn.btn-danger { + color: #fff; + background-color: #dc3545; +} + .btn:hover { border-color: #363636; } -- GitLab From 91172f6fcb4f736a1a6d66de7f483ea22d71d887 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Mon, 22 Apr 2019 12:37:30 -0400 Subject: [PATCH 36/49] Small updates to form input style --- ace-am/src/main/webapp/style.css | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ace-am/src/main/webapp/style.css b/ace-am/src/main/webapp/style.css index c49d282..b9704f3 100644 --- a/ace-am/src/main/webapp/style.css +++ b/ace-am/src/main/webapp/style.css @@ -300,10 +300,20 @@ vertical-align: top; width: 100%; max-width: 100%; border: 1px solid #363636; - border-radius: 1px; + border-radius: 3px; height: 1.5rem; padding: 0.275rem; box-sizing: border-box; + box-shadow: inset 0 1px 2px rgba(10, 10, 10, 0.1); +} + +.form-input-settings:hover { + border-color: #076475; +} + +.form-input-settings:focus { + border-color: #5883C9; + box-shadow: 0 0 0.125em rgb(77, 144, 255); } .form-legend { -- GitLab From 37c88692eda794189fd3e7dd981f2186e8ae7a47 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Mon, 22 Apr 2019 12:43:00 -0400 Subject: [PATCH 37/49] Specify unit of time --- ace-am/src/main/webapp/settings.jsp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ace-am/src/main/webapp/settings.jsp b/ace-am/src/main/webapp/settings.jsp index 0cd17f6..5f06d9d 100644 --- a/ace-am/src/main/webapp/settings.jsp +++ b/ace-am/src/main/webapp/settings.jsp @@ -123,7 +123,7 @@
- Amount of time to wait before retrying after an IMS failure + Amount of time to wait in milliseconds before retrying after an IMS failure -- GitLab From f7d22d472cb15366bd1c841f7092c8c757ad96e6 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Mon, 22 Apr 2019 14:45:24 -0400 Subject: [PATCH 38/49] Allow settings to be missing when reloading --- .../ace/monitor/settings/SettingsUtil.java | 40 ++++++++----------- 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsUtil.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsUtil.java index c9d885a..5ec9f55 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsUtil.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsUtil.java @@ -12,9 +12,7 @@ import javax.persistence.NoResultException; import javax.persistence.Query; import javax.persistence.TypedQuery; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import static edu.umiacs.ace.monitor.settings.SettingsConstants.*; @@ -37,7 +35,7 @@ public class SettingsUtil { try { return (SettingsParameter) q.getSingleResult(); } catch (NoResultException ex) { - // zzz + // ignore, return null } finally { em.close(); } @@ -140,16 +138,6 @@ public class SettingsUtil { em.close(); } - // Get the names of all current settings - public static Set getParamNames() { - List settings = getCurrentSettings(); - Set paramSet = new HashSet<>(); - for (SettingsParameter s : settings) { - paramSet.add(s.getName()); - } - return paramSet; - } - /** * Get the default values for all settings @@ -234,27 +222,27 @@ public class SettingsUtil { // Host q.setParameter("attr", PARAM_IMS); - s = (SettingsParameter) q.getSingleResult(); + s = getOrDefault(PARAM_IMS, ims, em); AuditThreadFactory.setIMS(s.getValue()); // Port q.setParameter("attr", PARAM_IMS_PORT); - s = (SettingsParameter) q.getSingleResult(); + s = getOrDefault(PARAM_IMS_PORT, imsPort, em); AuditThreadFactory.setImsPort(Integer.parseInt(s.getValue())); // SSL q.setParameter("attr", PARAM_IMS_SSL); - s = (SettingsParameter) q.getSingleResult(); + s = getOrDefault(PARAM_IMS_SSL, imsSSL, em); AuditThreadFactory.setSSL(Boolean.valueOf(s.getValue())); // Token Class q.setParameter("attr", PARAM_IMS_TOKEN_CLASS); - s = (SettingsParameter) q.getSingleResult(); + s = getOrDefault(PARAM_IMS_TOKEN_CLASS, imsTokenClass, em); AuditThreadFactory.setTokenClass(s.getValue()); // Audit Only q.setParameter("attr", PARAM_AUDIT_ONLY); - s = (SettingsParameter) q.getSingleResult(); + s = getOrDefault(PARAM_AUDIT_ONLY, auditOnly, em); AuditThreadFactory.setAuditOnly(Boolean.valueOf(s.getValue())); // Max Audits @@ -269,27 +257,27 @@ public class SettingsUtil { // Mail Server q.setParameter("attr", PARAM_SMTP_SERVER); - s = (SettingsParameter) q.getSingleResult(); + s = getOrDefault(PARAM_SMTP_SERVER, mailServer, em); SchedulerContextListener.setMailServer(s.getValue()); // Mail From q.setParameter("attr", PARAM_FROM); - s = (SettingsParameter) q.getSingleResult(); + s = getOrDefault(PARAM_FROM, mailFrom, em); SchedulerContextListener.setMailFrom(s.getValue()); // Audit Blocking q.setParameter("attr", PARAM_AUDIT_BLOCKING); - s = (SettingsParameter) q.getSingleResult(); + s = getOrDefault(PARAM_AUDIT_BLOCKING, auditBlocking, em); AuditThreadFactory.setBlocking(Boolean.valueOf(s.getValue())); q.setParameter("attr", PARAM_IMS_MAX_RETRY); - s = (SettingsParameter) q.getSingleResult(); + s = getOrDefault(PARAM_IMS_MAX_RETRY, imsMaxRetry, em); if (Strings.isNonNegativeInt(s.getValue())) { AuditThreadFactory.setImsRetryAttempts(Integer.parseInt(s.getValue())); } q.setParameter("attr", PARAM_IMS_RESET_TIMEOUT); - s = (SettingsParameter) q.getSingleResult(); + s = getOrDefault(PARAM_IMS_RESET_TIMEOUT, imsResetTimeout, em); if (Strings.isNonNegativeInt(s.getValue())) { AuditThreadFactory.setImsResetTimeout(Integer.parseInt(s.getValue())); } @@ -307,7 +295,8 @@ public class SettingsUtil { String defaultValue, EntityManager em) { SettingsParameter result = new SettingsParameter(attr, defaultValue, false); - TypedQuery q = em.createNamedQuery("SettingsParameter.getAttr", + TypedQuery q = em.createNamedQuery( + "SettingsParameter.getAttr", SettingsParameter.class); q.setParameter("attr", attr); @@ -315,7 +304,10 @@ public class SettingsUtil { result = q.getSingleResult(); // just in case if (result.getValue() == null || result.getValue().isEmpty()) { + em.getTransaction().begin(); result.setValue(defaultValue); + em.merge(result); + em.getTransaction().commit(); } } catch (NoResultException ex) { em.getTransaction().begin(); -- GitLab From f8083d35103f5636e0169b02cd5269d948122eea Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Mon, 22 Apr 2019 14:49:57 -0400 Subject: [PATCH 39/49] Clean up --- .../ace/monitor/settings/SettingsServlet.java | 6 ++---- .../umiacs/ace/monitor/settings/SettingsUtil.java | 13 +------------ 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsServlet.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsServlet.java index e004e5c..97055e9 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsServlet.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsServlet.java @@ -63,10 +63,8 @@ public class SettingsServlet extends EntityManagerServlet { } } - Map settingsMap = - settingsToMap(SettingsUtil.getCurrentSettings()); - Map customMap = - settingsToMap(SettingsUtil.getCustomSettings()); + Map settingsMap = settingsToMap(SettingsUtil.getCurrentSettings()); + Map customMap = settingsToMap(SettingsUtil.getCustomSettings()); request.setAttribute("currSettings", settingsMap); request.setAttribute("customSettings", customMap); diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsUtil.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsUtil.java index 5ec9f55..b422430 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsUtil.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsUtil.java @@ -217,31 +217,25 @@ public class SettingsUtil { // Update the settings our context listeners would normally do private static void reloadSettings() { EntityManager em = PersistUtil.getEntityManager(); - Query q = em.createNamedQuery("SettingsParameter.getAttr"); - SettingsParameter s = null; + SettingsParameter s; // Host - q.setParameter("attr", PARAM_IMS); s = getOrDefault(PARAM_IMS, ims, em); AuditThreadFactory.setIMS(s.getValue()); // Port - q.setParameter("attr", PARAM_IMS_PORT); s = getOrDefault(PARAM_IMS_PORT, imsPort, em); AuditThreadFactory.setImsPort(Integer.parseInt(s.getValue())); // SSL - q.setParameter("attr", PARAM_IMS_SSL); s = getOrDefault(PARAM_IMS_SSL, imsSSL, em); AuditThreadFactory.setSSL(Boolean.valueOf(s.getValue())); // Token Class - q.setParameter("attr", PARAM_IMS_TOKEN_CLASS); s = getOrDefault(PARAM_IMS_TOKEN_CLASS, imsTokenClass, em); AuditThreadFactory.setTokenClass(s.getValue()); // Audit Only - q.setParameter("attr", PARAM_AUDIT_ONLY); s = getOrDefault(PARAM_AUDIT_ONLY, auditOnly, em); AuditThreadFactory.setAuditOnly(Boolean.valueOf(s.getValue())); @@ -256,27 +250,22 @@ public class SettingsUtil { */ // Mail Server - q.setParameter("attr", PARAM_SMTP_SERVER); s = getOrDefault(PARAM_SMTP_SERVER, mailServer, em); SchedulerContextListener.setMailServer(s.getValue()); // Mail From - q.setParameter("attr", PARAM_FROM); s = getOrDefault(PARAM_FROM, mailFrom, em); SchedulerContextListener.setMailFrom(s.getValue()); // Audit Blocking - q.setParameter("attr", PARAM_AUDIT_BLOCKING); s = getOrDefault(PARAM_AUDIT_BLOCKING, auditBlocking, em); AuditThreadFactory.setBlocking(Boolean.valueOf(s.getValue())); - q.setParameter("attr", PARAM_IMS_MAX_RETRY); s = getOrDefault(PARAM_IMS_MAX_RETRY, imsMaxRetry, em); if (Strings.isNonNegativeInt(s.getValue())) { AuditThreadFactory.setImsRetryAttempts(Integer.parseInt(s.getValue())); } - q.setParameter("attr", PARAM_IMS_RESET_TIMEOUT); s = getOrDefault(PARAM_IMS_RESET_TIMEOUT, imsResetTimeout, em); if (Strings.isNonNegativeInt(s.getValue())) { AuditThreadFactory.setImsResetTimeout(Integer.parseInt(s.getValue())); -- GitLab From af79e3d1f64fca7a1462b15bbced2bdb23717785 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Mon, 22 Apr 2019 14:52:03 -0400 Subject: [PATCH 40/49] Set initial values to defaults --- .../ace/monitor/audit/AuditConfigurationContext.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditConfigurationContext.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditConfigurationContext.java index 3bdfd44..388b64a 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditConfigurationContext.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditConfigurationContext.java @@ -121,15 +121,17 @@ public final class AuditConfigurationContext implements ServletContextListener { String blocking = resultMap.getOrDefault(PARAM_AUDIT_BLOCKING, auditBlocking); AuditThreadFactory.setBlocking(Boolean.valueOf(blocking)); - int maxRetry = 0; + int maxRetry = Integer.parseInt(imsMaxRetry); String imsRetryString = resultMap.getOrDefault(PARAM_IMS_MAX_RETRY, imsMaxRetry); if (Strings.isNonNegativeInt(imsRetryString)) { maxRetry = Integer.parseInt(imsRetryString); } AuditThreadFactory.setImsRetryAttempts(maxRetry); - int resetTimeout = 0; - String imsResetTimeout = resultMap.getOrDefault(PARAM_IMS_RESET_TIMEOUT, SettingsConstants.imsResetTimeout); + int resetTimeout = Integer.parseInt(imsResetTimeout); + String imsResetTimeout = resultMap.getOrDefault( + PARAM_IMS_RESET_TIMEOUT, + SettingsConstants.imsResetTimeout); if (Strings.isNonNegativeInt(imsResetTimeout)) { resetTimeout = Integer.parseInt(imsResetTimeout); } -- GitLab From 0d231532ee9e220ca36a7dc1ffb4e19ed62447d7 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Mon, 22 Apr 2019 15:05:58 -0400 Subject: [PATCH 41/49] Logging --- .../src/main/java/edu/umiacs/ace/ims/api/IMSService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSService.java b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSService.java index d5ee408..8c5d6dc 100644 --- a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSService.java +++ b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/IMSService.java @@ -319,7 +319,7 @@ public final class IMSService { } } - print.trace("calling supplier: attempt = " + attempt + print.trace("Supplier: attempt = " + attempt + ", maxBlock = " + imsRetryAttempts + ", blocking = " + blocking); imsResult = supplier.get(); -- GitLab From 3bcb442b5488252e7b664461dc04b076d61caf8f Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Mon, 22 Apr 2019 15:45:25 -0400 Subject: [PATCH 42/49] Remove unnecessary imports --- .../edu/umiacs/ace/monitor/audit/AuditConfigurationContext.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditConfigurationContext.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditConfigurationContext.java index 388b64a..7b405d9 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditConfigurationContext.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditConfigurationContext.java @@ -56,8 +56,6 @@ import java.util.TimerTask; import java.util.stream.Collectors; import static edu.umiacs.ace.monitor.settings.SettingsConstants.*; -import static edu.umiacs.ace.monitor.settings.SettingsConstants.PARAM_IMS_MAX_RETRY; -import static edu.umiacs.ace.monitor.settings.SettingsConstants.imsMaxRetry; /** * Set the IMS for the AuditThread to use. Also, startup a background task -- GitLab From e4523e9bd4fc607ca9360b3634f80d0dc081e740 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Tue, 23 Apr 2019 13:13:58 -0400 Subject: [PATCH 43/49] Display settings in tabs --- ace-am/src/main/webapp/settings.jsp | 441 ++++++++++++++++------------ ace-am/src/main/webapp/style.css | 47 ++- 2 files changed, 294 insertions(+), 194 deletions(-) diff --git a/ace-am/src/main/webapp/settings.jsp b/ace-am/src/main/webapp/settings.jsp index 5f06d9d..3b8df1a 100644 --- a/ace-am/src/main/webapp/settings.jsp +++ b/ace-am/src/main/webapp/settings.jsp @@ -22,241 +22,296 @@
System Settings -
- - +
+ +
+ + +
+
+ + + Set this to true to disable internal user management. This should only be used in + conjunction with changes to the Authentication realm listed above. + + +
+
+ + + The maximum number of threads allowed for ingesting token stores. + + +
+ +
+ + Mail server to use when mailing reports. You will need to set this if you want reports to be mailed properly. - -
-
- - + +
+
+ + Set this e-mail address to an address e-mail should originate from - + +
-
- - Maximum number of running audits - -
-
- - Minimum time between file reads in milliseconds - -
-
- - +
+
+ + Maximum number of running audits + +
+
+ + Minimum time between file reads in milliseconds + +
+
+ + Maximum bytes per second per running audit: default = 0 = unlimited - -
-
- - Enable automated auditing - -
-
- - + +
+
+ + Enable automated auditing + +
+
+ + If true, do not attempt to register new items or contact the IMS during auditing. Default value of false is recommended. - + +
-
- - + +
+
+ + IMS hostname to use. Unless you deployed your own IMS, this should probably not be changed. - -
-
- - + +
+
+ + Port the IMS Host runs on. Typically 80 for http or 443 for https. - -
-
- - Use SSL when connecting to the IMS - +
+
+ + Use SSL when connecting to the IMS + -
-
- - Token class for the IMS host to use - -
-
- - Block audits when connectivity to the IMS is lost - -
-
- - Maximum amount of retries when IMS failure occur - -
-
- - + name="ims.ssl" value="${currSettings['ims.ssl']}"/> +
+
+ + Token class for the IMS host to use + +
+
+ + Block audits when connectivity to the IMS is lost + +
+
+ + Maximum amount of retries when IMS failure occur + +
+
+ + Amount of time to wait in milliseconds before retrying after an IMS failure - -
-
- - - Set this to true to disable internal user management. This should only be used in - conjunction with changes to the Authentication realm listed above. - - -
-
- - - The maximum number of threads allowed for ingesting token stores. - - -
-
- - The location of your logfile - + +
- - -
- - + + +
+
+ + The location of your logfile + +
+ + +
+ + Log everything to one file, will grow to infinity - -
- - -
- - + +
+ + +
+ + Use a rolling log file - -
-
- - + +
+
+ + Maximum file size allowed for your log - -
-
- - + +
+
+ + Maximum backup index allowed for your log - -
- - -
-
-
Generic logging properties, change only if you know what you are doing
-
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
-
-
Custom Settings:
-
- - + +
+ + +
+
+
Generic logging properties, change only if you know what you are doing
+
- - + +
- -
-
- Add setting +
+ +
-
- Delete settings +
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ +
+ + +
+ + +
+ +
+
+ Add setting +
+
+ Delete settings +
+
+
- +
+ diff --git a/ace-am/src/main/webapp/style.css b/ace-am/src/main/webapp/style.css index b9704f3..9b62941 100644 --- a/ace-am/src/main/webapp/style.css +++ b/ace-am/src/main/webapp/style.css @@ -31,7 +31,6 @@ input { font-size: 12px; } - /* * Taken from bootstrap. If we import a lot of rules break and we end up * with broken pages. The ui in general needs an overhaul but that's a lot @@ -376,3 +375,49 @@ vertical-align: top; .btn:hover { border-color: #363636; } + +/** + Tabs are pretty much the same as bulma.io with some small changes to match ACE styling + */ +.tabs { + display: flex; + font-size: 1rem; + justify-content: space-between; + overflow: hidden; + overflow-x: auto; + white-space: nowrap; +} + +.tabs a { + align-items: center; + border-bottom: 1px solid #939393; + color: #363636; + display: flex; + justify-content: center; + margin-bottom: -1px; + padding: 0.5em 1em; + vertical-align: top; + cursor: pointer; +} + +.tabs li { + display: block; +} + +.tabs li.is-active a { + border-bottom-color: #3273dc; + color: #3273dc +} + +.tab-block { + display: none; +} + +.tabs ul { + align-items: center; + border-bottom: solid 1px #939393; + display: flex; + flex-grow: 1; + flex-shrink: 0; + justify-content: flex-start; +} -- GitLab From fc60c881e0b90fc1ef32c9e0ccf8f1c2ce786fc8 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Tue, 23 Apr 2019 13:25:24 -0400 Subject: [PATCH 44/49] Invert function usage --- .../umiacs/ace/monitor/settings/SettingsUtil.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsUtil.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsUtil.java index b422430..fbe806d 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsUtil.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsUtil.java @@ -183,7 +183,7 @@ public class SettingsUtil { * @return true if collection, settings are not null and parameter is "true" */ public static boolean getBoolean(Collection c, String attr) { - if (!containsKey(c, attr)) { + if (notContains(c, attr)) { return false; } @@ -191,7 +191,7 @@ public class SettingsUtil { } public static String getString(Collection c, String attr) { - if (!containsKey(c, attr)) { + if (notContains(c, attr)) { return null; } @@ -199,7 +199,7 @@ public class SettingsUtil { } public static int getInt(Collection c, String attr, int def) { - if (!containsKey(c, attr)) { + if (notContains(c, attr)) { return def; } try { @@ -209,9 +209,9 @@ public class SettingsUtil { } } - private static boolean containsKey(Collection c, String attr) { - return (c != null && c.getSettings() != null - && c.getSettings().containsKey(attr)); + private static boolean notContains(Collection c, String attr) { + return (c == null || c.getSettings() == null + || !c.getSettings().containsKey(attr)); } // Update the settings our context listeners would normally do -- GitLab From e793a8d287a288cfaa16b49fc71d098f522e7f13 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Tue, 23 Apr 2019 13:30:20 -0400 Subject: [PATCH 45/49] Remove ingest.maxthreads usage --- .../monitor/settings/SettingsConstants.java | 4 +- .../ace/monitor/settings/SettingsUtil.java | 2 +- ace-am/src/main/webapp/settings.jsp | 49 ++++++++----------- 3 files changed, 23 insertions(+), 32 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsConstants.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsConstants.java index e796163..192944c 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsConstants.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsConstants.java @@ -31,7 +31,7 @@ public class SettingsConstants { public static final String PARAM_4J_CONV_PAT = "log4j.appender.A1.layout.ConversionPattern"; public static final String PARAM_4J_IRODS = "log4j.logger.edu.umiacs.irods"; public static final String PARAM_4J_CLASS = "log4j.logger.edu.umiacs"; - public static final String PARAM_INGEST = "ingest.maxthreads"; + // public static final String PARAM_INGEST = "ingest.maxthreads"; // Default Values public static final String auditBlocking = "false"; @@ -48,7 +48,7 @@ public class SettingsConstants { public static final String imsResetTimeout = "3000"; public static final String imsSSL = "false"; public static final String authManagement = "true"; - public static final String maxIngestThreads = "4"; + // public static final String maxIngestThreads = "4"; public static final String throttleWait = "0"; public static final String throttleBPS = "0"; // Yay log4j diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsUtil.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsUtil.java index fbe806d..cadbb98 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsUtil.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsUtil.java @@ -167,7 +167,7 @@ public class SettingsUtil { defaults.add(new SettingsParameter(PARAM_4J_IRODS, log4JLoggerIrods)); defaults.add(new SettingsParameter(PARAM_4J_LAYOUT, log4JA1Layout)); defaults.add(new SettingsParameter(PARAM_4J_ROOT_LOGGER, log4JRootLogger)); - defaults.add(new SettingsParameter(PARAM_INGEST, maxIngestThreads)); + // defaults.add(new SettingsParameter(PARAM_INGEST, maxIngestThreads)); defaults.add(new SettingsParameter(PARAM_AUDIT_ONLY, auditOnly)); defaults.add(new SettingsParameter(PARAM_AUDIT_SAMPLE, auditSample)); defaults.add(new SettingsParameter(PARAM_AUDIT_BLOCKING, auditBlocking)); diff --git a/ace-am/src/main/webapp/settings.jsp b/ace-am/src/main/webapp/settings.jsp index 3b8df1a..382b369 100644 --- a/ace-am/src/main/webapp/settings.jsp +++ b/ace-am/src/main/webapp/settings.jsp @@ -43,15 +43,6 @@
-
- - - The maximum number of threads allowed for ingesting token stores. - - -
-
+
+ + Block audits when connectivity to the IMS is lost + +
+
+ + Maximum amount of retries when IMS failure occur + +
+
+ + + Amount of time to wait in milliseconds before retrying after an IMS failure + + +
Enable automated auditing @@ -143,26 +154,6 @@
-
- - Block audits when connectivity to the IMS is lost - -
-
- - Maximum amount of retries when IMS failure occur - -
-
- - - Amount of time to wait in milliseconds before retrying after an IMS failure - - -
-- GitLab From 6ebb1ea062d5bcb210060af8449dcdfd70549135 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Tue, 23 Apr 2019 13:32:12 -0400 Subject: [PATCH 46/49] Deprecate ingest.maxthread constants --- .../ace/monitor/settings/SettingsConstants.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsConstants.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsConstants.java index 192944c..438325c 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsConstants.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/settings/SettingsConstants.java @@ -31,7 +31,6 @@ public class SettingsConstants { public static final String PARAM_4J_CONV_PAT = "log4j.appender.A1.layout.ConversionPattern"; public static final String PARAM_4J_IRODS = "log4j.logger.edu.umiacs.irods"; public static final String PARAM_4J_CLASS = "log4j.logger.edu.umiacs"; - // public static final String PARAM_INGEST = "ingest.maxthreads"; // Default Values public static final String auditBlocking = "false"; @@ -48,7 +47,6 @@ public class SettingsConstants { public static final String imsResetTimeout = "3000"; public static final String imsSSL = "false"; public static final String authManagement = "true"; - // public static final String maxIngestThreads = "4"; public static final String throttleWait = "0"; public static final String throttleBPS = "0"; // Yay log4j @@ -74,4 +72,16 @@ public class SettingsConstants { @Deprecated public static final String auditMaxBlockTime = "0"; + /** + * @deprecated no longer used + */ + @Deprecated + public static final String PARAM_INGEST = "ingest.maxthreads"; + + /** + * @deprecated no longer used + */ + @Deprecated + public static final String maxIngestThreads = "4"; + } -- GitLab From e64d2f6fd4f8f515f9c8401494fd70486c4387fd Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Wed, 24 Apr 2019 13:12:48 -0400 Subject: [PATCH 47/49] More formatting --- .../java/edu/umiacs/ace/monitor/audit/AuditThread.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java index dbfd3bf..6c5429d 100644 --- a/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java +++ b/ace-am/src/main/java/edu/umiacs/ace/monitor/audit/AuditThread.java @@ -789,12 +789,11 @@ public final class AuditThread extends Thread implements CancelCallback { if (digestStream == null) { em = PersistUtil.getEntityManager(); logManager.persistCollectionEvent(LogEnum.SYSTEM_ERROR, - "Cannot collect digests from remote site: " - + pc.getSite().getRemoteURL(), em); + "Cannot collect digests from remote site: " + pc.getSite().getRemoteURL(), + em); em.close(); em = null; - LOG.info("remote site returned null stream " + pc.getSite(). - getRemoteURL()); + LOG.info("remote site returned null stream " + pc.getSite().getRemoteURL()); continue; } CollectionCompare2 cc = new CollectionCompare2(digestStream, null); @@ -811,8 +810,7 @@ public final class AuditThread extends Thread implements CancelCallback { mim = new MonitoredItemManager(em); for (String unseenFile : cr.getUnseenTargetFiles()) { - LOG.trace("Item missing at remote " - + unseenFile + " " + pc.getSite()); + LOG.trace("Item missing at remote " + unseenFile + " " + pc.getSite()); MonitoredItem mi = mim.getItemByPath(unseenFile, coll); currentErrors.remove(mi); if (mi.getState() == 'A') { -- GitLab From 754c90840d799de75bfaae05927730eaff05b1e1 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Wed, 24 Apr 2019 13:14:27 -0400 Subject: [PATCH 48/49] Additional documentation --- .../ims/api/ImmediateTokenRequestBatch.java | 49 ++++++++++++++++++- .../umiacs/ace/ims/api/TokenValidator.java | 6 ++- 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/ImmediateTokenRequestBatch.java b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/ImmediateTokenRequestBatch.java index 2b4c8f5..8a744a1 100644 --- a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/ImmediateTokenRequestBatch.java +++ b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/ImmediateTokenRequestBatch.java @@ -46,8 +46,9 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; /** - * Thread which uses the {@link IMSService} to call `requestTokensImmediate` when processing - * ACE Collections which need to receive ACE Tokens for files. + * Thread which uses the {@link IMSService} to call + * {@link IMSService#requestTokensImmediate(String, List)} when processing ACE Collections which + * need to receive ACE Tokens for files. * * @author mmcgann */ @@ -84,6 +85,19 @@ class ImmediateTokenRequestBatch extends Thread implements TokenRequestBatch { this.start(); } + /** + * Add a {@link TokenRequest} to the {@code requests} list so that an ACE Token can be + * requested from the ACE IMS + *

+ * This method will block when trying to acquire a lock from the main thread. This can only + * happen IFF a batch of requests is not already being processed. If the {@code requests} has a + * size greater than or equal to the {@code maxQueueLength} after adding the + * {@link TokenRequest}, this will wake up the main thread in order to process the current batch + * ({@code requests}). + * + * @param request the {@link TokenRequest} to create an ACE Token for + * @throws InterruptedException if the thread is interrupted + */ public void add(TokenRequest request) throws InterruptedException { Check.notNull("request", request); @@ -102,6 +116,11 @@ class ImmediateTokenRequestBatch extends Thread implements TokenRequestBatch { } } + /** + * Signal that the {@link ImmediateTokenRequestBatch} should be shut down. Blocks until the + * lock can be acquired at which point signal the {@link Condition}. Once the final batch has + * been run, block again on {@link #join()}. + */ public void close() { lock.lock(); try { @@ -119,6 +138,15 @@ class ImmediateTokenRequestBatch extends Thread implements TokenRequestBatch { } } + /** + * Check if a batch is ready + * * requests is non empty AND + * * requests.size is greater than equal to maxQueueLength OR + * * processNow flag is set (the processCondition deadline elapsed) OR + * * shutdownRequested flag is set (close called) + * + * @return true if a batch should be processed, false otherwise + */ private boolean isBatchReady() { lock.lock(); try { @@ -131,6 +159,13 @@ class ImmediateTokenRequestBatch extends Thread implements TokenRequestBatch { } } + /** + * Process the {@code requests} present and receive {@link TokenResponse}s from the IMS for + * each request. This will acquire a lock while filling up a local queue of items to process, + * so any call to {@link #add(TokenRequest)} on {@code requests} will block until + * {@code requests} has been drained. Once complete, the lock is released and + * {@link TokenRequest}s can continue to be added. + */ private void processBatch() { List batch; lock.lock(); @@ -155,6 +190,16 @@ class ImmediateTokenRequestBatch extends Thread implements TokenRequestBatch { } } + /** + * Entry point for the main thread. Runs until {@link #close()} is called or an + * {@link Exception} is thrown while running {@link #processBatch()} or + * {@link Condition#awaitUntil(Date)}. + * + * This will attempt to acquire the {@code lock} so that it can block for {@code minWaitTime}. + * While it is calling {@link Condition#awaitUntil(Date)}, the lock can be reacquired by other + * treads. This allows {@link #add(TokenRequest)} to be used in order to fill the + * {@code requests} list for processing. + */ @Override public void run() { NDC.push("Request Thread (" + identifier + "): "); diff --git a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/TokenValidator.java b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/TokenValidator.java index 29e39b9..0958b6f 100644 --- a/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/TokenValidator.java +++ b/ace-ims-api/src/main/java/edu/umiacs/ace/ims/api/TokenValidator.java @@ -53,7 +53,11 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; /** - * Validate ACE Tokens + * Validate ACE Tokens using {@link IMSService#getRoundSummaries(List)} for the {@link AceToken} + * rounds which we have. + * + * Note: This follows the same type of processing as {@link ImmediateTokenRequestBatch}, which + * can be referenced for documentation. * * @author toaster */ -- GitLab From c6e0e22162023c120cfe8b35300c4c9645cca1e7 Mon Sep 17 00:00:00 2001 From: Michael Ritter Date: Wed, 24 Apr 2019 13:52:21 -0400 Subject: [PATCH 49/49] Remove debug printing --- ace-am/src/main/webapp/settings.jsp | 2 -- 1 file changed, 2 deletions(-) diff --git a/ace-am/src/main/webapp/settings.jsp b/ace-am/src/main/webapp/settings.jsp index 382b369..948f6d3 100644 --- a/ace-am/src/main/webapp/settings.jsp +++ b/ace-am/src/main/webapp/settings.jsp @@ -282,10 +282,8 @@ var active = document.getElementsByClassName('is-active')[0]; if (active != null) { var activeControl = active.getAttribute('aria-controls'); - console.log(activeControl); if (activeControl != null) { var block = document.getElementById(activeControl); - console.log(block); block.classList.toggle('tab-block'); } active.classList.toggle('is-active'); -- GitLab