Commit f564ef1b authored by shake's avatar shake
Browse files

Changed audit thread to not include file names when making a token request

Added ssl support to AuditThreadFactory and Threads
Added rest method for removing collections
Removed stray printlns

git-svn-id: https://subversion.umiacs.umd.edu/ace/trunk@198 f1b3a171-7291-4a19-a512-95ad0ad9394a
parent 2580f5ea
......@@ -83,7 +83,6 @@ public class FilterServlet extends EntityManagerServlet {
q.setParameter("coll", c);
q.executeUpdate();
for (FilterEntry fe : regexResults ) {
System.out.println(fe.getRegex() + " :: " + fe.getAffectedItem() + " :: " + fe.getCollection());
fe.setCollection(c);
em.persist(fe);
}
......
......@@ -66,6 +66,7 @@ public final class AuditConfigurationContext implements ServletContextListener {
private static final String PARAM_IMS = "ims";
private static final String PARAM_IMS_PORT = "ims.port";
private static final String PARAM_IMS_TOKEN_CLASS = "ims.tokenclass";
private static final String PARAM_IMS_SSL = "ims.ssl";
private static final String PARAM_DISABLE_AUDIT = "auto.audit.disable";
private static final String PARAM_THROTTLE_MAXAUDIT = "throttle.maxaudit";
private static final String PARAM_AUDIT_ONLY = "audit.only";
......@@ -121,6 +122,19 @@ public final class AuditConfigurationContext implements ServletContextListener {
AuditThreadFactory.setAuditOnly(false);
}
q.setParameter("attr", PARAM_IMS_SSL);
try {
s = (SettingsParameter) q.getSingleResult();
AuditThreadFactory.setSSL(Boolean.valueOf(s.getValue()));
} catch ( NoResultException ex ) {
EntityTransaction trans = em.getTransaction();
trans.begin();
em.persist(new SettingsParameter(SettingsConstants.PARAM_IMS_SSL,
SettingsConstants.imsSSL, false));
trans.commit();
AuditThreadFactory.setSSL(false);
}
PauseBean pb = new PauseBean();
ctx.setAttribute(ATTRIBUTE_PAUSE, pb);
......@@ -139,6 +153,12 @@ public final class AuditConfigurationContext implements ServletContextListener {
}
}
// q.setParam(attr, continuous audit)
// s = q.getSingleResult
// seed = s.getValue
//bgAudit = new BackgroundAuditorFactory();
//BackgroundAuditorFactory.start();
checkTimer = new Timer("Audit Check Timer");
checkTimer.schedule(new MyTimerTask(pb), 0, HOUR);
......@@ -151,6 +171,7 @@ public final class AuditConfigurationContext implements ServletContextListener {
}
AuditThreadFactory.cancellAll();
AuditTokens.cancellAll();
//BackgroundAuditorFactory.cancel();
}
public static class PauseBean {
......@@ -212,7 +233,7 @@ public final class AuditConfigurationContext implements ServletContextListener {
LOG.debug("No last sync for " + c.getName() + " running");
sa = StorageDriverFactory.createStorageAccess(c,
em);
AuditThreadFactory.createThread(c, sa, (MonitoredItem[])null);
AuditThreadFactory.createThread(c, sa, true, (MonitoredItem[])null);
} else {
long syncTime = c.getLastSync().getTime();
long currTime = System.currentTimeMillis();
......@@ -222,7 +243,7 @@ public final class AuditConfigurationContext implements ServletContextListener {
+ " greater than " + (checkperiod * HOUR * 24));
sa = StorageDriverFactory.createStorageAccess(c,
em);
AuditThreadFactory.createThread(c, sa, (MonitoredItem[]) null);
AuditThreadFactory.createThread(c, sa, true, (MonitoredItem[]) null);
} else {
LOG.trace("No Sync on " + c.getName());
}
......
......@@ -75,8 +75,8 @@ import org.apache.log4j.Logger;
import org.apache.log4j.NDC;
/**
*
*
*
*
* @author toaster
*/
public final class AuditThread extends Thread implements CancelCallback {
......@@ -92,6 +92,8 @@ public final class AuditThread extends Thread implements CancelCallback {
private boolean cancel = false;
private StorageDriver driver;
private long session;
private boolean auditOnly;
private boolean verbose = false;
// exposed for jsp display
private long totalErrors = 0;
private long newFilesFound = 0;
......@@ -105,19 +107,22 @@ public final class AuditThread extends Thread implements CancelCallback {
private String tokenClassName;
private LogEventManager logManager;
private AuditIterable<FileBean> iterableItems;
private boolean auditOnly;
AuditThread(Collection c, StorageDriver driver, boolean auditOnly,
public AuditThread(Collection c, StorageDriver driver, boolean auditOnly,
MonitoredItem... startItem) {
this.driver = driver;
this.coll = c;
this(c, driver, auditOnly, true, startItem);
}
public AuditThread(Collection c, StorageDriver driver, boolean auditOnly,
boolean verbose, MonitoredItem... startItem) {
this.auditOnly = auditOnly;
this.verbose = verbose;
this.coll = c;
this.driver = driver;
if (startItem != null && startItem[0] != null) {
baseItemPathList = Arrays.copyOf(startItem, startItem.length);
}
this.setName("audit: " + c.getName());
this.start();
}
public DriverStateBean[] getDriverStatus() {
......@@ -192,7 +197,8 @@ public final class AuditThread extends Thread implements CancelCallback {
logAuditStart();
callback = new FileAuditCallback(coll, session, this);
boolean auditTokens = SettingsUtil.getBoolean(coll, ConfigConstants.ATTR_AUDIT_TOKENS);
boolean auditTokens = SettingsUtil.getBoolean(coll,
ConfigConstants.ATTR_AUDIT_TOKENS);
/*
if (!openIms() || (auditTokens
&& !openTokenValidator(MessageDigest.getInstance(
......@@ -202,6 +208,7 @@ public final class AuditThread extends Thread implements CancelCallback {
*
*/
if( !auditOnly ) {
// If we can't open the IMS, we want to fallback
fallback = !openIms();
if ( !fallback ) {
if ( auditTokens &&
......@@ -237,8 +244,11 @@ public final class AuditThread extends Thread implements CancelCallback {
}
lastFileSeen = "Setting collection state";
setCollectionState();
logAuditFinish();
generateAuditReport();
if ( verbose ) {
logAuditFinish();
generateAuditReport();
}
NDC.pop();
}
......@@ -248,7 +258,7 @@ public final class AuditThread extends Thread implements CancelCallback {
private boolean openIms() {
try {
IMSService ims;
ims = IMSService.connect(imsHost, imsport);
ims = IMSService.connect(imsHost, imsport, AuditThreadFactory.useSSL());
batch = ims.createImmediateTokenRequestBatch(
tokenClassName, callback, 1000, 5000);
......@@ -273,36 +283,31 @@ public final class AuditThread extends Thread implements CancelCallback {
}
private boolean openTokenValidator(MessageDigest digest) {
// TokenValidator validator = null;
try {
IMSService ims;
ims = IMSService.connect(imsHost, imsport);
TokenAuditCallback callback = new TokenAuditCallback(itemMap, this,
ims = IMSService.connect(imsHost, imsport, AuditThreadFactory.useSSL());
TokenAuditCallback tokenCallback = new TokenAuditCallback(itemMap,
this,
coll,
session);
validator = ims.createTokenValidator(callback, 1000,
validator = ims.createTokenValidator(tokenCallback, 1000,
5000, digest);
// "SHA-256-0", new BatchCallback(), 1000, 5000);
// "SHA-256-0", new BatchCallback(), 1000, 5000);
return true;
} catch (IMSException e) {
EntityManager em;
LOG.error("Cannot connect to IMS ", e);
// LogEventManager lem;
em = PersistUtil.getEntityManager();
EntityTransaction trans = em.getTransaction();
trans.begin();
// em.persist(lem.imsCommError("", coll, e));
em.persist(logManager.createCollectionEvent(
LogEnum.UNKNOWN_IMS_COMMUNICATION_ERROR, e));
trans.commit();
em.close();
return false;
}
// return validator;
}
private void performAudit() {
......@@ -330,7 +335,7 @@ public final class AuditThread extends Thread implements CancelCallback {
}
if (cancel || abortException != null) {
// iterableItems.cancel();
// iterableItems.cancel();
return;
}
......@@ -356,7 +361,7 @@ public final class AuditThread extends Thread implements CancelCallback {
// let token batch finish before processing or items waiting tokens
// will appear as errors.
// Thread.sleep(2000);
// Thread.sleep(2000);
if (batch != null) {
batch.close();
batch = null;
......@@ -389,7 +394,7 @@ public final class AuditThread extends Thread implements CancelCallback {
* 1. we got an exception somewhere, abortException will be set
* 2. user requested cancel
* 3. we finished as expected
*
*
*/
private void logAuditFinish() {
EntityManager em = PersistUtil.getEntityManager();
......@@ -430,7 +435,6 @@ public final class AuditThread extends Thread implements CancelCallback {
private void logAuditStart() {
LOG.info("Starting audit on: " + coll.getName());
EntityManager em = PersistUtil.getEntityManager();
String message = null;
if (baseItemPathList != null) {
......@@ -536,10 +540,11 @@ public final class AuditThread extends Thread implements CancelCallback {
}
TokenRequest request = new TokenRequest();
request.setName(item.getPath());
//request.setName(item.getPath());
request.setName(item.getId().toString());
request.setHashValue(currentFile.getHash());
if (!Strings.isEmpty(item.getPath()) && !Strings.isEmpty(
currentFile.getHash()) && batch != null) {
if (!Strings.isEmpty(item.getPath()) &&
!Strings.isEmpty(currentFile.getHash()) && batch != null) {
try {
batch.add(request);
} catch (InterruptedException e) {
......@@ -554,8 +559,7 @@ public final class AuditThread extends Thread implements CancelCallback {
"Generated checksum: " + currentFile.getHash()
+ " expected checksum: " + item.getFileDigest());
LogEvent event;
if (currentFile.getHash().equals(
item.getFileDigest())) {
if (currentFile.getHash().equals(item.getFileDigest())) {
// File is active and intact
// log the transition if it already isn't A
// only toggle to A if state is not a remote error.
......@@ -566,8 +570,7 @@ public final class AuditThread extends Thread implements CancelCallback {
item.getPath());
item.setState('A');
item.setStateChange(new Date());
LOG.trace(
"Toggling state to Active for " + item.getPath());
LOG.trace("Toggling state to Active for " + item.getPath());
} else {
event = null;
}
......@@ -617,14 +620,14 @@ public final class AuditThread extends Thread implements CancelCallback {
totalErrors++;
} else {
// create new item with state of missing token
mim.addItem(fileName, parentName, false, coll, 'T',
MonitoredItem mi = mim.addItem(fileName, parentName, false, coll, 'T',
currentFile.getFileSize());
event[1] = null;
TokenRequest request = new TokenRequest();
request.setName(fileName);
request.setName(mi.getId().toString());
request.setHashValue(currentFile.getHash());
if (!Strings.isEmpty(fileName) && !Strings.isEmpty(
currentFile.getHash()) && batch != null) {
if (!Strings.isEmpty(fileName) &&
!Strings.isEmpty(currentFile.getHash()) && batch != null) {
try {
batch.add(request);
} catch (InterruptedException e) {
......@@ -648,8 +651,7 @@ public final class AuditThread extends Thread implements CancelCallback {
String parent = (currentFile.getPathList().length > i + 1
? currentFile.getPathList()[i + 1] : null);
parent = Strings.cleanStringForXml(parent, '_');
mim.createDirectory(currentFile.getPathList()[i], parent,
coll);
mim.createDirectory(currentFile.getPathList()[i], parent, coll);
}
}
......@@ -745,10 +747,10 @@ public final class AuditThread extends Thread implements CancelCallback {
try {
CompareResults cr = new CompareResults(cc);
cc.compareTo(cr, coll, null);
// cc.loadCollectionTable(coll, null);
// cc.doCompare();
// cc.loadCollectionTable(coll, null);
// cc.doCompare();
// For now, we don't care about files that only exist remotely
// cc.getUnseenTargetFiles();
// cc.getUnseenTargetFiles();
// State: P, only set if file is currently active as local errors
// take precedence
em = PersistUtil.getEntityManager();
......
......@@ -49,6 +49,7 @@ public class AuditThreadFactory {
private static int imsPort = 8080;
private static String tokenClass = "SHA-256";
private static boolean auditOnly = false;
private static boolean ssl = false;
public static void setIMS( String ims ) {
imsHost = ims;
......@@ -78,6 +79,10 @@ public class AuditThreadFactory {
AuditThreadFactory.auditOnly = auditOnlyMode;
}
public static boolean isAuditing() {
return !runningThreads.isEmpty();
}
/**
* Return a new or existing thread if any room is available New threads will start replication
*
......@@ -85,7 +90,7 @@ public class AuditThreadFactory {
* @param tri
* @return
*/
public static AuditThread createThread( Collection c, StorageDriver tri,
public static AuditThread createThread( Collection c, StorageDriver tri, boolean verbose,
MonitoredItem... startItem ) {
synchronized ( runningThreads ) {
if ( !runningThreads.containsKey(c) && runningThreads.size() < max_audits ) {
......@@ -94,11 +99,12 @@ public class AuditThreadFactory {
// {
// pathList = new String[startItem.size()];
// }
AuditThread newThread = new AuditThread(c, tri, auditOnly,
startItem);
AuditThread newThread = new AuditThread(c, tri, auditOnly,
verbose, startItem);
newThread.setImsHost(imsHost);
newThread.setImsport(imsPort);
newThread.setTokenClassName(tokenClass);
newThread.start();
runningThreads.put(c, newThread);
}
return runningThreads.get(c);
......@@ -142,6 +148,10 @@ public class AuditThreadFactory {
AuditThreadFactory.max_audits = max_audits;
}
public static void setSSL(Boolean ssl) {
AuditThreadFactory.ssl = ssl;
}
/**
* Return the current thread for a collection.
* @param c collection to fetch
......@@ -167,4 +177,9 @@ public class AuditThreadFactory {
runningThreads.remove(c);
}
}
public static boolean useSSL() {
return AuditThreadFactory.ssl;
}
}
......@@ -96,7 +96,7 @@ public class FileAuditCallback implements RequestBatchCallback {
tokensAdded++;
if ( tr.getStatusCode() == StatusCode.SUCCESS ) {
MonitoredItem item = null;
item = mim.getItemByPath(tr.getName(), coll);
item = mim.getItemById(tr.getName(), coll);
Token token = new Token();
// token.setCreateDate(new Date());
......@@ -122,7 +122,7 @@ public class FileAuditCallback implements RequestBatchCallback {
item.setToken(token);
item.setState('A');
em.merge(item);
em.persist(logManager.createItemEvent(LogEnum.ADD_TOKEN, tr.getName(),
em.persist(logManager.createItemEvent(LogEnum.ADD_TOKEN, item.getPath(),
"Round: " + tr.getRoundId()));
} else {
em.persist(logManager.createItemEvent(LogEnum.CREATE_TOKEN_ERROR, tr.getName(),
......
......@@ -92,7 +92,8 @@ public final class StartSyncServlet extends EntityManagerServlet {
if ( isTokenAudit ) {
AuditTokens.createThread(collection);
} else {
AuditThreadFactory.createThread(collection, driver, item);
// Move 'true' to separate variable for clarity
AuditThreadFactory.createThread(collection, driver, true, item);
}
response.sendRedirect("Status?collectionid=" + collection.getId());
......
......@@ -222,7 +222,7 @@ public class ManageCollectionServlet extends EntityManagerServlet {
return false;
}
public void populateCollection(HttpServletRequest req, Collection col) {
public static void populateCollection(HttpServletRequest req, Collection col) {
if (!Strings.isEmpty(req.getParameter(PARAM_NAME))) {
col.setName(req.getParameter(PARAM_NAME));
}
......
......@@ -33,6 +33,7 @@ package edu.umiacs.ace.monitor.core;
import edu.umiacs.ace.monitor.log.LogEnum;
import edu.umiacs.ace.monitor.log.LogEventManager;
import edu.umiacs.ace.util.PersistUtil;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
......@@ -118,6 +119,16 @@ public class MonitoredItemManager {
}
}
/*
*
* @param id to get
* @return the monitored item
*/
public MonitoredItem getItemById( String id, Collection c ) {
EntityManager pem = PersistUtil.getEntityManager();
return pem.find(MonitoredItem.class, Long.parseLong(id));
}
/**
* Set the state on all items that contain no tokens to false
*
......
......@@ -199,7 +199,6 @@ public class DefaultAccountContextListener implements ServletContextListener {
}
if ( !settingsMigration ) {
System.out.println("Adding modify system settings role");
UserRoles updateSysSettings = new UserRoles();
updateSysSettings.setRolename("Modify System Settings");
updateSysSettings.setUsername("admin");
......
......@@ -72,4 +72,12 @@ public class CollectionManagement {
return em.find(Collection.class, collId);
}
@GET
@Path("remove/{id}")
public void removeCollection(@PathParam("id") long collId) {
EntityManager em = PersistUtil.getEntityManager();
em.remove(em.find(Collection.class, collId));
em.close();
}
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment