Commit 829ef374 authored by Michael Ritter's avatar Michael Ritter
Browse files

#43 Fix for various problems when starting up

After the transition to moving settings into the database, there were
numerous errors which occurred if a setting was not yet persisted. This
aims to fix that by being proactive about recognizing when a clean
install is done and registering all settings in addition to a new
method to persist a default setting if it is not yet registered.
parent ffea97e9
......@@ -31,15 +31,16 @@
package edu.umiacs.ace.driver;
import edu.umiacs.ace.monitor.settings.SettingsParameter;
import edu.umiacs.ace.monitor.settings.SettingsUtil;
import edu.umiacs.ace.util.PersistUtil;
import edu.umiacs.util.Strings;
import java.util.Timer;
import java.util.TimerTask;
import org.apache.log4j.Logger;
import javax.persistence.EntityManager;
import javax.persistence.Query;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import org.apache.log4j.Logger;
import java.util.Timer;
import java.util.TimerTask;
/**
*
......@@ -94,24 +95,22 @@ public class QueryThrottle implements ServletContextListener {
@Override
public void contextInitialized( ServletContextEvent sce ) {
EntityManager em = PersistUtil.getEntityManager();
Query q = em.createNamedQuery("SettingsParameter.getAttr");
q.setParameter("attr", PARAM_TIME);
SettingsParameter s = (SettingsParameter) q.getSingleResult();
String time = s.getValue();
if ( Strings.isValidInt(time) ) {
setMinWait(Integer.parseInt(time));
// Todo: migrate these out of here
SettingsParameter blockTime = SettingsUtil.getOrDefault(PARAM_TIME,
String.valueOf(minWait), em);
SettingsParameter bps = SettingsUtil.getOrDefault(PARAM_BPS,
String.valueOf(maxBps), em);
if (Strings.isValidInt(blockTime.getValue())) {
setMinWait(Integer.parseInt(blockTime.getValue()));
LOG.info("Setting query throttle minwait to " + minWait);
}
q.setParameter("attr", PARAM_BPS);
s = (SettingsParameter) q.getSingleResult();
String maxBpsString = s.getValue();
if ( Strings.isValidLong(maxBpsString) ) {
setMaxBps(Long.parseLong(maxBpsString));
if (Strings.isValidLong(bps.getValue())) {
setMaxBps(Long.parseLong(bps.getValue()));
}
if ( minWait > 0 ) {
checkTimer = new Timer("srb timer");
checkTimer.schedule(new TimerTask() {
......@@ -122,8 +121,6 @@ public class QueryThrottle implements ServletContextListener {
}
}, 0, minWait);
}
}
@Override
......
......@@ -44,22 +44,23 @@ import org.apache.log4j.Logger;
import org.apache.log4j.NDC;
import javax.persistence.EntityManager;
import javax.persistence.EntityTransaction;
import javax.persistence.NoResultException;
import javax.persistence.Query;
import javax.persistence.TypedQuery;
import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import java.util.List;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import java.util.stream.Collectors;
/**
* Set the IMS for the AuditThread to use. Also, startup a background task
* to handle firing off monitor tasks.
*
* <p>
* TODO: Handle token validation
*
*
* @author toaster
*/
public final class AuditConfigurationContext implements ServletContextListener {
......@@ -77,174 +78,97 @@ public final class AuditConfigurationContext implements ServletContextListener {
private static final Logger LOG = Logger.getLogger(AuditConfigurationContext.class);
@Override
public void contextInitialized( ServletContextEvent arg0 ) {
public void contextInitialized(ServletContextEvent arg0) {
EntityManager em = PersistUtil.getEntityManager();
Query q = em.createNamedQuery("SettingsParameter.getAttr");
SettingsParameter s = null;
ServletContext ctx = arg0.getServletContext();
// set IMS for audit Thread from server parameter
q.setParameter("attr", SettingsConstants.PARAM_IMS);
s = (SettingsParameter) q.getSingleResult();
AuditThreadFactory.setIMS(s.getValue());
if ( Strings.isEmpty(AuditThreadFactory.getIMS()) ) {
throw new RuntimeException("IMS is empty");
}
q.setParameter("attr", SettingsConstants.PARAM_IMS_TOKEN_CLASS);
s = (SettingsParameter) q.getSingleResult();
if ( !Strings.isEmpty(s.getValue()) ) {
String tokenClass = s.getValue();
AuditThreadFactory.setTokenClass(tokenClass);
}
q.setParameter("attr", SettingsConstants.PARAM_IMS_PORT);
s = (SettingsParameter) q.getSingleResult();
if ( Strings.isValidInt(s.getValue()) ) {
int port = Integer.parseInt(s.getValue());
if ( port > 1 && port < 32768 ) {
AuditThreadFactory.setImsPort(port);
} else {
throw new RuntimeException("ims.port must be between 1 and 32768");
}
}
q.setParameter("attr", SettingsConstants.PARAM_AUDIT_ONLY);
try {
s = (SettingsParameter) q.getSingleResult();
AuditThreadFactory.setAuditOnly(Boolean.valueOf(s.getValue()));
}catch ( NoResultException ex ) {
EntityTransaction trans = em.getTransaction();
trans.begin();
em.persist(new SettingsParameter(SettingsConstants.PARAM_AUDIT_ONLY,
SettingsConstants.auditOnly, false));
trans.commit();
AuditThreadFactory.setAuditOnly(false);
}
q.setParameter("attr", SettingsConstants.PARAM_AUDIT_SAMPLE);
try {
s = (SettingsParameter) q.getSingleResult();
AuditThreadFactory.setAuditSampling(Boolean.valueOf(s.getValue()));
} catch (NoResultException ex) {
EntityTransaction trans = em.getTransaction();
trans.begin();
em.persist(new SettingsParameter(SettingsConstants.PARAM_AUDIT_SAMPLE,
SettingsConstants.auditSample, false));
trans.commit();
AuditThreadFactory.setAuditSampling(false);
}
q.setParameter("attr", SettingsConstants.PARAM_IMS_SSL);
try {
s = (SettingsParameter) q.getSingleResult();
AuditThreadFactory.setSSL(Boolean.valueOf(s.getValue()));
} catch (NoResultException ex) {
EntityTransaction trans = em.getTransaction();
trans.begin();
em.persist(new SettingsParameter(SettingsConstants.PARAM_IMS_SSL,
SettingsConstants.imsSSL, false));
trans.commit();
AuditThreadFactory.setSSL(false);
}
q.setParameter("attr", SettingsConstants.PARAM_AUDIT_BLOCKING);
try {
s = (SettingsParameter) q.getSingleResult();
AuditThreadFactory.setBlocking(Boolean.valueOf(s.getValue()));
} catch (NoResultException ex) {
EntityTransaction trans = em.getTransaction();
trans.begin();
em.persist(new SettingsParameter(SettingsConstants.PARAM_AUDIT_BLOCKING,
SettingsConstants.auditBlocking,
false));
trans.commit();
AuditThreadFactory.setBlocking(false);
}
q.setParameter("attr", SettingsConstants.PARAM_AUDIT_MAX_BLOCK_TIME);
try {
s = (SettingsParameter) q.getSingleResult();
int blockTime = 0;
String val = s.getValue();
if (Strings.isValidInt(val)) {
blockTime = Integer.parseInt(val);
}
// Just in case...
if ( blockTime < 0 ) {
blockTime = 0;
}
AuditThreadFactory.setMaxBlockTime(blockTime);
} catch (NoResultException ex) {
EntityTransaction trans = em.getTransaction();
trans.begin();
em.persist(new SettingsParameter(SettingsConstants.PARAM_AUDIT_MAX_BLOCK_TIME,
SettingsConstants.auditMaxBlockTime,
false));
trans.commit();
AuditThreadFactory.setBlocking(false);
TypedQuery<SettingsParameter> nq = em.createNamedQuery(
"SettingsParameter.getCurrentSettings",
SettingsParameter.class);
List<SettingsParameter> resultList = nq.getResultList();
Map<String, String> resultMap = resultList.stream()
.collect(Collectors.toMap((SettingsParameter::getName),
(SettingsParameter::getValue)));
em.close();
if (resultList.isEmpty()) {
SettingsUtil.updateSettings(SettingsUtil.getDefaultSettings());
}
// PauseBean first because of dependencies
ServletContext ctx = arg0.getServletContext();
PauseBean pb = new PauseBean();
ctx.setAttribute(ATTRIBUTE_PAUSE, pb);
q.setParameter("attr", SettingsConstants.PARAM_AUTO_AUDIT_ENABLE);
s = (SettingsParameter) q.getSingleResult();
// Invert the boolean because the PB checks if we're paused, not enabled
String enableAudits = s.getValue();
String enableAudits = resultMap.getOrDefault(
SettingsConstants.PARAM_AUTO_AUDIT_ENABLE,
SettingsConstants.autoAudit);
pb.setPaused(!Boolean.valueOf(enableAudits));
q.setParameter("attr", SettingsConstants.PARAM_THROTTLE_MAXAUDIT);
s = (SettingsParameter) q.getSingleResult();
String maxRun = s.getValue();
if ( Strings.isValidInt(maxRun) ) {
int audit = Integer.parseInt(maxRun);
if ( audit > 0 ) {
AuditThreadFactory.setMaxAudits(audit);
}
}
// q.setParam(attr, continuous audit)
// s = q.getSingleResult
// seed = s.getValue
//bgAudit = new BackgroundAuditorFactory();
//BackgroundAuditorFactory.start();
em.close();
checkTimer = new Timer("Audit Check Timer");
checkTimer.schedule(new MyTimerTask(pb), 0, HOUR);
}
// set IMS for audit Thread from server parameter
AuditThreadFactory.setIMS(resultMap.getOrDefault(
SettingsConstants.PARAM_IMS,
SettingsConstants.ims));
String tokenClass = resultMap.getOrDefault(
SettingsConstants.PARAM_IMS_TOKEN_CLASS,
SettingsConstants.imsTokenClass);
AuditThreadFactory.setTokenClass(tokenClass);
String port = resultMap.getOrDefault(
SettingsConstants.PARAM_IMS_PORT,
SettingsConstants.imsPort);
if (Strings.isValidInt(port)) {
AuditThreadFactory.setImsPort(Integer.parseInt(port));
}
/**
* TODO: Use this to eliminate a lot of the boilerplate above with this
*
private void doDbStuff(EntityManager em, String param, String defaultVal, Method method) throws InvocationTargetException, IllegalAccessException {
SettingsParameter s = null;
Query q = em.createNamedQuery("SettingsParameter.getAttr");
q.setParameter("attr", param);
try {
s = (SettingsParameter) q.getSingleResult();
method.invoke(null, s.getValue());
} catch (NoResultException ex) {
EntityTransaction trans = em.getTransaction();
trans.begin();
em.persist(new SettingsParameter(param,
defaultVal,
false));
trans.commit();
method.invoke(null, defaultVal);
String auditOnly = resultMap.getOrDefault(
SettingsConstants.PARAM_AUDIT_ONLY,
SettingsConstants.auditOnly);
AuditThreadFactory.setAuditOnly(Boolean.valueOf(auditOnly));
// keep this off for now even though it isn't used
AuditThreadFactory.setAuditSampling(false);
String imsSsl = resultMap.getOrDefault(
SettingsConstants.PARAM_IMS_SSL,
SettingsConstants.imsSSL);
AuditThreadFactory.setSSL(Boolean.valueOf(imsSsl));
String blocking = resultMap.getOrDefault(
SettingsConstants.PARAM_AUDIT_BLOCKING,
SettingsConstants.auditBlocking);
AuditThreadFactory.setBlocking(Boolean.valueOf(blocking));
String blockTimeS = resultMap.getOrDefault(
SettingsConstants.PARAM_AUDIT_MAX_BLOCK_TIME,
SettingsConstants.auditMaxBlockTime);
int blockTime = 0;
if (Strings.isValidInt(blockTimeS)) {
blockTime = Integer.parseInt(blockTimeS);
}
// Just in case...
if (blockTime < 0) {
blockTime = 0;
}
AuditThreadFactory.setMaxBlockTime(blockTime);
String maxAudit = resultMap.getOrDefault(
SettingsConstants.PARAM_THROTTLE_MAXAUDIT,
SettingsConstants.maxAudit);
if (Strings.isValidInt(maxAudit)) {
int audit = Integer.parseInt(maxAudit);
if (audit > 0) {
AuditThreadFactory.setMaxAudits(audit);
}
}
}
*/
@Override
public void contextDestroyed( ServletContextEvent arg0 ) {
if ( checkTimer != null ) {
public void contextDestroyed(ServletContextEvent arg0) {
if (checkTimer != null) {
checkTimer.cancel();
}
AuditThreadFactory.cancellAll();
......@@ -258,7 +182,7 @@ public final class AuditConfigurationContext implements ServletContextListener {
private PauseBean() {
}
public void setPaused( boolean paused ) {
public void setPaused(boolean paused) {
this.paused = paused;
}
......@@ -276,7 +200,7 @@ public final class AuditConfigurationContext implements ServletContextListener {
private PauseBean pb;
private MyTimerTask( PauseBean pb ) {
private MyTimerTask(PauseBean pb) {
this.pb = pb;
}
......@@ -292,7 +216,7 @@ public final class AuditConfigurationContext implements ServletContextListener {
List<Collection> items;
LOG.trace("START - checking for required audit");
try {
if ( pb.isPaused() ) {
if (pb.isPaused()) {
return;
}
......@@ -301,39 +225,39 @@ public final class AuditConfigurationContext implements ServletContextListener {
items = query.getResultList();
for ( Collection c : items ) {
for (Collection c : items) {
StorageDriver sa;
int checkperiod = SettingsUtil.getInt(c, ConfigConstants.ATTR_AUDIT_PERIOD, 0);
if ( checkperiod < 1 ) {
if (checkperiod < 1) {
LOG.trace("Skipping auditing for collection: " + c.getName()
+ " check period: " + checkperiod);
continue;
}
// if last sync is null, fire away since we haven't run yet
if ( c.getLastSync() == null ) {
if (c.getLastSync() == null) {
LOG.debug("No last sync for " + c.getName() + " running");
sa = StorageDriverFactory.createStorageAccess(c,
em);
AuditThreadFactory.createThread(c, sa, true, (MonitoredItem[])null);
AuditThreadFactory.createThread(c, sa, true, (MonitoredItem[]) null);
} else {
long syncTime = c.getLastSync().getTime();
long currTime = System.currentTimeMillis();
// if the next audit is due and we haven't already started the audit
long lastSyncTime = currTime - syncTime;
if ( lastSyncTime > ((long) (checkperiod * HOUR * 24)) &&
!auditing(c) ) {
if (lastSyncTime > ((long) (checkperiod * HOUR * 24)) &&
!auditing(c)) {
LOG.debug("last sync difference: " + (currTime - syncTime)
+ " greater than " + (checkperiod * HOUR * 24));
sa = StorageDriverFactory.createStorageAccess(c, em);
AuditThreadFactory.createThread(c, sa, true, (MonitoredItem[]) null);
} else {
LOG.trace("No Sync on " + c.getName());
}
}
}
} catch ( Throwable t ) {
} catch (Throwable t) {
LOG.error("Error testing to see if collections need auditing", t);
} finally {
LOG.trace("FINISH - checking for required audit");
......
......@@ -5,32 +5,31 @@
package edu.umiacs.ace.monitor.register;
import edu.umiacs.ace.monitor.settings.SettingsConstants;
import edu.umiacs.ace.monitor.settings.SettingsParameter;
import edu.umiacs.ace.monitor.settings.SettingsUtil;
import edu.umiacs.ace.util.PersistUtil;
import org.apache.log4j.NDC;
import javax.persistence.EntityManager;
import javax.persistence.Query;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import org.apache.log4j.NDC;
import static edu.umiacs.ace.monitor.settings.SettingsConstants.PARAM_INGEST;
/**
*
* @author shake
*/
public class IngestContextListener implements ServletContextListener {
public static final String PARAM_INGEST = "ingest.maxthreads";
public void contextInitialized(ServletContextEvent sce) {
NDC.push("[Ingest startup]");
EntityManager em = PersistUtil.getEntityManager();
Query threadQuery = em.createNamedQuery("SettingsParameter.getAttr");
threadQuery.setParameter("attr", PARAM_INGEST);
SettingsParameter s = (SettingsParameter) threadQuery.getSingleResult();
IngestThreadPool.setMaxThreads(Integer.parseInt(s.getValue()));
IngestThreadPool pool = IngestThreadPool.getInstance();
//pool.start();
SettingsParameter ingestSettings = SettingsUtil.getOrDefault(PARAM_INGEST,
SettingsConstants.maxIngestThreads, em);
IngestThreadPool.setMaxThreads(Integer.parseInt(ingestSettings.getValue()));
NDC.pop();
}
public void contextDestroyed(ServletContextEvent sce) {
......
......@@ -30,11 +30,20 @@
// $Id$
package edu.umiacs.ace.monitor.reporting;
import edu.umiacs.ace.monitor.settings.SettingsConstants;
import edu.umiacs.ace.monitor.settings.SettingsParameter;
import edu.umiacs.ace.monitor.settings.SettingsUtil;
import edu.umiacs.ace.util.PersistUtil;
import edu.umiacs.util.Strings;
import java.text.ParseException;
import java.util.Properties;
import org.apache.log4j.Logger;
import org.apache.log4j.NDC;
import org.quartz.CronTrigger;
import org.quartz.JobDetail;
import org.quartz.Scheduler;
import org.quartz.SchedulerException;
import org.quartz.SchedulerFactory;
import org.quartz.Trigger;
import javax.mail.Message;
import javax.mail.MessagingException;
import javax.mail.Session;
......@@ -42,17 +51,11 @@ import javax.mail.Transport;
import javax.mail.internet.InternetAddress;
import javax.mail.internet.MimeMessage;
import javax.persistence.EntityManager;
import javax.persistence.Query;
import javax.persistence.TypedQuery;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import org.apache.log4j.Logger;
import org.apache.log4j.NDC;
import org.quartz.CronTrigger;
import org.quartz.JobDetail;
import org.quartz.Scheduler;
import org.quartz.SchedulerException;
import org.quartz.SchedulerFactory;
import org.quartz.Trigger;
import java.text.ParseException;
import java.util.Properties;
/**
* Context listener to control the starting and stopping of the quartz scheduler used to generate
......@@ -75,27 +78,23 @@ public class SchedulerContextListener implements ServletContextListener {
NDC.push("[schedstartup]");
try {
EntityManager em = PersistUtil.getEntityManager();
Query q1 = em.createNamedQuery("SettingsParameter.getAttr");
q1.setParameter("attr", PARAM_SMTP_SERVER);
SettingsParameter s = (SettingsParameter)q1.getSingleResult();
SettingsParameter s = SettingsUtil.getOrDefault(PARAM_SMTP_SERVER,
SettingsConstants.mailServer, em);
mailserver = s.getValue();
q1.setParameter("attr", PARAM_FROM);
s = (SettingsParameter) q1.getSingleResult();
s = SettingsUtil.getOrDefault(PARAM_FROM, SettingsConstants.mailFrom, em);
mailfrom = s.getValue();
if (Strings.isEmpty(mailserver)) {
mailserver = "127.0.0.1";
}
schedFact = new org.quartz.impl.StdSchedulerFactory();
sched = schedFact.getScheduler();
sched.start();
LOG.debug("Starting the scheduler, registering all jobs");
Query q2 = em.createNamedQuery("ReportPolicy.listAll");
for (Object o : q2.getResultList()) {
addJob((ReportPolicy) o);
TypedQuery<ReportPolicy> query =
em.createNamedQuery("ReportPolicy.listAll", ReportPolicy.class);
for (ReportPolicy policy : query.getResultList()) {
addJob(policy);
}
} catch (Exception ex) {
LOG.error("Error starting report scheduling", ex);
......
......@@ -23,14 +23,11 @@ public class SettingsConstants {
public static final String PARAM_USER_AUTH = "auth.management";
public static final String PARAM_4J_FILE = "log4j.appender.A1.File";
public static final String PARAM_4J_APPENDER = "log4j.appender.A1";
public static final String PARAM_4J_FILE_SIZE =
"log4j.appender.A1.maxFileSize";
public static final String PARAM_4J_BACKUP_INDEX =
"log4j.appender.A1.maxBackupIndex";
public static final String PARAM_4J_FILE_SIZE = "log4j.appender.A1.maxFileSize";
public static final String PARAM_4J_BACKUP_INDEX = "log4j.appender.A1.maxBackupIndex";
public static final String PARAM_4J_ROOT_LOGGER = "log4j.rootLogger";
public static final String PARAM_4J_LAYOUT = "log4j.appender.A1.layout";
public static final String PARAM_4J_CONV_PAT =
"log4j.appender.A1.layout.ConversionPattern";
public static final String PARAM_4J_CONV_PAT = "log4j.appender.A1.layout.ConversionPattern";
public static final String PARAM_4J_IRODS = "log4j.logger.edu.umiacs.irods";
public static final String PARAM_4J_CLASS ="log4j.logger.edu.umiacs";
public static final String PARAM_INGEST = "ingest.maxthreads";
......@@ -59,8 +56,8 @@ public class SettingsConstants {
public static final String log4JA1MaxBackupIndex = "5";
public static final String log4JRootLogger = "FATAL, A1";
public static final String log4JA1Layout = "org.apache.log4j.PatternLayout";
public static final String log4JA1layoutConversationPattern
= "%d{[dd/MMM/yyyy:HH:mm:ss]} %x%m%n";
public static final String log4JA1layoutConversationPattern =
"%d{[dd/MMM/yyyy:HH:mm:ss]} %x%m%n";
public static final String log4JLoggerIrods = "ERROR";
public static final String log4JLoggerUMIACS = "TRACE";
......
......@@ -2,57 +2,52 @@ package edu.umiacs.ace.monitor.settings;
import edu.umiacs.ace.util.EntityManagerServlet;
import edu.umiacs.util.Strings;
import org.apache.commons.fileupload.FileItemIterator;
import org.apache.commons.fileupload.FileItemStream;