Commit 425c899e authored by toaster's avatar toaster
Browse files

settings migration added

git-svn-id: https://subversion.umiacs.umd.edu/ace/trunk@130 f1b3a171-7291-4a19-a512-95ad0ad9394a
parent acc553e0
......@@ -22,6 +22,19 @@
Notes for specific versions.
- if updating from versions prior to 1.6, see the 1.6 notes.
1.7
ACE 1.7 changed how collections are stored. This will require a a migration
to run prior to ace starting up. You MUST install the 1.6-1.7 patch prior
to running ace 1.7. Failure to install the sql patch will result in ace not
starting. This update should take less than a second and you will see log
lines that look similar to:
[05/Aug/2011:12:46:36] [SETTINGS]Found new settings table
[05/Aug/2011:12:46:36] [SETTINGS]Found old column name, migration triggered
...
...
[05/Aug/2011:12:46:38] [SETTINGS]Settings migration successfully finished
1.6
The 1.6 version changed how the ACE tokens are stored in the database.
When you start the new version after an upgrade, ACE will attempt to
......
......@@ -80,15 +80,15 @@
</dependency>
<!-- for json -->
<dependency>
<!--<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
<version>1.0.1</version>
</dependency>
</dependency>-->
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
<version>1.0.1</version>
<version>1.8.1</version>
</dependency>
<dependency>
......
......@@ -30,6 +30,8 @@
// $Id$
package edu.umiacs.ace.driver;
import edu.umiacs.ace.monitor.core.ConfigConstants;
import edu.umiacs.ace.monitor.core.SettingsUtil;
import edu.umiacs.ace.util.EntityManagerServlet;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.core.MonitoredItem;
......@@ -88,7 +90,8 @@ public final class RetrieveItemServlet extends EntityManagerServlet {
return;
}
if ( !coll.isProxyData() ) {
if (SettingsUtil.getBoolean(coll, ConfigConstants.ATTR_PROXY_DATA)) {
// if ( !coll.isProxyData() ) {
response.setStatus(HttpServletResponse.SC_FORBIDDEN);
return;
}
......
......@@ -30,6 +30,7 @@
// $Id$
package edu.umiacs.ace.monitor.audit;
import edu.umiacs.ace.monitor.core.SettingsUtil;
import edu.umiacs.ace.driver.StorageDriver;
import edu.umiacs.ace.driver.StorageDriverFactory;
import edu.umiacs.ace.util.PersistUtil;
......@@ -168,9 +169,10 @@ public final class AuditConfigurationContext implements ServletContextListener {
for ( Collection c : items ) {
StorageDriver sa;
if ( c.getCheckPeriod() < 1 ) {
int checkperiod = SettingsUtil.getInt(c, PARAM_IMS, 0);
if ( checkperiod < 1 ) {
LOG.trace("Skipping auditing for collection: " + c.getName()
+ " check period: " + c.getCheckPeriod());
+ " check period: " + checkperiod);
continue;
}
// if last sync is null, fire away since we haven't run yet
......@@ -182,10 +184,10 @@ public final class AuditConfigurationContext implements ServletContextListener {
} else {
long syncTime = c.getLastSync().getTime();
long currTime = System.currentTimeMillis();
if ( ((long) (currTime - syncTime)) > ((long) (c.getCheckPeriod() * HOUR
if ( ((long) (currTime - syncTime)) > ((long) (checkperiod * HOUR
* 24)) ) {
LOG.debug("last sync difference: " + (currTime - syncTime)
+ " greater than " + (c.getCheckPeriod() * HOUR * 24));
+ " greater than " + (checkperiod * HOUR * 24));
sa = StorageDriverFactory.createStorageAccess(c,
em);
AuditThreadFactory.createThread(c, sa, (MonitoredItem[]) null);
......
......@@ -53,6 +53,8 @@ import edu.umiacs.ace.monitor.reporting.ReportSummary;
import edu.umiacs.ace.monitor.reporting.SchedulerContextListener;
import edu.umiacs.ace.monitor.reporting.SummaryGenerator;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.core.ConfigConstants;
import edu.umiacs.ace.monitor.core.SettingsUtil;
import edu.umiacs.ace.monitor.log.LogEnum;
import edu.umiacs.ace.monitor.log.LogEvent;
import edu.umiacs.ace.monitor.peers.PeerCollection;
......@@ -187,7 +189,8 @@ public final class AuditThread extends Thread implements CancelCallback {
logAuditStart();
callback = new FileAuditCallback(coll, session, this);
if (!openIms() || (coll.isAuditTokens()
boolean auditTokens = SettingsUtil.getBoolean(coll, ConfigConstants.ATTR_AUDIT_TOKENS);
if (!openIms() || (auditTokens
&& !openTokenValidator(MessageDigest.getInstance(
"SHA-256")))) {
return;
......@@ -288,6 +291,7 @@ public final class AuditThread extends Thread implements CancelCallback {
private void performAudit() {
// 1. Setup audit
//TODO Clean up filtering creation
PathFilter filter = new SimpleFilter(coll);
Date startDate = new Date();
......@@ -371,22 +375,16 @@ public final class AuditThread extends Thread implements CancelCallback {
*
*/
private void logAuditFinish() {
// LogEventManager lem;
EntityManager em = PersistUtil.getEntityManager();
// lem = new LogEventManager(em, session);
if (abortException != null) {
if (abortException instanceof InterruptedException
|| abortException.getCause() instanceof InterruptedException) {
LOG.trace("Audit ending with Interrupt");
// new LogEventManager(em, session).finishFileAudit(coll,
// "Audit Interrupted");
logManager.persistCollectionEvent(LogEnum.FILE_AUDIT_FINISH,
"Audit Interrupted", em);
} else {
LOG.error("Uncaught exception in audit thread", abortException);
// new LogEventManager(em, session).abortSite(coll,
// "Uncaught audit thread exception ", abortException);
String message = Strings.exceptionAsString(abortException);
logManager.persistCollectionEvent(
LogEnum.SYSTEM_ERROR, message, em);
......@@ -431,8 +429,6 @@ public final class AuditThread extends Thread implements CancelCallback {
private LogEvent setItemError(MonitoredItem item, String errorMessage) {
totalErrors++;
LOG.debug("Got error from currentFile: " + item.getPath());
// lem.errorReading(item.getPath(), coll,
// errorMessage);
item.setState('M');
return logManager.createItemEvent(LogEnum.ERROR_READING,
item.getPath(), errorMessage);
......@@ -442,7 +438,6 @@ public final class AuditThread extends Thread implements CancelCallback {
private void processFile(FileBean currentFile) {
EntityManager em = PersistUtil.getEntityManager();
MonitoredItemManager mim;
// LogEventManager lem;
LOG.trace(
"Driver returned Item: " + currentFile.getPathList()[0]
......@@ -450,7 +445,6 @@ public final class AuditThread extends Thread implements CancelCallback {
+ currentFile.getErrorMessage() + " hash: "
+ currentFile.getHash());
// lem = new LogEventManager(em, session);
mim = new MonitoredItemManager(em);
String fileName = currentFile.getPathList()[0];
......@@ -514,7 +508,6 @@ public final class AuditThread extends Thread implements CancelCallback {
LogEvent event;
if (item.getState() != 'T') {
// lem.missingToken(item.getPath(), coll);
event = logManager.createItemEvent(LogEnum.MISSING_TOKEN,
item.getPath());
item.setStateChange(new Date());
......@@ -553,7 +546,6 @@ public final class AuditThread extends Thread implements CancelCallback {
// we handle those later
if (item.getState() != 'A' && item.getState() != 'P'
&& item.getState() != 'D') {
// lem.fileOnline(item.getPath(), coll);
event = logManager.createItemEvent(LogEnum.FILE_ONLINE,
item.getPath());
item.setState('A');
......@@ -568,8 +560,6 @@ public final class AuditThread extends Thread implements CancelCallback {
AceToken token = TokenUtil.convertToAceToken(item.getToken());
// add to token check queue
if (validator != null) {
// TokenResponse tResp = (TokenResponse) item.getToken().
// getToken();
itemMap.put(token, item);
try {
validator.add(
......@@ -607,8 +597,6 @@ public final class AuditThread extends Thread implements CancelCallback {
coll.getDirectory() + fileName);
if (currentFile.isError()) {
mim.addItem(fileName, parentName, false, coll, 'M', 0);
// lem.errorReading(fileName, coll,
// currentFile.getErrorMessage());
event[1] = logManager.createItemEvent(LogEnum.ERROR_READING, fileName,
currentFile.getErrorMessage());
totalErrors++;
......@@ -617,7 +605,6 @@ public final class AuditThread extends Thread implements CancelCallback {
mim.addItem(fileName, parentName, false, coll, 'T',
currentFile.getFileSize());
event[1] = null;
// lem.foundNewFile(fileName, coll);
TokenRequest request = new TokenRequest();
request.setName(fileName);
request.setHashValue(currentFile.getHash());
......@@ -655,8 +642,8 @@ public final class AuditThread extends Thread implements CancelCallback {
}
private String[] createMailList() {
String[] maillist = (coll.getEmailList() == null
? null : coll.getEmailList().split("\\s*,\\s*"));
String addrs = SettingsUtil.getString(coll, ConfigConstants.ATTR_EMAIL_RECIPIENTS);
String[] maillist = (addrs == null ? null : addrs.split("\\s*,\\s*"));
return maillist;
}
......@@ -694,7 +681,6 @@ public final class AuditThread extends Thread implements CancelCallback {
EntityManager em = PersistUtil.getEntityManager();
MonitoredItemManager mim = new MonitoredItemManager(em);
EntityTransaction trans = em.getTransaction();
// LogEventManager lem = new LogEventManager(em, session);
trans.begin();
for (MonitoredItem mi : mim.listItemsBefore(coll, d)) {
......@@ -705,7 +691,6 @@ public final class AuditThread extends Thread implements CancelCallback {
mi.setStateChange(new Date());
em.persist(logManager.createItemEvent(LogEnum.FILE_MISSING,
mi.getPath()));
// em.persist(lem.missingFile(mi.getPath(), coll));
}
mi.setLastVisited(new Date());
em.merge(mi);
......@@ -753,7 +738,6 @@ public final class AuditThread extends Thread implements CancelCallback {
// take precedence
em = PersistUtil.getEntityManager();
mim = new MonitoredItemManager(em);
// LogEventManager lem = new LogEventManager(em, session);
for (String unseenFile : cr.getUnseenTargetFiles()) {
LOG.trace("Item missing at remote "
......@@ -791,8 +775,6 @@ public final class AuditThread extends Thread implements CancelCallback {
String errorMsg = "Expected digest: " + mi.getFileDigest() + " Saw: "
+ dd.getTargetDigest() + " site: " + pc.getSite().getRemoteURL();
em.persist(logManager.createItemEvent(LogEnum.REMOTE_FILE_CORRUPT, dd.getName(), errorMsg));
// em.persist(lem.corruptRemoteFile(dd.getName(), coll,
// mi.getFileDigest(), dd.getTargetDigest(), pc));
trans.commit();
}
}
......@@ -803,7 +785,6 @@ public final class AuditThread extends Thread implements CancelCallback {
em.close();
em = null;
}
// cc.cleanup();
}
}
// Any files remaining in currentErrors should transition back
......
......@@ -36,12 +36,17 @@ import edu.umiacs.util.Argument;
import java.io.Serializable;
import java.util.Date;
import java.util.List;
import java.util.Map;
import javax.persistence.CascadeType;
import javax.persistence.CollectionTable;
import javax.persistence.Column;
import javax.persistence.ElementCollection;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.JoinColumn;
import javax.persistence.MapKeyColumn;
import javax.persistence.NamedQueries;
import javax.persistence.NamedQuery;
import javax.persistence.OneToMany;
......@@ -80,14 +85,15 @@ public class Collection implements Serializable {
@Temporal(TemporalType.TIMESTAMP)
private Date lastSync;
private String storage;
private int checkPeriod;
private boolean proxyData;
private boolean auditTokens;
private char state;
@Column(name = "COLGROUP")
private String group;
private String emailList;
private String digestAlgorithm;
@ElementCollection
@CollectionTable(name="settings", joinColumns=@JoinColumn(name="COLLECTION_ID"))
@MapKeyColumn(name="ATTR")
@Column(name="VALUE")
private Map<String,String> settings;
public void setId( Long id ) {
this.id = id;
......@@ -97,6 +103,13 @@ public class Collection implements Serializable {
return id;
}
public Map<String, String> getSettings() {
return settings;
}
public void setSettings(Map<String, String> settings) {
this.settings = settings;
}
public void setPeerCollections( List<PeerCollection> peerCollections ) {
this.peerCollections = peerCollections;
}
......@@ -179,30 +192,6 @@ public class Collection implements Serializable {
this.storage = storage;
}
public int getCheckPeriod() {
return checkPeriod;
}
public void setCheckPeriod( int checkPeriod ) {
this.checkPeriod = checkPeriod;
}
public boolean isProxyData() {
return proxyData;
}
public void setProxyData( boolean proxyData ) {
this.proxyData = proxyData;
}
public void setAuditTokens( boolean auditTokens ) {
this.auditTokens = auditTokens;
}
public boolean isAuditTokens() {
return auditTokens;
}
public String getGroup() {
return group;
}
......@@ -211,11 +200,4 @@ public class Collection implements Serializable {
this.group = group;
}
public String getEmailList() {
return emailList;
}
public void setEmailList( String emailList ) {
this.emailList = emailList;
}
}
/*
* Copyright (c) 2007-2011, University of Maryland
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted provided
* that the following conditions are met:
*
* Redistributions of source code must retain the above copyright notice, this list of conditions
* and the following disclaimer.
*
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions
* and the following disclaimer in the documentation and/or other materials provided with the
* distribution.
*
* Neither the name of the University of Maryland nor the names of its contributors may be used to
* endorse or promote products derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
* PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
* TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* ACE Components were written in the ADAPT Project at the University of
* Maryland Institute for Advanced Computer Study.
*/
package edu.umiacs.ace.monitor.core;
import edu.umiacs.ace.monitor.audit.AuditConfigurationContext;
import edu.umiacs.ace.monitor.audit.AuditConfigurationContext.PauseBean;
import edu.umiacs.ace.util.PersistUtil;
import edu.umiacs.sql.SQL;
import edu.umiacs.util.Strings;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import javax.sql.DataSource;
import org.apache.log4j.Logger;
import org.apache.log4j.NDC;
/**
* Listener to migrate pre-1.7 ACE settings to the new key/value table
*
* @author toaster
*/
public class CollectionSettingsMigrationListener implements ServletContextListener {
private static final Logger LOG = Logger.getLogger(CollectionSettingsMigrationListener.class);
public void contextInitialized(ServletContextEvent sce) {
PauseBean pb =
(PauseBean) sce.getServletContext().getAttribute(AuditConfigurationContext.ATTRIBUTE_PAUSE);
NDC.push("[SETTINGS]");
Connection conn = null;
boolean migrated;
try {
if (pb == null) {
LOG.error("Could not pause auditing (pb null), stopping application to prevent damage");
throw new RuntimeException("PauseBean not present in servlet context");
}
boolean oldState = pb.isPaused();
pb.setPaused(true);
DataSource ds = PersistUtil.getDataSource();
try {
conn = ds.getConnection();
migrated = hasMigrated(conn);
} catch (SQLException e) {
LOG.fatal("Error getting SQL connection or table data", e);
throw new RuntimeException("Error grabbing SQL connection or table data", e);
}
if (!migrated) {
LOG.info("Settings Migration starting");
sce.getServletContext().setAttribute("globalMessage", "Auditing Paused: Settings migration in progress");
try {
clearTable(conn);
moveSettings(conn);
modifyCollectionTable(conn);
pb.setPaused(oldState);
LOG.info("Settings migration successfully finished");
} catch (Exception e) {
LOG.error("Error migrating old tokens: ", e);
sce.getServletContext().setAttribute("globalMessage", "Error migrating settings, please check logs");
throw new RuntimeException("Settings migration failed, check logs for details", e);
}
} else {
LOG.info("Skipping settings migration (already performed)");
pb.setPaused(oldState);
}
} finally {
sce.getServletContext().setAttribute("globalMessage", "");
if (conn != null) {
SQL.release(conn);
}
NDC.pop();
}
}
private void moveSettings(Connection conn) throws SQLException {
LOG.trace("Starting settings migration");
String stmt = "SELECT ID,EMAILLIST,PROXYDATA,CHECKPERIOD,AUDITTOKENS FROM collection";
String insertSt = "INSERT INTO settings(ATTR,VALUE,COLLECTION_ID) values (?,?,?)";
PreparedStatement pstmt = conn.prepareStatement(stmt);
PreparedStatement insertStmt = conn.prepareStatement(insertSt);
ResultSet rs = pstmt.executeQuery();
try {
while (rs.next()) {
long id = rs.getLong("ID");
String emailList = rs.getString("EMAILLIST");
if (!Strings.isEmpty(emailList)) {
performInsert(ConfigConstants.ATTR_EMAIL_RECIPIENTS, emailList, id, insertStmt);
}
boolean proxyData = rs.getBoolean("PROXYDATA");
performInsert(ConfigConstants.ATTR_PROXY_DATA, Boolean.toString(proxyData), id, insertStmt);
int checkperiod = rs.getInt("CHECKPERIOD");
performInsert(ConfigConstants.ATTR_AUDIT_PERIOD, Integer.toString(checkperiod), id, insertStmt);
boolean auditTokens = rs.getBoolean("AUDITTOKENS");
performInsert(ConfigConstants.ATTR_AUDIT_TOKENS, Boolean.toString(auditTokens), id, insertStmt);
}
} finally {
SQL.release(pstmt);
SQL.release(insertStmt);
SQL.release(rs);
LOG.trace("Exiting settings migration");
}
}
private void performInsert(String attr, String value, long id,
PreparedStatement insertStmt) throws SQLException {
insertStmt.clearParameters();
insertStmt.setString(1, attr);
insertStmt.setString(2, value);
insertStmt.setLong(3, id);
int result = insertStmt.executeUpdate();
LOG.trace("Updated setting: " + attr + " value: " + value + " id: " + id + " results: " + 1);
if (result != 1) {
LOG.error("Attempt to insert new parameter failed, result = " + result);
throw new RuntimeException("Attempt to insert new parameter failed, result = " + result);
}
}
private boolean hasMigrated(Connection conn) throws SQLException {
DatabaseMetaData dbm = conn.getMetaData();
String types[] = {"TABLE"};
boolean newTable = false;
boolean migrated = true;
ResultSet rs = dbm.getTables(null, null, null, types);
// check for new table
try {
while (rs.next()) {
String tabName = rs.getString("TABLE_NAME");
if ("settings".equals(tabName)) {
LOG.info("Found new settings table");
newTable = true;
}
}
} catch (SQLException e) {
LOG.error("Error retrieving table metadata");
throw e;
} finally {
SQL.release(rs);
}
if (!newTable) {
LOG.info("SQL patch to 1.7+ has not been installed, table 'settings' does no exist, shutting down!");
throw new IllegalStateException("SQL patch to 1.7+ has not been installed, table 'settings' does no exist");
}
// Do we have old columns?
rs = dbm.getColumns(null, null, "collection", null);
try {
while (rs.next()) {
String name = rs.getString("COLUMN_NAME");
if ("AUDITTOKENS".equals(name)) {
LOG.info("Found old column name, migration triggered");
migrated = false;
}
}
} catch (SQLException e) {
LOG.error("Error retrieving table metadata");
throw e;
} finally {
SQL.release(rs);
}
return migrated;
}
private void modifyCollectionTable(Connection conn) throws SQLException {
LOG.trace("Removing old columns");
String removeEmail = "ALTER TABLE collection DROP COLUMN EMAILLIST";
String removeProxy = "ALTER TABLE collection DROP COLUMN PROXYDATA";
String removePeriod = "ALTER TABLE collection DROP COLUMN CHECKPERIOD";
String removeTokens = "ALTER TABLE collection DROP COLUMN AUDITTOKENS";
PreparedStatement pstmt = null;
try {
LOG.trace("Removing EMAILLIST");
pstmt = conn.prepareStatement(removeEmail);
doRemove(pstmt);
LOG.trace("Removing PROXYDATA");
pstmt = conn.prepareStatement(removeProxy);
doRemove(pstmt);
LOG.trace("Removing CHECKPERIOD");
pstmt = conn.prepareStatement(removePeriod);
doRemove(pstmt);