Skip to content

GitLab

Commit 601c94b0 authored by toaster's avatar toaster
Browse files

token to acetoken conversion, first steps. 

git-svn-id: https://subversion.umiacs.umd.edu/ace/trunk@31 f1b3a171-7291-4a19-a512-95ad0ad9394a
parent 78e6e1be
Show whitespace changes
Inline Side-by-side
ace-am/src/java/edu/umiacs/ace/monitor/access/ListContentsServlet.java
View file @ 601c94b0
......@@ -38,6 +38,7 @@ import edu.umiacs.ace.util.EntityManagerServlet;
import edu.umiacs.ace.util.PersistUtil;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.core.Token;
import edu.umiacs.ace.util.DbTokenWriter;
import edu.umiacs.sql.SQL;
import java.io.IOException;
import java.sql.Connection;
......@@ -113,8 +114,9 @@ public class ListContentsServlet extends EntityManagerServlet {
os.println("#Filename | Algorithm | Digest");
}
TokenResponseStoreWriter writer =
new TokenResponseStoreWriter(os, AuditThreadFactory.getIMS());
// TokenResponseStoreWriter writer =
// new TokenResponseStoreWriter(os, AuditThreadFactory.getIMS());
DbTokenWriter writer = new DbTokenWriter(AuditThreadFactory.getIMS(),os);
// query for items
try {
db = PersistUtil.getDataSource();
......@@ -162,7 +164,7 @@ public class ListContentsServlet extends EntityManagerServlet {
} else if ( TYPE_STORE.equals(output) ) {
Token tok = em.getReference(Token.class, rs.getLong(3));
if ( tok != null ) {
writer.startToken((TokenResponse) tok.getToken());
writer.startToken(tok);
writer.addIdentifier(formatPath(rs.getString(1)));
writer.writeTokenEntry();
}
......
ace-am/src/java/edu/umiacs/ace/monitor/access/browse/BrowseServlet.java
View file @ 601c94b0
......@@ -31,14 +31,16 @@
package edu.umiacs.ace.monitor.access.browse;
import edu.umiacs.ace.ims.api.TokenValidator;
import edu.umiacs.ace.ims.ws.TokenResponse;
import edu.umiacs.ace.hashtree.Proof;
import edu.umiacs.ace.hashtree.ProofValidator;
import edu.umiacs.ace.monitor.core.MonitoredItem;
import edu.umiacs.ace.util.EntityManagerServlet;
import edu.umiacs.ace.monitor.audit.AuditThreadFactory;
import edu.umiacs.ace.monitor.audit.AuditTokens;
import edu.umiacs.ace.monitor.access.browse.DirectoryTree.DirectoryNode;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.util.HashValue;
import edu.umiacs.ace.util.TokenUtil;
import java.io.IOException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
......@@ -138,9 +140,13 @@ public class BrowseServlet extends EntityManagerServlet {
if (master.getToken() != null)
{
TokenResponse resp = (TokenResponse)master.getToken().getToken();
MessageDigest digest = MessageDigest.getInstance(resp.getDigestService());
retBean.itemProof = TokenValidator.calculateRoot(digest, master.getFileDigest(), resp.getProofElements());
// TokenResponse resp = (TokenResponse)master.getToken().getToken();
MessageDigest digest = MessageDigest.getInstance(master.getToken().getProofAlgorithm());
ProofValidator pv = new ProofValidator();
Proof proof = TokenUtil.extractProof(master.getToken());
byte[] root = pv.rootHash(digest, proof, HashValue.asBytes(master.getFileDigest()));
retBean.itemProof = HashValue.asHexString(root);
}
return retBean;
} catch ( EntityNotFoundException e ) {
......
ace-am/src/java/edu/umiacs/ace/monitor/audit/AuditThread.java
View file @ 601c94b0
......@@ -40,7 +40,6 @@ import edu.umiacs.ace.ims.api.IMSService;
import edu.umiacs.ace.ims.api.TokenRequestBatch;
import edu.umiacs.ace.ims.api.TokenValidator;
import edu.umiacs.ace.ims.ws.TokenRequest;
import edu.umiacs.ace.ims.ws.TokenResponse;
import edu.umiacs.ace.monitor.access.CollectionCountContext;
import edu.umiacs.ace.util.PersistUtil;
import edu.umiacs.ace.monitor.compare.CollectionCompare;
......@@ -56,6 +55,8 @@ import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.log.LogEnum;
import edu.umiacs.ace.monitor.log.LogEvent;
import edu.umiacs.ace.monitor.peers.PeerCollection;
import edu.umiacs.ace.token.AceToken;
import edu.umiacs.ace.util.TokenUtil;
import edu.umiacs.util.Strings;
import java.io.InputStream;
import java.security.MessageDigest;
......@@ -78,8 +79,8 @@ import org.apache.log4j.NDC;
public final class AuditThread extends Thread implements CancelCallback {
private static final Logger LOG = Logger.getLogger(AuditThread.class);
private Map<TokenResponse, MonitoredItem> itemMap =
new ConcurrentHashMap<TokenResponse, MonitoredItem>();
private Map<AceToken, MonitoredItem> itemMap =
new ConcurrentHashMap<AceToken, MonitoredItem>();
private String imsHost;
private int imsport;
private Collection coll;
......@@ -213,7 +214,8 @@ public final class AuditThread extends Thread implements CancelCallback {
validator.close();
validator = null;
}
lastFileSeen = "Setting collection state";
setCollectionState();
logAuditFinish();
generateAuditReport();
NDC.pop();
......@@ -327,6 +329,7 @@ public final class AuditThread extends Thread implements CancelCallback {
// let token batch finish before processing or items waiting tokens
// will appear as errors.
// Thread.sleep(2000);
if ( batch != null ) {
batch.close();
batch = null;
......@@ -338,8 +341,7 @@ public final class AuditThread extends Thread implements CancelCallback {
// harvest remote collections
lastFileSeen = "comparing to peer sites";
compareToPeers();
lastFileSeen = "Setting collection state";
setCollectionState();
}
......@@ -537,14 +539,15 @@ public final class AuditThread extends Thread implements CancelCallback {
}
item.setLastSeen(new Date());
item.setSize(currentFile.getFileSize());
AceToken token = TokenUtil.convertToAceToken(item.getToken());
// add to token check queue
if ( validator != null ) {
TokenResponse tResp = (TokenResponse) item.getToken().
getToken();
itemMap.put(tResp, item);
// TokenResponse tResp = (TokenResponse) item.getToken().
// getToken();
itemMap.put(token, item);
try {
validator.add(
item.getFileDigest(), tResp);
item.getFileDigest(), token);
} catch ( InterruptedException e ) {
abortException = e;
}
......
ace-am/src/java/edu/umiacs/ace/monitor/audit/AuditTokens.java
View file @ 601c94b0
......@@ -36,10 +36,11 @@ import edu.umiacs.ace.ims.ws.TokenResponse;
import edu.umiacs.ace.util.PersistUtil;
import edu.umiacs.ace.monitor.core.MonitoredItem;
import edu.umiacs.ace.monitor.core.MonitoredItemManager;
import edu.umiacs.ace.monitor.core.Token;
import edu.umiacs.ace.monitor.log.LogEventManager;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.log.LogEnum;
import edu.umiacs.ace.token.AceToken;
import edu.umiacs.ace.util.TokenUtil;
import edu.umiacs.util.Strings;
import java.security.MessageDigest;
import java.util.Date;
......@@ -60,8 +61,8 @@ public class AuditTokens extends Thread implements CancelCallback {
private static final Map<Collection, AuditTokens> runningThreads =
new HashMap<Collection, AuditTokens>();
// private Map<TokenResponse, Token> tokenMap = new ConcurrentHashMap<TokenResponse, Token>();
private Map<TokenResponse, MonitoredItem> itemMap =
new ConcurrentHashMap<TokenResponse, MonitoredItem>();
private Map<AceToken, MonitoredItem> itemMap =
new ConcurrentHashMap<AceToken, MonitoredItem>();
private static final Logger LOG = Logger.getLogger(AuditTokens.class);
private Collection collection;
private boolean cancel = false;
......@@ -234,12 +235,12 @@ public class AuditTokens extends Thread implements CancelCallback {
for ( Object o : q.getResultList() ) {
MonitoredItem item = (MonitoredItem) o;
Token t = item.getToken();
AceToken t = TokenUtil.convertToAceToken(item.getToken());
if ( t != null ) {
TokenResponse response = (TokenResponse) t.getToken();
itemMap.put(response, item);
// TokenResponse response = IMSUtil.c;
itemMap.put(t, item);
tokensSeen++;
validator.add(item.getFileDigest(), response);
validator.add(item.getFileDigest(), t);
}
if ( cancel ) {
break;
......
ace-am/src/java/edu/umiacs/ace/monitor/audit/FileAuditCallback.java
View file @ 601c94b0
......@@ -31,6 +31,7 @@
package edu.umiacs.ace.monitor.audit;
import edu.umiacs.ace.exception.StatusCode;
import edu.umiacs.ace.ims.api.IMSUtil;
import edu.umiacs.ace.ims.api.RequestBatchCallback;
import edu.umiacs.ace.util.PersistUtil;
import edu.umiacs.ace.monitor.core.MonitoredItem;
......@@ -41,6 +42,7 @@ import edu.umiacs.ace.ims.ws.TokenRequest;
import edu.umiacs.ace.ims.ws.TokenResponse;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.log.LogEnum;
import edu.umiacs.ace.token.AceToken;
import edu.umiacs.util.Strings;
import java.io.Serializable;
import java.util.Date;
......@@ -100,10 +102,18 @@ public class FileAuditCallback implements RequestBatchCallback {
item = mim.getItemByPath(tr.getName(), coll);
Token token = new Token();
token.setCreateDate(new Date());
// token.setCreateDate(new Date());
token.setValid(true);
token.setLastValidated(new Date());
token.setToken((Serializable) tr);
// AceToken at = IMSUtil.convertResponse(tr, AuditThreadFactory.getIMS());
// token.setLastValidated();
// token.setToken((Serializable) tr);
token.setImsService(tr.getTokenClassName());
token.setProofText(IMSUtil.formatProof(tr));
token.setRound(tr.getRoundId());
token.setProofAlgorithm(tr.getDigestService());
token.setCreateDate(tr.getTimestamp().toGregorianCalendar().getTime());
token.setParentCollection(coll);
if ( !map.containsKey(tr.getName()) || item == null ) {
LOG.error("No request for response: " + tr.getName() + " or item null, item: "
......
ace-am/src/java/edu/umiacs/ace/monitor/audit/TokenAuditCallback.java
View file @ 601c94b0
......@@ -40,6 +40,7 @@ import java.util.Map;
import edu.umiacs.ace.ims.ws.TokenResponse;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.log.LogEnum;
import edu.umiacs.ace.token.AceToken;
import edu.umiacs.util.Strings;
import javax.persistence.EntityManager;
import javax.persistence.EntityTransaction;
......@@ -52,14 +53,14 @@ import org.apache.log4j.Logger;
public final class TokenAuditCallback implements ValidationCallback {
private static final Logger LOG = Logger.getLogger(ValidationCallback.class);
private Map<TokenResponse, MonitoredItem> itemMap;
private Map<AceToken, MonitoredItem> itemMap;
private long totalErrors = 0;
private long validTokens = 0;
private CancelCallback cancel;
// private Collection collection;
LogEventManager logManager;
public TokenAuditCallback( Map<TokenResponse, MonitoredItem> itemMap,
public TokenAuditCallback( Map<AceToken, MonitoredItem> itemMap,
CancelCallback callback, Collection collection, long session ) {
this.itemMap = itemMap;
this.cancel = callback;
......@@ -92,7 +93,7 @@ public final class TokenAuditCallback implements ValidationCallback {
}
@Override
public void validToken( TokenResponse response ) {
public void validToken( AceToken response ) {
MonitoredItem item = itemMap.get(response);
if ( item == null ) {
return;
......@@ -107,8 +108,8 @@ public final class TokenAuditCallback implements ValidationCallback {
if ( !token.getValid() ) {
token.setValid(true);
// LogEventManager lem = new LogEventManager(session, collection);
String path = response.getName();
em.persist(logManager.createItemEvent(LogEnum.TOKEN_VALID, path));
// String path = response.getName();
em.persist(logManager.createItemEvent(LogEnum.TOKEN_VALID, item.getPath()));
// em.persist(lem.validToken(path, collection));
if ( item.getState() == 'I' ) {
item.setState('A');
......@@ -123,11 +124,11 @@ public final class TokenAuditCallback implements ValidationCallback {
em.merge(item);
trans.commit();
em.close();
LOG.trace("Token valid: " + response.getName());
LOG.trace("Token valid: " + item.getPath());
}
@Override
public void invalidToken( TokenResponse response, String correctCSI,
public void invalidToken( AceToken response, String correctCSI,
String calculatedCSI ) {
MonitoredItem item = itemMap.get(response);
......@@ -143,9 +144,9 @@ public final class TokenAuditCallback implements ValidationCallback {
if ( token.getValid() ) {
token.setValid(false);
String path = response.getName();
// String path = response.getName();
String message = "Generated CSI: " + calculatedCSI + " IMS (correct) CSI: " + correctCSI;
em.persist(logManager.createItemEvent(LogEnum.TOKEN_INVALID, path, message));
em.persist(logManager.createItemEvent(LogEnum.TOKEN_INVALID, item.getPath(), message));
if ( item.getState() == 'A' ) {
item.setState('I');
item.setStateChange(new Date());
......@@ -161,6 +162,6 @@ public final class TokenAuditCallback implements ValidationCallback {
em.close();
totalErrors++;
LOG.trace("Token invalid: " + response.getName());
LOG.trace("Token invalid: " + item.getPath());
}
}
ace-am/src/java/edu/umiacs/ace/monitor/core/DownloadTokenServlet.java
View file @ 601c94b0
......@@ -28,13 +28,11 @@
* Maryland Institute for Advanced Computer Study.
*/
// $Id$
package edu.umiacs.ace.monitor.core;
import edu.umiacs.ace.ims.api.XMLTokenWriter;
import edu.umiacs.ace.ims.ws.TokenResponse;
import edu.umiacs.ace.monitor.audit.AuditThreadFactory;
import edu.umiacs.ace.util.DbTokenWriter;
import edu.umiacs.ace.util.EntityManagerServlet;
import edu.umiacs.util.Strings;
import java.io.IOException;
import javax.persistence.EntityManager;
import javax.servlet.ServletException;
......@@ -52,20 +50,16 @@ public class DownloadTokenServlet extends EntityManagerServlet {
protected void processRequest( HttpServletRequest request,
HttpServletResponse response, EntityManager em )
throws ServletException, IOException {
String idStr = request.getParameter(PARAM_TOKEN_ID);
if ( !Strings.isValidLong(idStr) ) {
throw new ServletException("Must supply id");
}
long id = Long.parseLong(idStr);
Token token = em.getReference(Token.class, id);
Token token = getToken(request, em);
if ( token == null ) {
throw new ServletException("No token for id: " + id);
throw new ServletException("No valid token ID specified");
}
XMLTokenWriter writer = new XMLTokenWriter();
response.setContentType("text/xml");
writer.write(response.getOutputStream(), (TokenResponse) token.getToken());
DbTokenWriter writer =
new DbTokenWriter(AuditThreadFactory.getIMS(), response.getOutputStream());
writer.startToken(token);
response.setContentType("text/plain");
writer.writeTokenEntry();
}
}
ace-am/src/java/edu/umiacs/ace/monitor/core/Token.java
View file @ 601c94b0
......@@ -31,7 +31,6 @@
package edu.umiacs.ace.monitor.core;
import edu.umiacs.ace.monitor.core.Collection;
import java.io.Serializable;
import java.util.Date;
import javax.persistence.Column;
......@@ -39,7 +38,6 @@ import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Lob;
import javax.persistence.ManyToOne;
import javax.persistence.NamedQueries;
import javax.persistence.NamedQuery;
......@@ -53,7 +51,7 @@ import javax.persistence.TemporalType;
* @author toaster
*/
@Entity
@Table(name = "token")
@Table(name = "acetoken")
@NamedQueries({
@NamedQuery(name = "Token.listByCollection", query =
"SELECT t FROM MonitoredItem i, Token t WHERE i.parentCollection = :coll AND i.token = t"),
......@@ -71,17 +69,20 @@ public class Token implements Serializable {
@Temporal(TemporalType.TIMESTAMP)
private Date createDate;
private boolean valid;
@Column(name = "java_token_response")
@Lob
private Serializable token;
// @Column(name = "java_token_response")
// @Lob
// private Serializable token;
@ManyToOne
private Collection parentCollection;
// private String fileDigest;
// New items, from token itself
// private String proofText;
// private String imsService;
// private String proofAlgorithm;
// private long round;
@Column(name = "PROOFTEXT")
private String proofText;
@Column(name = "IMSSERVICE")
private String imsService;
@Column(name = "PROOFALGORITHM")
private String proofAlgorithm;
@Column(name = "ROUND")
private long round;
public void setId( Long id ) {
this.id = id;
......@@ -115,6 +116,38 @@ public class Token implements Serializable {
return "edu.umiacs.ace.monitor.items.Token[id=" + id + "]";
}
public String getImsService() {
return imsService;
}
public String getProofAlgorithm() {
return proofAlgorithm;
}
public String getProofText() {
return proofText;
}
public long getRound() {
return round;
}
public void setImsService( String imsService ) {
this.imsService = imsService;
}
public void setProofAlgorithm( String proofAlgorithm ) {
this.proofAlgorithm = proofAlgorithm;
}
public void setProofText( String proofText ) {
this.proofText = proofText;
}
public void setRound( long round ) {
this.round = round;
}
public Date getLastValidated() {
return lastValidated;
}
......@@ -138,14 +171,14 @@ public class Token implements Serializable {
public void setValid( boolean valid ) {
this.valid = valid;
}
public Serializable getToken() {
return token;
}
public void setToken( Serializable token ) {
this.token = token;
}
//
// public Serializable getToken() {
// return token;
// }
//
// public void setToken( Serializable token ) {
// this.token = token;
// }
public void setParentCollection( Collection parentCollection ) {
this.parentCollection = parentCollection;
......@@ -154,13 +187,4 @@ public class Token implements Serializable {
public Collection getParentCollection() {
return parentCollection;
}
// public String getFileDigest()
// {
// return fileDigest;
// }
//
// public void setFileDigest(String fileDigest)
// {
// this.fileDigest = fileDigest;
// }
}
ace-am/src/java/edu/umiacs/ace/monitor/core/TokenMigrationContextListener.java 0 → 100644
View file @ 601c94b0
/*
* Copyright (c) 2007-2010, University of Maryland
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted provided
* that the following conditions are met:
*
* Redistributions of source code must retain the above copyright notice, this list of conditions
* and the following disclaimer.
*
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions
* and the following disclaimer in the documentation and/or other materials provided with the
* distribution.
*
* Neither the name of the University of Maryland nor the names of its contributors may be used to
* endorse or promote products derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
* PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
* TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* ACE Components were written in the ADAPT Project at the University of
* Maryland Institute for Advanced Computer Study.
*/
package edu.umiacs.ace.monitor.core;
import edu.umiacs.ace.ims.api.IMSUtil;
import edu.umiacs.ace.ims.ws.TokenResponse;
import edu.umiacs.ace.monitor.audit.AuditConfigurationContext;
import edu.umiacs.ace.monitor.audit.AuditConfigurationContext.PauseBean;
import edu.umiacs.ace.util.PersistUtil;
import edu.umiacs.sql.SQL;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.sql.Blob;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import javax.sql.DataSource;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import org.apache.log4j.Logger;
import org.apache.log4j.NDC;
/**
* Token migration listner to migrate tokens from the older pre-1.6 version to the current 1.6+
* version. This will only run once and will stop any auditing from occurring until finished
*
* @author toaster
*/
public class TokenMigrationContextListener implements ServletContextListener {
private static final Logger LOG = Logger.getLogger(TokenMigrationContextListener.class);
@Override
public void contextInitialized( ServletContextEvent sce ) {
PauseBean pb =
(PauseBean) sce.getServletContext().getAttribute(AuditConfigurationContext.ATTRIBUTE_PAUSE);
NDC.push("[MIGRATION]");
Connection conn = null;
boolean migrated;
try {
if ( pb == null ) {
LOG.error("Could not pause auditing (pb null), stopping application to prevent damage");
throw new RuntimeException("PauseBean not present in servlet context");
}
boolean oldState = pb.isPaused();
pb.setPaused(true);
DataSource ds = PersistUtil.getDataSource();
try {
conn = ds.getConnection();
migrated = hasMigrated(conn);
} catch ( SQLException e ) {
LOG.fatal("Error getting SQL connection or table data", e);
throw new RuntimeException("Error grabbing SQL connection or table data", e);
}
if ( !migrated ) {
LOG.info("Token Migration starting");
sce.getServletContext().setAttribute("globalMessage", "Auditing Paused: Token migration in progress");
try {
clearTable(conn);
moveTokens(conn);
//dropOldTable(conn);
pb.setPaused(oldState);
LOG.info("Token migration successfully finished");
} catch ( Exception e ) {
LOG.error("Error migrating old tokens: ", e);
sce.getServletContext().setAttribute("globalMessage", "Error migrating tokens, please check logs");
}
} else {
LOG.info("Skipping token migration (already performed)");
pb.setPaused(oldState);
}
} finally {
sce.getServletContext().setAttribute("globalMessage", "");
if ( conn != null ) {
SQL.release(conn);
}
NDC.pop();
}
}
//
// private Runnable migrationTask = new Runnable() {
//
// @Override