Commit 5b60abb2 authored by shake's avatar shake
Browse files

Added support for ingesting tokens and fixed a few minor bugs/typos

git-svn-id: https://subversion.umiacs.umd.edu/ace/trunk@145 f1b3a171-7291-4a19-a512-95ad0ad9394a
parent 21ea2f09
......@@ -202,8 +202,13 @@
<artifactId>jersey-json</artifactId>
<version>1.6</version>
</dependency>
</dependencies>
<dependency>
<groupId>edu.umiacs.ace</groupId>
<artifactId>ace-ims-ws</artifactId>
<version>1.7-SNAPSHOT</version>
<type>jar</type>
</dependency>
</dependencies>
</project>
......
......@@ -56,7 +56,7 @@ import org.apache.log4j.Logger;
/**
* Servlet to manage browsing a collection. This will store a directorytree in
* a users session
*
*
* @author toaster
*/
public class BrowseServlet extends EntityManagerServlet {
......@@ -70,7 +70,7 @@ public class BrowseServlet extends EntityManagerServlet {
public static final String SESSION_DIRECTORY_TREE = "directoryTree";
public static final String PAGE_ISAUDITING = "auditing";
/**
/**
* Processes requests for both HTTP <code>GET</code> and <code>POST</code> methods.
* @param request servlet request
* @param response servlet response
......@@ -118,11 +118,11 @@ public class BrowseServlet extends EntityManagerServlet {
dt.toggleItem(itemId);
}
// else
// {
// session.setAttribute(SESSION_FILE,
// loadFileBean(dt.getDirectoryNode(itemId)));
// }
// else
// {
// session.setAttribute(SESSION_FILE,
// loadFileBean(dt.getDirectoryNode(itemId)));
// }
}
request.setAttribute(PAGE_ISAUDITING, isRunning);
RequestDispatcher dispatcher = request.getRequestDispatcher("browse.jsp");
......@@ -138,21 +138,24 @@ public class BrowseServlet extends EntityManagerServlet {
retBean.root = master;
retBean.name = node.getName();
if (master.getToken() != null)
{
// TokenResponse resp = (TokenResponse)master.getToken().getToken();
MessageDigest digest = MessageDigest.getInstance(master.getToken().getProofAlgorithm());
ProofValidator pv = new ProofValidator();
Proof proof = TokenUtil.extractProof(master.getToken());
byte[] root = pv.rootHash(digest, proof, HashValue.asBytes(master.getFileDigest()));
retBean.itemProof = HashValue.asHexString(root);
if (master.getToken() != null) {
// TokenResponse resp = (TokenResponse)master.getToken().getToken();
MessageDigest digest = MessageDigest.getInstance(master.getToken().getProofAlgorithm());
ProofValidator pv = new ProofValidator();
Proof proof = TokenUtil.extractProof(master.getToken());
String fileDigest = master.getFileDigest();
if ( fileDigest != null ){
byte[] root = pv.rootHash(digest, proof, HashValue.asBytes(master.getFileDigest()));
retBean.itemProof = HashValue.asHexString(root);
}else {
retBean.itemProof = null;
}
}
return retBean;
} catch ( EntityNotFoundException e ) {
return null;
} catch (NoSuchAlgorithmException e)
{
} catch (NoSuchAlgorithmException e) {
LOG.error("Cannot create ald",e);
return null;
}
......
......@@ -81,12 +81,12 @@ public final class TokenAuditCallback implements ValidationCallback {
totalErrors++;
LOG.error("Exception throw registering", throwable);
EntityManager em = PersistUtil.getEntityManager();
EntityTransaction trans = em.getTransaction();
trans.begin();
// EntityTransaction trans = em.getTransaction();
// trans.begin();
String msg = "Exception in batch thread" + Strings.exceptionAsString(throwable);
logManager.persistCollectionEvent(LogEnum.SYSTEM_ERROR, msg, em);
// lem.abortSite(collection, "Exception in batch thread", throwable);
trans.commit();
// trans.commit();
em.close();
cancel.cancel();
......
......@@ -60,6 +60,8 @@ import javax.persistence.TemporalType;
* P - partner does not have this file
* D - partner file has different digest
* @author toaster
* 1.6+
* R - file registered but not ready for auditing
*/
@Entity
@Table(name = "monitored_item")
......
/*
* Copyright (c) 2007-2010, University of Maryland
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted provided
* that the following conditions are met:
*
* Redistributions of source code must retain the above copyright notice, this list of conditions
* and the following disclaimer.
*
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions
* and the following disclaimer in the documentation and/or other materials provided with the
* distribution.
*
* Neither the name of the University of Maryland nor the names of its contributors may be used to
* endorse or promote products derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
* PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
* TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* ACE Components were written in the ADAPT Project at the University of
* Maryland Institute for Advanced Computer Study.
*/
package edu.umiacs.ace.monitor.core;
import edu.umiacs.ace.hashtree.ProofHash;
import edu.umiacs.ace.hashtree.ProofNode;
import edu.umiacs.util.Strings;
import edu.umiacs.ace.monitor.core.Token;
import edu.umiacs.ace.util.HashValue;
import edu.umiacs.util.Check;
import java.util.Date;
/**
*
* @author shake
*/
public class TokenBuilder {
private String digestAlg;
private String imsService;
private long round = -1;
private Date date;
StringBuilder proof = new StringBuilder();
public void reset() {
digestAlg = null;
imsService = null;
round = -1;
date = null;
proof = new StringBuilder();
}
public void setDate(Date date) {
this.date = Check.notNull("Date", date);
}
public void setImsService(String imsService) {
this.imsService = Check.noWhitespace("IMS Service", imsService);
}
public void setDigestAlgorithm(String digestAlg) {
this.digestAlg = Check.noWhitespace("Digest", digestAlg);
}
public void setRound(long round) {
this.round = Check.isPositive("Round", round);
}
public void startProofLevel() {
if ( proof.length() != 0 ) {
proof.append("\r\n");
}
}
public void addHashLevel(int inheritIdx, ProofNode hashes) {
int pos = 0;
int size = hashes.getHashes().size();
if ( inheritIdx < 0 || inheritIdx > hashes.getHashes().size() ) {
throw new IllegalArgumentException("Supplied index outside hashes list: "
+ inheritIdx + "/" + hashes.getHashes().size());
}
if ( hashes.getHashes() == null || hashes.getHashes().size() < 1 ) {
throw new IllegalArgumentException("Hash list must have at least 1 hash");
}
for ( ProofHash hash : hashes.getHashes() ) {
if ( pos == inheritIdx ) {
proof.append("X:");
}
String hashStr = HashValue.asHexString(hash.getHash());
proof.append(hashStr);
if ( pos != size-1 ) {
proof.append(":");
}
pos++;
}
// Last case -- inheritIdx is equal to the size of the hash list
if( pos == inheritIdx ) {
proof.append(":X");
}
}
public void writeProof() {
if ( Strings.isEmpty(proof) ) {
//throw Exception
throw new IllegalStateException("Proof not complete");
}
proof.append("\r\n");
}
public Token createToken() {
if ( Strings.isEmpty(proof) || Strings.isEmpty(digestAlg) || Strings.isEmpty(imsService)
|| date == null || round == -1 ) {
throw new IllegalStateException("All token parameters not filled out");
}
Token token = new Token();
token.setCreateDate(date);
token.setImsService(imsService);
token.setRound(round);
token.setProofAlgorithm(digestAlg);
// Trim just in case...
token.setProofText(proof.toString()); //.trim());
return token;
}
}
......@@ -96,6 +96,8 @@ public enum LogEnum {
* Finish a sync run on a master site
*/
TOKEN_AUDIT_FINISH(23, "Token Audit Finish", "Auditing of this collection's tokens finished"),
TOKEN_INGEST_UPDATE(24, "Token Ingest Update", "Token was out of date and has been updated"),
FILE_REGISTER(25, "File Registered", "New file registered but is not ready for auditing"),
SYSTEM_ERROR(99, "System Error", "Unknown system error occurred, check server logs");
private int type;
private String shortName;
......@@ -167,6 +169,8 @@ public enum LogEnum {
return TOKEN_AUDIT_START;
case 23:
return TOKEN_AUDIT_FINISH;
case 24:
return TOKEN_INGEST_UPDATE;
case 99:
return SYSTEM_ERROR;
......
/*
* Copyright (c) 2007-2010, University of Maryland
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted provided
* that the following conditions are met:
*
* Redistributions of source code must retain the above copyright notice, this list of conditions
* and the following disclaimer.
*
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions
* and the following disclaimer in the documentation and/or other materials provided with the
* distribution.
*
* Neither the name of the University of Maryland nor the names of its contributors may be used to
* endorse or promote products derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
* PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
* TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* ACE Components were written in the ADAPT Project at the University of
* Maryland Institute for Advanced Computer Study.
*/
package edu.umiacs.ace.monitor.register;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.core.MonitoredItem;
import edu.umiacs.ace.util.PersistUtil;
import edu.umiacs.util.Strings;
import java.util.Date;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import javax.persistence.EntityManager;
import javax.persistence.EntityTransaction;
import javax.persistence.NoResultException;
import javax.persistence.Query;
/**
*
* @author shake
*/
public class IngestDirectory extends Thread{
private Collection coll;
private Set<String> identifiers;
private Set<String> existingParents = new HashSet<String>();
private EntityManager em = PersistUtil.getEntityManager();
private int numTransactions = 0;
public IngestDirectory(Set<String> identifiers , Collection coll){
this.identifiers = identifiers;
this.coll = coll;
}
@Override
public void run() {
if ( identifiers == null || coll == null ) {
return;
}
EntityTransaction trans = em.getTransaction();
trans.begin();
for ( String identifier : identifiers ) {
extractAndRegisterParentDirs(identifier);
}
trans.commit();
}
private void extractAndRegisterParentDirs(String path) {
// We don't have a FileBean, so build the pathList ourselves
StringBuilder fullPath = new StringBuilder(path);
List <String> pathList = new LinkedList<String>();
int index = 0;
while( (index = fullPath.lastIndexOf("/")) != 0 ) {
pathList.add(fullPath.toString());
fullPath.delete(index, fullPath.length());
}
pathList.add(fullPath.toString());
// Same as AuditThread, but with our pathList
String parentName = (pathList.size() > 1
? pathList.get(1) : null);
// 1. make sure directory path is registered
if (parentName != null) {
parentName = Strings.cleanStringForXml(parentName, '_');
for ( int i = 1; i < pathList.size(); i++) {
String parent = (pathList.size() > i + 1 ? pathList.get(i+1) : null);
parent = Strings.cleanStringForXml(parent, '_');
createDirectory(pathList.get(i), parent);
if ( numTransactions > 10000 ) {
em.flush();
em.clear();
}
}
}
}
private void createDirectory(String directory, String root) {
MonitoredItem mi;
if ( existingParents.contains(directory) || directory == null ) {
return;
}
if ( (mi = getItemByPath(directory)) != null ) {
Date d = new Date();
mi.setLastSeen(d);
mi.setLastVisited(d);
mi.setState('A');
em.merge(mi);
numTransactions++;
existingParents.add(directory);
} else {
addItem(directory, root, true, 'A', 0);
}
}
public MonitoredItem getItemByPath( String path ) {
Query q = em.createNamedQuery("MonitoredItem.getItemByPath");
q.setParameter("path", path);
q.setParameter("coll", coll);
try {
return (MonitoredItem) q.getSingleResult();
} catch ( NoResultException ex ) {
return null;
}
}
public MonitoredItem addItem( String path, String parentDir,boolean directory,
char initialState, long size ) {
MonitoredItem mi = new MonitoredItem();
mi.setDirectory(directory);
mi.setLastSeen(new Date());
mi.setLastVisited(new Date());
mi.setStateChange(new Date());
mi.setParentCollection(coll);
mi.setParentPath(parentDir);
mi.setPath(path);
mi.setState(initialState);
mi.setSize(size);
em.persist(mi);
numTransactions++;
return mi;
}
}
/*
* Copyright (c) 2007-2010, University of Maryland
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted provided
* that the following conditions are met:
*
* Redistributions of source code must retain the above copyright notice, this list of conditions
* and the following disclaimer.
*
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions
* and the following disclaimer in the documentation and/or other materials provided with the
* distribution.
*
* Neither the name of the University of Maryland nor the names of its contributors may be used to
* endorse or promote products derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
* PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
* TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* ACE Components were written in the ADAPT Project at the University of
* Maryland Institute for Advanced Computer Study.
*/
package edu.umiacs.ace.monitor.register;
import edu.umiacs.ace.monitor.core.Collection;
import edu.umiacs.ace.monitor.core.MonitoredItem;
import edu.umiacs.ace.monitor.core.Token;
import edu.umiacs.ace.monitor.log.LogEventManager;
import edu.umiacs.ace.token.TokenStoreEntry;
import edu.umiacs.ace.token.TokenStoreReader;
import edu.umiacs.ace.util.EntityManagerServlet;
import edu.umiacs.ace.util.TokenUtil;
import edu.umiacs.util.Strings;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
import javax.persistence.EntityManager;
import javax.servlet.RequestDispatcher;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.HttpSession;
import org.apache.commons.fileupload.FileItemIterator;
import org.apache.commons.fileupload.FileItemStream;
import org.apache.commons.fileupload.FileUploadException;
import org.apache.commons.fileupload.servlet.ServletFileUpload;
import org.apache.commons.fileupload.util.Streams;
//import org.apache.tomcat.util.http.fileupload.FileUploadBase;
/**
* TODO Servlet to ingest token stores
* Servlet to ingest token stores
* @author toaster
*/
public class IngestStore extends EntityManagerServlet {
// private static final Logger LOG = Logger.getLogger(IngestStore.class);
private LogEventManager logManager;
// private long session;
public static final String PAGE_RESULTS = "results";
@Override
protected void processRequest( HttpServletRequest request, HttpServletResponse response,
EntityManager em ) throws ServletException, IOException {
Collection coll = getCollection(request, em);
Collection coll = null; //getCollection(request, em);
MonitoredItem item = getItem(request, em);
TokenStoreReader tokenReader = null;
RequestDispatcher dispatcher;
Map<String, Token> batchTokens = new HashMap<String, Token>();
if (item != null && !item.isDirectory())
throw new ServletException("Selected item is not a directory " + item.getPath());
// request.
if (!ServletFileUpload.isMultipartContent(request)) {
throw new ServletException("No file attached");
}
// request.
ServletFileUpload upload = new ServletFileUpload();
try {
FileItemIterator iter = upload.getItemIterator(request);
while( iter.hasNext() ) {
FileItemStream fileItem = iter.next();
InputStream stream = fileItem.openStream();
if (fileItem.isFormField()) {
// Basic find collection id and set the collection
if ( PARAM_COLLECTION_ID.equals(fileItem.getFieldName()) ) {
String collectionId = Streams.asString(stream);
if ( Strings.isValidLong(collectionId) ) {
long collId = Long.parseLong(collectionId);
coll = em.getReference(Collection.class, collId);
if ( coll == null ) {
throw new ServletException("Collection Id " +
collId + " not found.");
}
}
}
} else {
// Read tokens in from input stream and add to AceToken set
// May want to log this stuff
tokenReader = new TokenStoreReader(fileItem.openStream());
if ( tokenReader == null ) {
throw new ServletException("Token file is corrupt");
}
while ( tokenReader.hasNext() ) {
TokenStoreEntry tokenEntry = tokenReader.next();
Token token = TokenUtil.convertFromAceToken(tokenEntry.getToken());
if ( !token.getProofAlgorithm().equals(coll.getDigestAlgorithm()) ) {
throw new ServletException("Token digest differs from"
+ " collection digest.");
}
batchTokens.put(tokenEntry.getIdentifiers().get(0), token);
}
}
}
} catch (FileUploadException ex) {
throw new ServletException(ex);
}
// Sanity check
if ( coll == null || tokenReader == null ) {
throw new ServletException("Bad upload parameters");
}
IngestThreadFactory threads = new IngestThreadFactory(batchTokens, coll);
HttpSession session = request.getSession();
session.setAttribute(PAGE_RESULTS, threads);