Added support for ingesting tokens and fixed a few minor bugs/typos

git-svn-id: https://subversion.umiacs.umd.edu/ace/trunk@145 f1b3a171-7291-4a19-a512-95ad0ad9394a

ace-am/pom.xml

@@ -202,8 +202,13 @@
                     <artifactId>jersey-json</artifactId>
                     <version>1.6</version>
                 </dependency>
-
-            </dependencies>
+        <dependency>
+            <groupId>edu.umiacs.ace</groupId>
+            <artifactId>ace-ims-ws</artifactId>
+            <version>1.7-SNAPSHOT</version>
+            <type>jar</type>
+        </dependency>
+    </dependencies>
         </project>
 
 

ace-am/src/main/java/edu/umiacs/ace/monitor/access/browse/BrowseServlet.java

@@ -56,7 +56,7 @@ import org.apache.log4j.Logger;
 /**
  * Servlet to manage browsing a collection. This will store a directorytree in
  * a users session
- * 
+ *
  * @author toaster
  */
 public class BrowseServlet extends EntityManagerServlet {
@@ -70,7 +70,7 @@ public class BrowseServlet extends EntityManagerServlet {
     public static final String SESSION_DIRECTORY_TREE = "directoryTree";
     public static final String PAGE_ISAUDITING = "auditing";
 
-    /** 
+    /**
      * Processes requests for both HTTP <code>GET</code> and <code>POST</code> methods.
      * @param request servlet request
      * @param response servlet response
@@ -118,11 +118,11 @@ public class BrowseServlet extends EntityManagerServlet {
                 dt.toggleItem(itemId);
 
             }
-//            else
-//            {
-//                session.setAttribute(SESSION_FILE,
-//                        loadFileBean(dt.getDirectoryNode(itemId)));
-//            }
+            //            else
+            //            {
+            //                session.setAttribute(SESSION_FILE,
+            //                        loadFileBean(dt.getDirectoryNode(itemId)));
+            //            }
         }
         request.setAttribute(PAGE_ISAUDITING, isRunning);
         RequestDispatcher dispatcher = request.getRequestDispatcher("browse.jsp");
@@ -138,21 +138,24 @@ public class BrowseServlet extends EntityManagerServlet {
             retBean.root = master;
             retBean.name = node.getName();
 
-            if (master.getToken() != null)
-            {
-//            TokenResponse resp = (TokenResponse)master.getToken().getToken();
-            MessageDigest digest = MessageDigest.getInstance(master.getToken().getProofAlgorithm());
-
-            ProofValidator pv = new ProofValidator();
-            Proof proof = TokenUtil.extractProof(master.getToken());
-            byte[] root = pv.rootHash(digest, proof, HashValue.asBytes(master.getFileDigest()));
-            retBean.itemProof = HashValue.asHexString(root);
+            if (master.getToken() != null) {
+                //            TokenResponse resp = (TokenResponse)master.getToken().getToken();
+                MessageDigest digest = MessageDigest.getInstance(master.getToken().getProofAlgorithm());
+
+                ProofValidator pv = new ProofValidator();
+                Proof proof = TokenUtil.extractProof(master.getToken());
+                String fileDigest = master.getFileDigest();
+                if ( fileDigest != null ){
+                    byte[] root = pv.rootHash(digest, proof, HashValue.asBytes(master.getFileDigest()));
+                    retBean.itemProof = HashValue.asHexString(root);
+                }else {
+                    retBean.itemProof = null;
+                }
             }
             return retBean;
         } catch ( EntityNotFoundException e ) {
             return null;
-        } catch (NoSuchAlgorithmException e)
-        {
+        } catch (NoSuchAlgorithmException e) {
             LOG.error("Cannot create ald",e);
             return null;
         }

ace-am/src/main/java/edu/umiacs/ace/monitor/audit/TokenAuditCallback.java

@@ -81,12 +81,12 @@ public final class TokenAuditCallback implements ValidationCallback {
         totalErrors++;
         LOG.error("Exception throw registering", throwable);
         EntityManager em = PersistUtil.getEntityManager();
-        EntityTransaction trans = em.getTransaction();
-        trans.begin();
+        // EntityTransaction trans = em.getTransaction();
+        // trans.begin();
         String msg = "Exception in batch thread" + Strings.exceptionAsString(throwable);
         logManager.persistCollectionEvent(LogEnum.SYSTEM_ERROR, msg, em);
 //        lem.abortSite(collection, "Exception in batch thread", throwable);
-        trans.commit();
+        // trans.commit();
         em.close();
         cancel.cancel();
 

ace-am/src/main/java/edu/umiacs/ace/monitor/core/MonitoredItem.java

@@ -60,6 +60,8 @@ import javax.persistence.TemporalType;
  * P - partner does not have this file
  * D - partner file has different digest
  * @author toaster
+ * 1.6+
+ * R - file registered but not ready for auditing
  */
 @Entity
 @Table(name = "monitored_item")

ace-am/src/main/java/edu/umiacs/ace/monitor/core/TokenBuilder.java

+/*
+ * Copyright (c) 2007-2010, University of Maryland
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted provided
+ * that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this list of conditions
+ * and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice, this list of conditions
+ * and the following disclaimer in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * Neither the name of the University of Maryland nor the names of its contributors may be used to
+ * endorse or promote products derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+ * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ACE Components were written in the ADAPT Project at the University of
+ * Maryland Institute for Advanced Computer Study.
+ */
+
+package edu.umiacs.ace.monitor.core;
+
+import edu.umiacs.ace.hashtree.ProofHash;
+import edu.umiacs.ace.hashtree.ProofNode;
+import edu.umiacs.util.Strings;
+import edu.umiacs.ace.monitor.core.Token;
+import edu.umiacs.ace.util.HashValue;
+import edu.umiacs.util.Check;
+import java.util.Date;
+
+/**
+ *
+ * @author shake
+ */
+public class TokenBuilder {
+    private String digestAlg;
+    private String imsService;
+    private long round = -1;
+    private Date date;
+    StringBuilder proof = new StringBuilder();
+
+    public void reset() {
+        digestAlg = null;
+        imsService = null;
+        round = -1;
+        date = null;
+        proof = new StringBuilder();
+    }
+
+    public void setDate(Date date) {
+        this.date = Check.notNull("Date", date);
+    }
+
+    public void setImsService(String imsService) {
+        this.imsService = Check.noWhitespace("IMS Service", imsService);
+    }
+
+    public void setDigestAlgorithm(String digestAlg) {
+        this.digestAlg = Check.noWhitespace("Digest", digestAlg);
+    }
+
+    public void setRound(long round) {
+        this.round = Check.isPositive("Round", round);
+        
+    }
+
+    public void startProofLevel() {
+        if ( proof.length() != 0 ) {
+            proof.append("\r\n");
+        }
+    }
+
+    public void addHashLevel(int inheritIdx, ProofNode hashes) {
+        int pos = 0;
+        int size = hashes.getHashes().size();
+
+        if ( inheritIdx < 0 || inheritIdx > hashes.getHashes().size() ) {
+            throw new IllegalArgumentException("Supplied index outside hashes list: "
+                    + inheritIdx + "/" + hashes.getHashes().size());
+        }
+
+        if ( hashes.getHashes() == null || hashes.getHashes().size() < 1 ) {
+            throw new IllegalArgumentException("Hash list must have at least 1 hash");
+        }
+
+        for ( ProofHash hash : hashes.getHashes() ) {
+            if ( pos == inheritIdx ) {
+                proof.append("X:");
+            }
+
+            String hashStr = HashValue.asHexString(hash.getHash());
+            proof.append(hashStr);
+
+            if ( pos != size-1 ) {
+                proof.append(":");
+            }
+            pos++;
+        }
+
+        // Last case -- inheritIdx is equal to the size of the hash list
+        if( pos == inheritIdx ) {
+            proof.append(":X");
+        }
+    }
+
+    public void writeProof() {
+        if ( Strings.isEmpty(proof) ) {
+            //throw Exception
+            throw new IllegalStateException("Proof not complete");
+        }
+
+        proof.append("\r\n");
+    }
+
+    public Token createToken() {
+        if ( Strings.isEmpty(proof) || Strings.isEmpty(digestAlg) || Strings.isEmpty(imsService)
+                || date == null || round == -1 ) {
+            throw new IllegalStateException("All token parameters not filled out");
+        }
+
+        Token token = new Token();
+        token.setCreateDate(date);
+        token.setImsService(imsService);
+        token.setRound(round);
+        token.setProofAlgorithm(digestAlg);
+        // Trim just in case...
+        token.setProofText(proof.toString()); //.trim());
+
+        return token;
+    }
+    
+}

ace-am/src/main/java/edu/umiacs/ace/monitor/log/LogEnum.java

@@ -96,6 +96,8 @@ public enum LogEnum {
      * Finish a sync run on a master site
      */
     TOKEN_AUDIT_FINISH(23, "Token Audit Finish", "Auditing of this collection's tokens finished"),
+    TOKEN_INGEST_UPDATE(24, "Token Ingest Update", "Token was out of date and has been updated"),
+    FILE_REGISTER(25, "File Registered", "New file registered but is not ready for auditing"),
     SYSTEM_ERROR(99, "System Error", "Unknown system error occurred, check server logs");
     private int type;
     private String shortName;
@@ -167,6 +169,8 @@ public enum LogEnum {
                 return TOKEN_AUDIT_START;
             case 23:
                 return TOKEN_AUDIT_FINISH;
+            case 24:
+                return TOKEN_INGEST_UPDATE;
 
             case 99:
                 return SYSTEM_ERROR;

ace-am/src/main/java/edu/umiacs/ace/monitor/register/IngestDirectory.java

+/*
+ * Copyright (c) 2007-2010, University of Maryland
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted provided
+ * that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this list of conditions
+ * and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice, this list of conditions
+ * and the following disclaimer in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * Neither the name of the University of Maryland nor the names of its contributors may be used to
+ * endorse or promote products derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+ * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ACE Components were written in the ADAPT Project at the University of
+ * Maryland Institute for Advanced Computer Study.
+ */
+
+package edu.umiacs.ace.monitor.register;
+
+import edu.umiacs.ace.monitor.core.Collection;
+import edu.umiacs.ace.monitor.core.MonitoredItem;
+import edu.umiacs.ace.util.PersistUtil;
+import edu.umiacs.util.Strings;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import javax.persistence.EntityManager;
+import javax.persistence.EntityTransaction;
+import javax.persistence.NoResultException;
+import javax.persistence.Query;
+
+/**
+ *
+ * @author shake
+ */
+public class IngestDirectory extends Thread{
+    private Collection coll;
+    private Set<String> identifiers;
+    private Set<String> existingParents = new HashSet<String>();
+    private EntityManager em = PersistUtil.getEntityManager();
+    private int numTransactions = 0;
+
+    public IngestDirectory(Set<String> identifiers , Collection coll){
+        this.identifiers = identifiers;
+        this.coll = coll;
+    }
+
+    @Override
+    public void run() {
+        if ( identifiers == null || coll == null ) {
+            return;
+        }
+
+        EntityTransaction trans = em.getTransaction();
+        trans.begin();
+        for ( String identifier : identifiers ) {
+            extractAndRegisterParentDirs(identifier);
+        }
+        trans.commit();
+
+    }
+
+    private void extractAndRegisterParentDirs(String path) {
+        // We don't have a FileBean, so build the pathList ourselves
+        StringBuilder fullPath = new StringBuilder(path);
+        List <String> pathList = new LinkedList<String>();
+        int index = 0;
+        while( (index = fullPath.lastIndexOf("/")) != 0 ) {
+            pathList.add(fullPath.toString());
+            fullPath.delete(index, fullPath.length());
+        }
+        pathList.add(fullPath.toString());
+
+        // Same as AuditThread, but with our pathList
+        String parentName = (pathList.size() > 1
+                ? pathList.get(1) : null);
+
+        // 1. make sure directory path is registered
+        if (parentName != null) {
+            parentName = Strings.cleanStringForXml(parentName, '_');
+            for ( int i = 1; i < pathList.size(); i++) {
+                String parent = (pathList.size() > i + 1 ? pathList.get(i+1) : null);
+                parent = Strings.cleanStringForXml(parent, '_');
+                createDirectory(pathList.get(i), parent);
+
+                if ( numTransactions > 10000 ) {
+                    em.flush();
+                    em.clear();
+                }
+            }
+        }
+    }
+
+    private void createDirectory(String directory, String root) {
+        MonitoredItem mi;
+        if ( existingParents.contains(directory) || directory == null ) {
+            return;
+        }
+        if ( (mi = getItemByPath(directory)) != null ) {
+            Date d = new Date();
+            mi.setLastSeen(d);
+            mi.setLastVisited(d);
+            mi.setState('A');
+            em.merge(mi);
+            numTransactions++;
+            existingParents.add(directory);
+        } else {
+            addItem(directory, root, true, 'A', 0);
+        }
+    }
+
+
+    public MonitoredItem getItemByPath( String path ) {
+        Query q = em.createNamedQuery("MonitoredItem.getItemByPath");
+        q.setParameter("path", path);
+        q.setParameter("coll", coll);
+        try {
+            return (MonitoredItem) q.getSingleResult();
+        } catch ( NoResultException ex ) {
+            return null;
+        }
+
+    }
+
+    public MonitoredItem addItem( String path, String parentDir,boolean directory,
+            char initialState, long size ) {
+
+        MonitoredItem mi = new MonitoredItem();
+        mi.setDirectory(directory);
+        mi.setLastSeen(new Date());
+        mi.setLastVisited(new Date());
+        mi.setStateChange(new Date());
+        mi.setParentCollection(coll);
+        mi.setParentPath(parentDir);
+        mi.setPath(path);
+        mi.setState(initialState);
+        mi.setSize(size);
+        em.persist(mi);
+        numTransactions++;
+
+        return mi;
+    }
+
+}

ace-am/src/main/java/edu/umiacs/ace/monitor/register/IngestStore.java

+/*
+ * Copyright (c) 2007-2010, University of Maryland
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted provided
+ * that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this list of conditions
+ * and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice, this list of conditions
+ * and the following disclaimer in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * Neither the name of the University of Maryland nor the names of its contributors may be used to
+ * endorse or promote products derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+ * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ACE Components were written in the ADAPT Project at the University of
+ * Maryland Institute for Advanced Computer Study.
+ */
+
 package edu.umiacs.ace.monitor.register;
 
 import edu.umiacs.ace.monitor.core.Collection;
 import edu.umiacs.ace.monitor.core.MonitoredItem;
+import edu.umiacs.ace.monitor.core.Token;
+import edu.umiacs.ace.monitor.log.LogEventManager;
+import edu.umiacs.ace.token.TokenStoreEntry;
+import edu.umiacs.ace.token.TokenStoreReader;
 import edu.umiacs.ace.util.EntityManagerServlet;
+import edu.umiacs.ace.util.TokenUtil;
+import edu.umiacs.util.Strings;
 import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
 import javax.persistence.EntityManager;
+import javax.servlet.RequestDispatcher;
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
+import javax.servlet.http.HttpSession;
+import org.apache.commons.fileupload.FileItemIterator;
+import org.apache.commons.fileupload.FileItemStream;
+import org.apache.commons.fileupload.FileUploadException;
+import org.apache.commons.fileupload.servlet.ServletFileUpload;
+import org.apache.commons.fileupload.util.Streams;
 //import org.apache.tomcat.util.http.fileupload.FileUploadBase;
 
 /**
- * TODO Servlet to ingest token stores
+ * Servlet to ingest token stores
  * @author toaster
  */
 public class IngestStore extends EntityManagerServlet {
-    
+    // private static final Logger LOG = Logger.getLogger(IngestStore.class);
+    private LogEventManager logManager;
+    // private long session;
+    public static final String PAGE_RESULTS = "results";
+
     @Override
     protected void processRequest( HttpServletRequest request, HttpServletResponse response,
             EntityManager em ) throws ServletException, IOException {
-
-        Collection coll = getCollection(request, em);
+        Collection coll = null; //getCollection(request, em);
         MonitoredItem item = getItem(request, em);
+        TokenStoreReader tokenReader = null;
+        RequestDispatcher dispatcher;
+        Map<String, Token> batchTokens = new HashMap<String, Token>();
 
         if (item != null && !item.isDirectory())
             throw new ServletException("Selected item is not a directory " + item.getPath());
 
-//        request.
+        if (!ServletFileUpload.isMultipartContent(request)) {
+            throw new ServletException("No file attached");
+        }
+
+        //        request.
+        ServletFileUpload upload = new ServletFileUpload();
+        try {
+            FileItemIterator iter = upload.getItemIterator(request);
+            while( iter.hasNext() ) {
+                FileItemStream fileItem = iter.next();
+                InputStream stream = fileItem.openStream();
+
+                if (fileItem.isFormField()) {
+
+                    // Basic find collection id and set the collection
+                    if ( PARAM_COLLECTION_ID.equals(fileItem.getFieldName()) ) {
+                        String collectionId = Streams.asString(stream);
+                        if ( Strings.isValidLong(collectionId) ) {
+                            long collId = Long.parseLong(collectionId);
+                            coll = em.getReference(Collection.class, collId);
+
+                            if ( coll == null ) {
+                                throw new ServletException("Collection Id " + 
+                                    collId + " not found.");
+                            }
+                        }
+                    }
+                } else {
+                    // Read tokens in from input stream and add to AceToken set
+                    // May want to log this stuff
+                    tokenReader = new TokenStoreReader(fileItem.openStream());
+
+                    if ( tokenReader == null ) {
+                        throw new ServletException("Token file is corrupt");
+                    }
+
+                    while ( tokenReader.hasNext() ) {
+                        TokenStoreEntry tokenEntry = tokenReader.next();
+                        Token token = TokenUtil.convertFromAceToken(tokenEntry.getToken());
+
+                        if ( !token.getProofAlgorithm().equals(coll.getDigestAlgorithm()) ) {
+                            throw new ServletException("Token digest differs from"
+                                    + " collection digest.");
+                        }
+                        batchTokens.put(tokenEntry.getIdentifiers().get(0), token);
+                    }
+                }
+
+            }
+        } catch (FileUploadException ex) {
+            throw new ServletException(ex);
+        }
+
+        // Sanity check
+        if ( coll == null || tokenReader == null ) {
+            throw new ServletException("Bad upload parameters");
+        }
+
+        IngestThreadFactory threads = new IngestThreadFactory(batchTokens, coll);
+        HttpSession session = request.getSession();
+        session.setAttribute(PAGE_RESULTS, threads);
+        threads.start();
+