Commit 24b17dea authored by toaster's avatar toaster
Browse files

jackrabbit sample and collection compare patch

git-svn-id: https://subversion.umiacs.umd.edu/ace/trunk@134 f1b3a171-7291-4a19-a512-95ad0ad9394a
parent ea7687f7
......@@ -175,20 +175,36 @@
<artifactId>irods-api</artifactId>
<version>1.6</version>
</dependency>
<!--<dependency>
<groupId>org.apache.jackrabbit</groupId>
<artifactId>jackrabbit-core</artifactId>
<version>2.2.7</version>
<exclusions>
<exclusion>
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>jcl-over-slf4j</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
</exclusions>
</dependency> -->
<!-- web services -->
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-server</artifactId>
<version>1.6</version>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
<version>1.6</version>
</dependency>
</dependencies>
</project>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-server</artifactId>
<version>1.6</version>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
<version>1.6</version>
</dependency>
</dependencies>
</project>
......@@ -36,14 +36,20 @@ import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import javax.persistence.EntityManager;
import javax.persistence.Query;
import javax.sql.DataSource;
import org.apache.log4j.Logger;
import org.eclipse.persistence.config.QueryHints;
import org.eclipse.persistence.queries.ScrollableCursor;
/**
* Currently does all comparison in memory, this limits collections to a million
......@@ -54,12 +60,15 @@ import org.apache.log4j.Logger;
*/
public final class CollectionCompare2 {
private Map<String, String> sourceMap = new HashMap<String, String>();
private Map<String, String> sourceReverseMap = new HashMap<String, String>();
private Map<String, String> sourceMap;
private Map<String, String> sourceReverseMap;
private static final Logger LOG = Logger.getLogger(CollectionCompare2.class);
private List<String> parseErrors = new ArrayList<String>();
public CollectionCompare2(InputStream sourceFile, String prefix) {
public CollectionCompare2(InputStream sourceFile, String prefix, int hint) {
sourceMap = new TreeMap<String, String>();
sourceReverseMap = new TreeMap<String, String>();
LOG.trace("initializing collection compare with hint: " + hint);
try {
parseInputStream(sourceFile, prefix);
} catch (IOException e) {
......@@ -80,19 +89,48 @@ public final class CollectionCompare2 {
EntityManager em = PersistUtil.getEntityManager();
long time = System.currentTimeMillis();
long total = 0;
DataSource db;
MonitoredItem mi;
Connection conn = null;
PreparedStatement stmt = null;
ResultSet rs = null;
try {
LOG.info("Starting collection compare on " + c.getName() + " source size: " + sourceMap.size());
Query q = em.createNamedQuery("MonitoredItem.listFilesInCollection");
q.setParameter("coll", c);
List items = q.getResultList();
for (Object o : items) {
db = PersistUtil.getDataSource();
conn = db.getConnection();
stmt = conn.prepareStatement(
"SELECT monitored_item.PATH, monitored_item.FILEDIGEST, monitored_item.TOKEN_ID "
+ "FROM monitored_item "
+ "WHERE monitored_item.PARENTCOLLECTION_ID = ? "
+ "AND monitored_item.DIRECTORY = 0",
ResultSet.TYPE_FORWARD_ONLY,
ResultSet.CONCUR_READ_ONLY);
stmt.setLong(1, c.getId());
stmt.setFetchSize(Integer.MIN_VALUE);
rs = stmt.executeQuery();
// Query q = em.createNamedQuery("MonitoredItem.listFilesInCollection");
//// q.setLockMode(LockModeType.NONE);
// q.setHint(QueryHints.JDBC_FETCH_SIZE, 5000);
//// q.setHint(QueryHints.RESULT_SET_TYPE, ResultSetType.ForwardOnly);
// q.setHint("eclipselink.cursor.scrollable",true);
// q.setParameter("coll", c);
//// em.getTransaction().begin();
// ScrollableCursor cursor = (ScrollableCursor) q.getSingleResult();
//// List items = q.getResultList();
//
//// for (Object o : items) {
// Object o;
// while ((o = cursor.next()) != null) {
while (rs.next()) {
total++;
MonitoredItem aceItem = (MonitoredItem) o;
String acePath = aceItem.getPath();
String aceDigest = aceItem.getFileDigest();
if ((total % 100000) == 0) {
LOG.trace("Compared " + total);
}
// MonitoredItem aceItem = (MonitoredItem) o;
String acePath = rs.getString(1);//aceItem.getPath();
String aceDigest = rs.getString(2);//aceItem.getFileDigest();
if (sourceMap.containsKey(acePath)) {
cr.fileExistsAtTarget(acePath);
......@@ -113,12 +151,13 @@ public final class CollectionCompare2 {
LOG.error("Error during load and compare: ", e);
} finally {
SQL.release(rs);
SQL.release(conn);
LOG.info("Finished collection compare on: "
+ c.getName() + " time: " + (System.currentTimeMillis() - time) + " tested: " + total);
cr.finished();
em.close();
}
}
private void parseInputStream(InputStream sourceFile, String prefix) throws IOException {
......@@ -126,25 +165,50 @@ public final class CollectionCompare2 {
BufferedReader input = new BufferedReader(new InputStreamReader(sourceFile));
String line = input.readLine();
long total = 0;
// ignore ace manifest header
if (line != null && line.matches("^[A-Z0-9\\-]+:.+$")) {
line = input.readLine();
}
while (line != null) {
total++;
if ((total % 100000) == 0) {
LOG.trace("Loaded " + total);
}
String tokens[] = line.split("\\s+", 2);
if (tokens == null || tokens.length != 2) {
LOG.error("Error processing line: " + line);
parseErrors.add("Corrupt Line: " + line);
} else {
String path = tokens[1];
// temp hack to make sure all paths start w/ / as ace expectes
if (!path.startsWith("/"))
if (!path.startsWith("/")) {
path = "/" + path;
}
sourceMap.put(path, tokens[0]);
sourceReverseMap.put(tokens[0], path);
}
line = input.readLine();
}
}
}
......@@ -20,20 +20,20 @@ import java.util.TreeSet;
public final class CompareResults {
private Set<String> unseenSupplied;
private Set<String> unseenTarget = new HashSet<String>();
private Set<String> unseenTarget;
private Set<DifferingName> differingNames = new TreeSet<DifferingName>();
private Set<DifferingDigest> differingDigests = new TreeSet<DifferingDigest>();
private boolean running = true;
private String message = null;
public CompareResults(CollectionCompare2 cc2) {
unseenSupplied = new HashSet<String>(cc2.getSourceMap().keySet());
public CompareResults(CollectionCompare2 cc2, int hint) {
unseenTarget = new TreeSet<String>();
//differingNames = new TreeSet<DifferingName>(hint);
unseenSupplied = new TreeSet<String>(cc2.getSourceMap().keySet());
}
void finished()
{
running = false;
void finished() {
running = false;
}
public boolean isRunning() {
......@@ -43,6 +43,7 @@ public final class CompareResults {
public String getMessage() {
return message;
}
void fileExistsAtTarget(String file) {
unseenSupplied.remove(file);
}
......
......@@ -31,6 +31,7 @@
package edu.umiacs.ace.monitor.compare;
import edu.umiacs.ace.monitor.access.CollectionCountContext;
import edu.umiacs.ace.util.EntityManagerServlet;
import edu.umiacs.ace.remote.JsonGateway;
import edu.umiacs.ace.monitor.peers.PartnerSite;
......@@ -75,6 +76,7 @@ public class CompareServlet extends EntityManagerServlet {
protected void processRequest( HttpServletRequest request,
HttpServletResponse response, EntityManager em )
throws ServletException, IOException {
int hint = 0;
Collection c = null;
MonitoredItem monItem = null;
RequestDispatcher dispatcher;
......@@ -109,6 +111,10 @@ public class CompareServlet extends EntityManagerServlet {
"Collection does not exist: " + colId);
}
hint = (int)CollectionCountContext.getFileCount(c);
if (hint == -1)
hint = 10000;
}
} else if ( PARAM_ITEM_ID.equals(item.getFieldName()) ) {
......@@ -155,8 +161,8 @@ public class CompareServlet extends EntityManagerServlet {
"item " + monItem + " filter " + inputFilter + " loading attached file");
CollectionCompare2 cc = new CollectionCompare2(
item.openStream(), inputFilter);
CompareResults cr = new CompareResults(cc);
item.openStream(), inputFilter,hint);
CompareResults cr = new CompareResults(cc,hint);
Thread t = new Thread(new TableCompareRunnable(cr,cc, c,
monItem), "Compare Thread " + c.getName());
t.start();
......@@ -172,8 +178,8 @@ public class CompareServlet extends EntityManagerServlet {
LOG.debug("Remote digest request " + partner.getRemoteURL());
CollectionCompare2 cc = new CollectionCompare2(
JsonGateway.getGateway().getDigestList(partner,
remoteCollection), inputFilter);
CompareResults cr = new CompareResults(cc);
remoteCollection), inputFilter,hint);
CompareResults cr = new CompareResults(cc,hint);
Thread t = new Thread(new TableCompareRunnable(cr,cc, c,
monItem), "Compare Thread " + c.getName());
......
......@@ -28,7 +28,6 @@
* Maryland Institute for Advanced Computer Study.
*/
// $Id$
package edu.umiacs.ace.monitor.reporting;
import edu.umiacs.ace.util.PersistUtil;
......@@ -72,13 +71,13 @@ public class SchedulerContextListener implements ServletContextListener {
private static String mailfrom;
@Override
public void contextInitialized( ServletContextEvent event ) {
public void contextInitialized(ServletContextEvent event) {
NDC.push("[schedstartup]");
try {
mailserver = event.getServletContext().getInitParameter(
PARAM_SMTP_SERVER);
mailfrom = event.getServletContext().getInitParameter(PARAM_FROM);
if ( Strings.isEmpty(mailserver) ) {
if (Strings.isEmpty(mailserver)) {
mailserver = "127.0.0.1";
}
......@@ -89,10 +88,10 @@ public class SchedulerContextListener implements ServletContextListener {
EntityManager em = PersistUtil.getEntityManager();
Query q = em.createNamedQuery("ReportPolicy.listAll");
for ( Object o : q.getResultList() ) {
for (Object o : q.getResultList()) {
addJob((ReportPolicy) o);
}
} catch ( Exception ex ) {
} catch (Exception ex) {
LOG.error("Error starting report scheduling", ex);
} finally {
NDC.pop();
......@@ -101,26 +100,28 @@ public class SchedulerContextListener implements ServletContextListener {
@Override
public void contextDestroyed(
ServletContextEvent arg0 ) {
ServletContextEvent arg0) {
try {
sched.shutdown();
if (sched != null) { // npe if startup never worked
sched.shutdown();
}
LOG.debug("Shutting down the scheduler");
} catch ( Exception ex ) {
} catch (Exception ex) {
LOG.error("Error stopping report scheduling", ex);
}
}
public static final void removeJob( ReportPolicy job ) {
public static final void removeJob(ReportPolicy job) {
try {
LOG.debug("Removing job " + job.getName());
sched.deleteJob("report-" + job.getId(), null);
} catch ( SchedulerException e ) {
} catch (SchedulerException e) {
LOG.error("Error removing " + job.getName(), e);
throw new RuntimeException("Error removeing job " + job.getName());
}
}
public static final void addJob( ReportPolicy job ) {
public static final void addJob(ReportPolicy job) {
JobDetail detail = new JobDetail("report-" + job.getId(), null,
ReportJob.class);
detail.getJobDataMap().put(ReportJob.ATTR_POLICY, job);
......@@ -131,19 +132,19 @@ public class SchedulerContextListener implements ServletContextListener {
LOG.debug(
"Adding job " + job.getName() + " cron " + job.getCronString());
sched.scheduleJob(detail, cronTrig);
} catch ( ParseException pe ) {
} catch (ParseException pe) {
LOG.error("Error creating trigger", pe);
throw new RuntimeException(
"Error parsing cron trigger in job" + job.getName());
} catch ( SchedulerException e ) {
} catch (SchedulerException e) {
LOG.error("Error adding " + job.getName(), e);
throw new RuntimeException("Error adding job " + job.getName());
}
}
public static void mailReport( ReportSummary report, String[] mailList )
public static void mailReport(ReportSummary report, String[] mailList)
throws MessagingException {
if ( report == null || mailList == null || mailList.length == 0 ) {
if (report == null || mailList == null || mailList.length == 0) {
return;
}
......@@ -165,7 +166,7 @@ public class SchedulerContextListener implements ServletContextListener {
msg.setFrom(addressFrom);
InternetAddress[] addressTo = new InternetAddress[mailList.length];
for ( int i = 0; i < mailList.length; i++ ) {
for (int i = 0; i < mailList.length; i++) {
addressTo[i] = new InternetAddress(mailList[i]);
}
msg.setRecipients(Message.RecipientType.TO, addressTo);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment