Skip to content
GitLab
Commit ad2fe4aa authored by Michael Ritter's avatar Michael Ritter
Browse files

Merge branch 'release-1.1'

parents 1f882e04 dd02c7fc
Hide whitespace changes
Inline Side-by-side
amqp-core/pom.xml
View file @ ad2fe4aa
......@@ -5,7 +5,7 @@
<parent>
<groupId>org.chronopolis</groupId>
<artifactId>chronopolis</artifactId>
<version>1.0.1-RELEASE</version>
<version>1.1.0-RELEASE</version>
</parent>
<packaging>jar</packaging>
......
chron-db/pom.xml
View file @ ad2fe4aa
......@@ -6,11 +6,11 @@
<parent>
<groupId>org.chronopolis</groupId>
<artifactId>chronopolis</artifactId>
<version>1.0.1-RELEASE</version>
<version>1.1.0-RELEASE</version>
</parent>
<groupId>org.chronopolis</groupId>
<artifactId>chron-db</artifactId>
<version>1.0.1-RELEASE</version>
<version>1.1.0-RELEASE</version>
<dependencies>
......@@ -40,4 +40,4 @@
</dependencies>
</project>
\ No newline at end of file
</project>
common/pom.xml
View file @ ad2fe4aa
......@@ -5,12 +5,12 @@
<parent>
<groupId>org.chronopolis</groupId>
<artifactId>chronopolis</artifactId>
<version>1.0.1-RELEASE</version>
<version>1.1.0-RELEASE</version>
</parent>
<groupId>org.chronopolis</groupId>
<artifactId>chron-common</artifactId>
<version>1.0.1-RELEASE</version>
<version>1.1.0-RELEASE</version>
<name>Common</name>
<url>http://maven.apache.org</url>
<properties>
......
common/src/main/java/org/chronopolis/common/ace/Tokenizer.java
View file @ ad2fe4aa
......@@ -6,16 +6,21 @@ import edu.umiacs.ace.ims.api.TokenRequestBatch;
import edu.umiacs.ace.ims.ws.TokenRequest;
import org.chronopolis.common.digest.Digest;
import org.chronopolis.common.digest.DigestUtil;
import org.chronopolis.common.util.Filter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Formatter;
import java.util.HashSet;
import java.util.Set;
import java.util.regex.Pattern;
/**
* Class to create ACE tokens from a BagIt bag.
......@@ -25,6 +30,23 @@ import java.util.Set;
* Created by shake on 2/4/15.
*/
public class Tokenizer {
/**
* Factory class to allow for mocking of IMS connections
*
*/
public static class IMSFactory {
public TokenRequestBatch createIMSConnection(RequestBatchCallback callback) {
IMSService ims;
// TODO: Use the AceSettings to get the ims host name
ims = IMSService.connect("ims.umiacs.umd.edu", SSL_PORT, true);
return ims.createImmediateTokenRequestBatch("SHA-256",
callback,
MAX_QUEUE_LEN,
TIMEOUT);
}
}
private static final int SSL_PORT = 443;
private static final int MAX_QUEUE_LEN = 1000;
private static final int TIMEOUT = 5000;
......@@ -40,17 +62,22 @@ public class Tokenizer {
private Path tagmanifest;
private String tagIdentifier;
private String tagDigest;
private Set<Path> extraTagmanifests;
private final RequestBatchCallback callback;
private TokenRequestBatch batch;
private IMSFactory factory;
public Tokenizer(final Path bag,
final String fixityAlgorithm,
final RequestBatchCallback callback) {
this.bag = bag;
this.fixityAlgorithm = Digest.fromString(fixityAlgorithm);
this.extraTagmanifests = new HashSet<>();
this.callback = callback;
this.tagDigest = null;
this.factory = new Tokenizer.IMSFactory();
addManifests();
}
......@@ -59,12 +86,12 @@ public class Tokenizer {
+ fixityAlgorithm.getBagitIdentifier()
+ ".txt";
Path tagManifest = bag.resolve("tagmanifest-"
+ fixityAlgorithm.getBagitIdentifier()
+ ".txt");
Path tagManifest = bag.resolve(tagIdentifier);
Path manifest = bag.resolve("manifest-"
+ fixityAlgorithm.getBagitIdentifier()
+ ".txt");
// ACE holds files in a relative context, without the preceding directories
Path aceTag = Paths.get("/tagmanifest-"
+ fixityAlgorithm.getBagitIdentifier()
+ ".txt");
......@@ -81,6 +108,19 @@ public class Tokenizer {
this.aceTag = aceTag;
this.manifest = manifest;
this.tagmanifest = tagManifest;
scanForExtra();
}
private void scanForExtra() {
String identifier = fixityAlgorithm.getBagitIdentifier();
try (DirectoryStream<Path> stream = Files.newDirectoryStream(bag, new TagFilter(identifier))) {
for (Path path : stream) {
extraTagmanifests.add(path);
}
} catch (IOException e) {
throw new RuntimeException("Could not read bag directory");
}
}
/**
......@@ -91,8 +131,8 @@ public class Tokenizer {
* @throws IOException
* @throws InterruptedException
*/
public void tokenize(Set<Path> filter) throws IOException, InterruptedException {
batch = createIMSConnection();
public void tokenize(Filter<Path> filter) throws IOException, InterruptedException {
batch = factory.createIMSConnection(callback);
try {
// Digest the manifest
......@@ -105,6 +145,16 @@ public class Tokenizer {
// Then the tag-manifest
tokenize(filter, tagmanifest);
// Digest any extra tagmanifests
for (Path tag: extraTagmanifests) {
String alg = fixityAlgorithm.getName();
if (!filter.contains(tag)) {
String digest = DigestUtil.digest(tag, alg);
addTokenRequest(tag, digest);
filter.add(tag);
}
}
} finally {
// Make sure the batch gets closed if we are interrupted
batch.close();
......@@ -123,7 +173,7 @@ public class Tokenizer {
* @throws IOException
* @throws InterruptedException
*/
private boolean tokenize(Set<Path> filter, Path manifest) throws IOException,
private boolean tokenize(Filter<Path> filter, Path manifest) throws IOException,
InterruptedException {
String line;
boolean corrupt = false;
......@@ -153,6 +203,7 @@ public class Tokenizer {
if (digest.equals(calculatedDigest)) {
addTokenRequest(path, digest);
filter.add(ace.resolve(rel));
} else {
log.error("Error in file {}: digest found {} (expected {})",
new Object[]{
......@@ -201,20 +252,22 @@ public class Tokenizer {
batch.add(req);
}
/**
* Create a connection to the IMS Service for ACE
*
* @return {@link TokenRequestBatch}
*/
private TokenRequestBatch createIMSConnection() {
IMSService ims;
// TODO: Use the AceSettings to get the ims host name
ims = IMSService.connect("ims.umiacs.umd.edu", SSL_PORT, true);
return ims.createImmediateTokenRequestBatch("SHA-256",
callback,
MAX_QUEUE_LEN,
TIMEOUT);
}
private class TagFilter implements DirectoryStream.Filter<Path> {
// Use a negative look ahead with a word boundary to exclude certain words
private final String regex = "tagmanifest-(?!\\b%s\\b)\\w+.txt";
private final Pattern pattern;
private TagFilter(String tagType) {
Formatter f = new Formatter();
this.pattern = Pattern.compile(f.format(regex, tagType).toString());
}
@Override
public boolean accept(Path path) throws IOException {
// We don't care about using what was found, just if it exists
return pattern.matcher(path.getFileName().toString()).find();
}
}
}
common/src/main/java/org/chronopolis/common/dpn/DPNBag.java deleted 100644 → 0
View file @ 1f882e04
package org.chronopolis.common.dpn;
import org.joda.time.DateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* Representation of a Bag in the DPN-REST Server
*
* Created by shake on 5/6/15.
*/
public class DPNBag {
private char bagType; // One of 'R', 'D', 'I'
private String uuid;
private String localId;
private String firstVersionUuid;
private String ingestNode;
private String adminNode;
private List<String> interpretive;
private List<String> rights;
private List<String> replicatingNodes;
private List<Fixity> fixities;
private DateTime createdAt;
private DateTime updatedAt;
private Long size;
private Long version;
public DPNBag() {
this.interpretive = new ArrayList<>();
this.rights = new ArrayList<>();
this.replicatingNodes = new ArrayList<>();
this.fixities = new ArrayList<>();
}
public char getBagType() {
return bagType;
}
public DPNBag setBagType(char bagType) {
this.bagType = bagType;
return this;
}
public String getUuid() {
return uuid;
}
public DPNBag setUuid(String uuid) {
this.uuid = uuid;
return this;
}
public String getLocalId() {
return localId;
}
public DPNBag setLocalId(String localId) {
this.localId = localId;
return this;
}
public String getFirstVersionUuid() {
return firstVersionUuid;
}
public DPNBag setFirstVersionUuid(String firstVersionUuid) {
this.firstVersionUuid = firstVersionUuid;
return this;
}
public String getIngestNode() {
return ingestNode;
}
public DPNBag setIngestNode(String ingestNode) {
this.ingestNode = ingestNode;
return this;
}
public String getAdminNode() {
return adminNode;
}
public DPNBag setAdminNode(String adminNode) {
this.adminNode = adminNode;
return this;
}
public List<String> getInterpretive() {
return interpretive;
}
public DPNBag setInterpretive(List<String> interpretive) {
this.interpretive = interpretive;
return this;
}
public DPNBag addInterpretive(String interpretive) {
this.interpretive.add(interpretive);
return this;
}
public List<String> getRights() {
return rights;
}
public DPNBag setRights(List<String> rights) {
this.rights = rights;
return this;
}
public DPNBag addRights(String uuid) {
rights.add(uuid);
return this;
}
public List<String> getReplicatingNodes() {
return replicatingNodes;
}
public DPNBag setReplicatingNodes(List<String> replicatingNodes) {
this.replicatingNodes = replicatingNodes;
return this;
}
public DPNBag addReplicatingNode(String name) {
replicatingNodes.add(name);
return this;
}
public List<Fixity> getFixities() {
return fixities;
}
public DPNBag setFixities(List<Fixity> fixities) {
this.fixities = fixities;
return this;
}
/**
* TODO: Do a get before adding a new value
*
* @param algorithm
* @param digest
* @return
*/
public DPNBag addFixity(String algorithm, String digest) {
this.fixities.add(new Fixity(digest));
return this;
}
public DateTime getCreatedAt() {
return createdAt;
}
public DPNBag setCreatedAt(DateTime createdAt) {
this.createdAt = createdAt;
return this;
}
public DateTime getUpdatedAt() {
return updatedAt;
}
public DPNBag setUpdatedAt(DateTime updatedAt) {
this.updatedAt = updatedAt;
return this;
}
public Long getSize() {
return size;
}
public DPNBag setSize(Long size) {
this.size = size;
return this;
}
public Long getVersion() {
return version;
}
public DPNBag setVersion(Long version) {
this.version = version;
return this;
}
}
common/src/main/java/org/chronopolis/common/dpn/DPNService.java deleted 100644 → 0
View file @ 1f882e04
package org.chronopolis.common.dpn;
import retrofit.http.Body;
import retrofit.http.POST;
/**
* Created by shake on 9/30/14.
*/
public interface DPNService {
@POST("/api-v1/bag/")
DPNBag createBag(@Body DPNBag bag);
}
common/src/main/java/org/chronopolis/common/dpn/Fixity.java deleted 100644 → 0
View file @ 1f882e04
package org.chronopolis.common.dpn;
/**
* Needed because the django serialization is broken
*
* Created by shake on 5/14/15.
*/
public class Fixity {
private String sha256;
public Fixity() {
}
public Fixity(String sha256) {
this.sha256 = sha256;
}
public String getSha256() {
return sha256;
}
public Fixity setSha256(String sha256) {
this.sha256 = sha256;
return this;
}
}
common/src/main/java/org/chronopolis/common/dpn/TokenInterceptor.java
View file @ ad2fe4aa
......@@ -15,7 +15,7 @@ public class TokenInterceptor implements RequestInterceptor {
@Override
public void intercept(RequestFacade requestFacade) {
String tokenAuth = "token " + token;
String tokenAuth = "Token token=" + token;
requestFacade.addHeader("Authorization", tokenAuth);
requestFacade.addHeader("Accept", "*/*");
......
common/src/main/java/org/chronopolis/common/settings/DPNSettings.java
View file @ ad2fe4aa
......@@ -20,6 +20,9 @@ public class DPNSettings {
@Value("${dpn.api-key:admin}")
private String apiKey;
@Value("${dpn.node:chron}")
private String dpnNode;
private List<String> dpnEndpoints;
public String getApiKey() {
......@@ -44,4 +47,12 @@ public class DPNSettings {
}
}
public String getDpnNode() {
return dpnNode;
}
public DPNSettings setDpnNode(String dpnNode) {
this.dpnNode = dpnNode;
return this;
}
}
common/src/main/java/org/chronopolis/common/util/Filter.java 0 → 100644
View file @ ad2fe4aa
package org.chronopolis.common.util;
/**
* Representation of a simple filter
*
* Created by shake on 8/24/15.
*/
public interface Filter<E> {
boolean add(E e);
boolean contains(E e);
}
duracloud-backend/pom.xml
View file @ ad2fe4aa
......@@ -7,32 +7,49 @@
<parent>
<groupId>org.chronopolis</groupId>
<artifactId>chronopolis</artifactId>
<version>1.0.1-RELEASE</version>
<version>1.1.0-RELEASE</version>
</parent>
<groupId>org.chronopolis</groupId>
<artifactId>duracloud-backend</artifactId>
<version>1.0.1-RELEASE</version>
<version>1.1.0-RELEASE</version>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<dependencies>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>springloaded</artifactId>
<version>1.2.4.RELEASE</version>
</dependency>
</dependencies>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>org.chronopolis</groupId>
<artifactId>rest-common</artifactId>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
</dependency>
<dependency>
<groupId>org.chronopolis</groupId>
<artifactId>amqp-core</artifactId>
<artifactId>rest-common</artifactId>
</dependency>
<dependency>
<groupId>org.chronopolis</groupId>
<artifactId>chron-messaging</artifactId>
<artifactId>chron-common</artifactId>
</dependency>
<dependency>
<groupId>org.chronopolis</groupId>
<artifactId>chron-db</artifactId>
<artifactId>earth-core</artifactId>
</dependency>
<dependency>
......@@ -65,4 +82,4 @@
</dependencies>
</project>
\ No newline at end of file
</project>
duracloud-backend/src/main/java/org/chronopolis/intake/duracloud/Application.java 0 → 100644
View file @ ad2fe4aa
package org.chronopolis.intake.duracloud;
import org.chronopolis.common.ace.CredentialRequestInterceptor;
import org.chronopolis.common.mail.MailUtil;
import org.chronopolis.common.settings.DPNSettings;
import org.chronopolis.common.settings.IngestAPISettings;
import org.chronopolis.common.settings.SMTPSettings;
import org.chronopolis.earth.api.LocalAPI;
import org.chronopolis.intake.duracloud.batch.BaggingTasklet;
import org.chronopolis.intake.duracloud.batch.ReplicationTasklet;
import org.chronopolis.intake.duracloud.batch.SnapshotJobManager;
import org.chronopolis.intake.duracloud.batch.SnapshotTasklet;
import org.chronopolis.intake.duracloud.batch.support.APIHolder;
import org.chronopolis.intake.duracloud.config.IntakeSettings;
import org.chronopolis.intake.duracloud.remote.BridgeAPI;
import org.chronopolis.intake.duracloud.scheduled.Bridge;
import org.chronopolis.rest.api.ErrorLogger;
import org.chronopolis.rest.api.IngestAPI;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.JobScope;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;