Commit 0fed3842 authored by Michael Ritter's avatar Michael Ritter
Browse files

Decrease the size we flush our entitymanager at and reduce number of persists

parent de27cb5b
...@@ -106,7 +106,7 @@ public class IngestDirectory extends RecursiveAction { ...@@ -106,7 +106,7 @@ public class IngestDirectory extends RecursiveAction {
parent = Strings.cleanStringForXml(parent, '_'); parent = Strings.cleanStringForXml(parent, '_');
createDirectory(pathList.get(i), parent); createDirectory(pathList.get(i), parent);
if ( numTransactions > 10000 ) { if ( numTransactions % 30 == 0 ) {
em.flush(); em.flush();
em.clear(); em.clear();
} }
......
...@@ -174,14 +174,14 @@ public class IngestThread extends RecursiveAction { ...@@ -174,14 +174,14 @@ public class IngestThread extends RecursiveAction {
token.setParentCollection(coll); token.setParentCollection(coll);
// Token // Token
em.persist(event[1]); // em.persist(token);
em.persist(token);
item.setToken(token); item.setToken(token);
//Finish adding the item //Finish adding the item
em.persist(event[0]); em.persist(event[0]);
em.merge(item); em.persist(event[1]);
numTransactions += 4; em.persist(item);
numTransactions += 3;
newTokens.add(identifier); newTokens.add(identifier);
}else{ }else{
...@@ -193,7 +193,7 @@ public class IngestThread extends RecursiveAction { ...@@ -193,7 +193,7 @@ public class IngestThread extends RecursiveAction {
// With large Token Stores, we get a large number of transactions // With large Token Stores, we get a large number of transactions
// Flushing and Clearing the EM helps to clear some memory // Flushing and Clearing the EM helps to clear some memory
// TODO: W/ fork join this isn't needed anymore, unless we want to flush at a lower number // TODO: W/ fork join this isn't needed anymore, unless we want to flush at a lower number
if ( numTransactions > 7000 ) { if ( numTransactions > 30 ) {
em.flush(); em.flush();
em.clear(); em.clear();
numTransactions = 0; numTransactions = 0;
...@@ -229,12 +229,13 @@ public class IngestThread extends RecursiveAction { ...@@ -229,12 +229,13 @@ public class IngestThread extends RecursiveAction {
if ( update ) { if ( update ) {
LogEvent event = logManager.createItemEvent(LogEnum.TOKEN_INGEST_UPDATE, LogEvent event = logManager.createItemEvent(LogEnum.TOKEN_INGEST_UPDATE,
identifier, coll.getDirectory() + identifier); identifier, coll.getDirectory() + identifier);
em.persist(token); // em.persist(token);
item.setToken(token); item.setToken(token);
// TODO: Why set 'I'? It's not necessarily invalid, maybe 'R' would be better
item.setState('I'); item.setState('I');
em.merge(item); em.merge(item);
em.persist(event); em.persist(event);
numTransactions += 3; numTransactions += 2;
updatedTokens.add(identifier); updatedTokens.add(identifier);
}else{ }else{
unchangedTokens.add(identifier); unchangedTokens.add(identifier);
...@@ -286,8 +287,8 @@ public class IngestThread extends RecursiveAction { ...@@ -286,8 +287,8 @@ public class IngestThread extends RecursiveAction {
mi.setState(initialState); mi.setState(initialState);
mi.setSize(size); mi.setSize(size);
em.persist(mi); // em.persist(mi);
numTransactions++; // numTransactions++;
return mi; return mi;
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment