Commit e5b8bc76 authored by Michael Ritter's avatar Michael Ritter
Browse files

#49 Remove All support

parent d6ecc6f0
......@@ -50,7 +50,7 @@ import javax.persistence.TemporalType;
/**
*
* state:
* state:
* A - active
* C - corrupt, file and checksum do not match
* M - local file is missing
......@@ -94,6 +94,8 @@ import javax.persistence.TemporalType;
"SELECT count(m) FROM MonitoredItem m WHERE m.directory = true AND m.parentCollection = :coll"),
@NamedQuery(name = "MonitoredItem.countErrorsInCollection", query =
"SELECT count(m) FROM MonitoredItem m WHERE m.state <> 'A' AND m.parentCollection = :coll"),
@NamedQuery(name = "MonitoredItem.itemsByState", query =
"SELECT m FROM MonitoredItem m WHERE m.parentCollection = :coll AND m.directory = false AND m.state = :state ORDER BY m.id"),
@NamedQuery(name = "MonitoredItem.listDuplicates", query =
"SELECT m FROM MonitoredItem m WHERE m.fileDigest = :digest AND m.parentCollection = :coll"),
@NamedQuery(name = "MonitoredItem.listNullTokens", query =
......
......@@ -39,19 +39,28 @@ import edu.umiacs.ace.monitor.log.LogEnum;
import edu.umiacs.ace.monitor.log.LogEventManager;
import edu.umiacs.ace.util.EntityManagerServlet;
import edu.umiacs.ace.util.PersistUtil;
import edu.umiacs.sql.SQL;
import edu.umiacs.util.Strings;
import org.apache.log4j.Logger;
import javax.annotation.Nullable;
import javax.persistence.EntityManager;
import javax.persistence.EntityNotFoundException;
import javax.persistence.EntityTransaction;
import javax.persistence.TypedQuery;
import javax.servlet.RequestDispatcher;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.HttpSession;
import javax.sql.DataSource;
import java.io.IOException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.HashSet;
import java.util.Locale;
import java.util.Optional;
import java.util.Set;
/**
......@@ -62,45 +71,45 @@ import java.util.Set;
*/
public class RemoveItemServlet extends EntityManagerServlet {
public static final String PARAM_REDIRECT = "redirect";
public static final String DEFAULT_REDIRECT = "browse.jsp";
public static final String REMOVAL = "removal";
private static final String PARAM_TYPE = "type";
private static final String PARAM_REDIRECT = "redirect";
private static final String DEFAULT_REDIRECT = "browse.jsp";
private static final String REMOVAL = "removal";
private static final Logger LOG = Logger.getLogger(RemoveItemServlet.class);
@Override
protected void processRequest( HttpServletRequest request,
HttpServletResponse response, EntityManager em ) throws ServletException, IOException {
MonitoredItem item;
Set<Collection> mutations;
protected void processRequest(HttpServletRequest request,
HttpServletResponse response, EntityManager em) throws ServletException, IOException {
long[] itemIds;
Set<Collection> mutations;
HttpSession session = request.getSession();
DirectoryTree dt =
(DirectoryTree) session.getAttribute(BrowseServlet.SESSION_DIRECTORY_TREE);
long itemId = getParameter(request, PARAM_ITEM_ID, 0);
long eventSession = System.currentTimeMillis();
if ( itemId > 0 ) {
try {
item = em.getReference(MonitoredItem.class, itemId);
removeItem(item, em, eventSession, dt);
mutations = ImmutableSet.of(item.getParentCollection());
} catch (EntityNotFoundException exception) {
mutations = ImmutableSet.of();
LOG.warn("MonitoredItem " + itemId + " already removed. Ignoring.", exception);
}
String type = getParameter(request, PARAM_TYPE, null);
long itemId = getParameter(request, PARAM_ITEM_ID, 0);
long collectionid = getParameter(request, PARAM_COLLECTION_ID, 0);
if (type != null && !type.isEmpty() && collectionid > 0) {
mutations = referenceFor(Collection.class, collectionid, em)
.map(c -> removeForType(c, type, em, eventSession))
.orElseGet(ImmutableSet::of);
} else if (itemId > 0) {
mutations = referenceFor(MonitoredItem.class, itemId, em)
.map(mi -> removeItem(mi, em, eventSession, dt))
.map(ImmutableSet::of).orElseGet(ImmutableSet::of);
} else {
itemIds = getParameterList(request,REMOVAL, 0);
itemIds = getParameterList(request, REMOVAL, 0);
mutations = new HashSet<>();
if(itemIds != null) {
for(long l:itemIds) {
if(l > 0) {
try {
item = em.getReference(MonitoredItem.class, l);
removeItem(item, em, eventSession, dt);
mutations.add(item.getParentCollection());
} catch (EntityNotFoundException exception) {
LOG.warn("MonitoredItem " + itemId + " already removed. Ignoring.", exception);
}
if (itemIds != null) {
for (long l : itemIds) {
if (l > 0) {
referenceFor(MonitoredItem.class, l, em)
.map(i -> removeItem(i, em, eventSession, dt))
.ifPresent(mutations::add);
}
}
}
......@@ -112,7 +121,7 @@ public class RemoveItemServlet extends EntityManagerServlet {
}
String redirect = request.getParameter(PARAM_REDIRECT);
if ( Strings.isEmpty(redirect) ) {
if (Strings.isEmpty(redirect)) {
redirect = DEFAULT_REDIRECT;
}
......@@ -120,37 +129,123 @@ public class RemoveItemServlet extends EntityManagerServlet {
dispatcher.forward(request, response);
}
private void removeItem(MonitoredItem item, EntityManager em, Long session, DirectoryTree dt) {
if ( item != null ) {
LogEventManager lem = new LogEventManager(session, item.getParentCollection());
lem.persistItemEvent(LogEnum.REMOVE_ITEM, item.getPath(), null, em);
if ( !item.isDirectory() ) {
String parent = item.getParentPath();
Collection c = item.getParentCollection();
EntityTransaction trans = em.getTransaction();
trans.begin();
if ( item.getToken() != null ) {
em.remove(item.getToken());
}
em.remove(item);
trans.commit();
reloadTree(dt, parent, c, em);
} else {
new MyDeleteThread(item, dt).start();
private Set<Collection> removeForType(Collection collection,
String type,
EntityManager em,
long eventSession) {
TypedQuery<MonitoredItem> query = em.createNamedQuery(
"MonitoredItem.itemsByState",
MonitoredItem.class);
char state = 0;
query.setParameter("coll", collection);
// Only remove corrupt or missing items for now
switch (type.toLowerCase(Locale.ENGLISH)) {
case "corrupt":
state = 'C';
break;
case "missing":
state = 'M';
break;
default:
LOG.warn("Not remove type of " + type + "; needs to be corrupt or missing");
}
if (state != 0) {
DataSource dataSource;
Connection connection = null;
try {
dataSource = PersistUtil.getDataSource();
connection = dataSource.getConnection();
// Create log_event items
// We do this through a PreparedStatement because we can be working on many
// items at once time which can be very slow when iterating each item individually
PreparedStatement logStatement = connection.prepareStatement(
"INSERT INTO logevent(session, path, date, logtype, collection_id) " +
"SELECT ?, path, NOW(), ?, parentcollection_id FROM monitored_item m " +
"WHERE m.parentcollection_id = ? AND m.state = ?");
connection.setAutoCommit(false);
logStatement.setLong(1, eventSession);
logStatement.setInt(2, LogEnum.REMOVE_ITEM.getType());
logStatement.setLong(3, collection.getId());
logStatement.setString(4, String.valueOf(state));
// And remove with a PreparedStatement for the same reason
PreparedStatement deleteStatement = connection.prepareStatement(
"DELETE FROM monitored_item WHERE parentcollection_id = ? AND state = ?");
deleteStatement.setLong(1, collection.getId());
deleteStatement.setString(2, String.valueOf(state));
logStatement.executeUpdate();
logStatement.close();
deleteStatement.executeUpdate();
deleteStatement.close();
// I don't think we need this since we close the connection when we finish
connection.setAutoCommit(true);
} catch (SQLException e) {
LOG.error("Error removing items ", e);
rollback(connection);
} finally {
SQL.release(connection);
}
}
return ImmutableSet.of(collection);
}
private void rollback(@Nullable Connection connection) {
try {
if (connection != null) {
connection.rollback();
}
} catch (SQLException e) {
LOG.warn("Unable to rollback last remove transaction!", e);
}
}
private <T> Optional<T> referenceFor(Class<T> clazz, Long id, EntityManager em) {
try {
return Optional.of(em.getReference(clazz, id));
} catch (EntityNotFoundException ex) {
LOG.warn("EntityNotFound " + id, ex);
return Optional.empty();
}
}
private Collection removeItem(MonitoredItem item, EntityManager em, Long session, DirectoryTree dt) {
Collection c = null;
if (item != null) {
LogEventManager lem = new LogEventManager(session, item.getParentCollection());
lem.persistItemEvent(LogEnum.REMOVE_ITEM, item.getPath(), null, em);
if (!item.isDirectory()) {
String parent = item.getParentPath();
c = item.getParentCollection();
EntityTransaction trans = em.getTransaction();
trans.begin();
if (item.getToken() != null) {
em.remove(item.getToken());
}
em.remove(item);
trans.commit();
reloadTree(dt, parent, c, em);
} else {
new MyDeleteThread(item, dt).start();
}
}
return c;
}
private static void reloadTree(DirectoryTree dt,
String parent,
Collection c,
EntityManager em ) {
if ( dt == null ) {
EntityManager em) {
if (dt == null) {
return;
}
MonitoredItemManager mim = new MonitoredItemManager(em);
MonitoredItem mi = mim.getItemByPath(parent, c);
if ( mi != null ) {
if (mi != null) {
dt.toggleItem(mi.getId());
dt.toggleItem(mi.getId());
}
......@@ -163,7 +258,7 @@ public class RemoveItemServlet extends EntityManagerServlet {
private EntityManager em;
private DirectoryTree dt;
private MyDeleteThread( MonitoredItem item, DirectoryTree dt ) {
private MyDeleteThread(MonitoredItem item, DirectoryTree dt) {
super("Delete thread " + item.getId());
this.item = item;
this.dt = dt;
......@@ -188,7 +283,7 @@ public class RemoveItemServlet extends EntityManagerServlet {
clearDir(em.merge(item));
trans.commit();
reloadTree(dt, parent, c, em);
} catch ( Throwable t ) {
} catch (Throwable t) {
LOG.error("Error removing", t);
} finally {
em.close();
......@@ -200,11 +295,11 @@ public class RemoveItemServlet extends EntityManagerServlet {
}
}
private void clearDir( MonitoredItem item ) {
private void clearDir(MonitoredItem item) {
LOG.trace("Removing dir: " + item.getPath());
for ( MonitoredItem mi : mim.listChildren(item.getParentCollection(),
item.getPath()) ) {
if ( mi.isDirectory() ) {
for (MonitoredItem mi : mim.listChildren(item.getParentCollection(),
item.getPath())) {
if (mi.isDirectory()) {
clearDir(mi);
} else {
LOG.trace("Removing file: " + mi.getPath());
......
<%@page pageEncoding="UTF-8"%>
<%@page pageEncoding="UTF-8" %>
<%--
The taglib directive below imports the JSTL library. If you uncomment it,
you must also add the JSTL library to the project. The Add Library... action
on Libraries node in Projects view can be used to add the JSTL 1.1 library.
--%>
<%@taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"%>
<%@ taglib prefix="fn" uri="http://java.sun.com/jsp/jstl/functions" %>
<%@taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c" %>
<%@taglib prefix="fn" uri="http://java.sun.com/jsp/jstl/functions" %>
<%@taglib tagdir="/WEB-INF/tags" prefix="h" %>
<!DOCTYPE html
PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>Collection Errors</title>
<jsp:include page="imports.jsp"/>
<!--<script src="jquery-1.7.1.min.js" type="text/javascript"></script> -->
<style type="text/css">
body {
width: 752px !important;
margin-top: 8px !important;
padding-right: 0px !important;
}
#summaryTable {
margin-left: 50px;
margin-right: auto;
}
.lblTd {
padding-left: 50px;
}
.dataTd {
padding-left: 10px;
}
#reportTable {
width: 90%;
margin: 20px auto 10px;
border: 1px solid #000000;
border-collapse: separate;
}
#reportTable thead {
border-bottom: 1px solid #000000;
background-color: #e8e8e8;
}
#navtable {
width: 100%;
}
.datecol {
width: 150px;
}
.tblLinks {
padding-left: 50px;
}
</style>
</head>
<body>
<jsp:include page="header.jsp" />
<table id="summaryTable">
<tr>
<td class="lblTd">Active Files</td>
<td class="dataTd"><h:DefaultValue test="${collection.activeFiles > -1}" success="${collection.activeFiles}" failure="0"/></td>
<td class="lblTd">Corrupt Remote Files</td>
<td class="dataTd"><h:DefaultValue test="${collection.remoteCorrupt > -1}" success="${collection.remoteCorrupt}" failure="0"/></td>
<td class="lblTd">Files without Tokens</td>
<td class="dataTd"><h:DefaultValue test="${collection.missingTokens} > -1}" success="${collection.missingTokens}" failure="0"/></td>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
<title>Collection Errors</title>
<jsp:include page="imports.jsp"/>
</tr>
<tr>
<td class="lblTd">Missing Files</td>
<td class="dataTd"><h:DefaultValue test="${collection.missingFiles > -1}" success="${collection.missingFiles}" failure="0"/></td>
<td class="lblTd">Missing Remote Files</td>
<td class="dataTd"><h:DefaultValue test="${collection.remoteMissing > -1}" success="${collection.remoteMissing}" failure="0"/></td>
<td class="lblTd">Files with possible corrupt digests</td>
<td class="dataTd"><h:DefaultValue test="${collection.invalidDigests > -1}" success="${collection.invalidDigests}" failure="0"/></td>
</tr>
<tr>
<td class="lblTd">Corrupt Files</td>
<td class="dataTd"><h:DefaultValue test="${collection.corruptFiles > -1}" success="${collection.corruptFiles}" failure="0"/></td>
</tr>
<tr>
<td class="tblLinks"><a href="Report?collectionid=${collection.collection.id}&amp;text=1&amp;count=-1">Download List</a></td>
<td class="tblLinks"><a href="StartSync?collectionid=${collection.collection.id}&amp;type=corrupt">Audit Corrupt Files</a></td>
<td class="tblLinks"><a href="EventLog?sessionId=${session}">Recent Events</a></td>
</tr>
</table>
<style type="text/css">
body {
width: 752px !important;
margin-top: 8px !important;
padding-right: 0px !important;
}
#summaryTable {
margin-left: 50px;
margin-right: auto;
}
.lblTd {
padding-left: 50px;
}
.dataTd {
padding-left: 10px;
}
#reportTable {
width: 100%;
margin: 20px auto 10px;
border: 1px solid #000000;
border-collapse: separate;
}
#reportTable thead {
border-bottom: 1px solid #000000;
background-color: #e8e8e8;
}
#navtable {
width: 100%;
}
<span>Select All:&nbsp;</span> <input type="checkbox" id="selectall"/>
.datecol {
width: 150px;
}
<form method="GET" action="RemoveItem">
<input type="hidden" name="redirect" value="Report"/>
<input type="hidden" name="collectionid" value="${collection.collection.id}" />
.tblLinks {
padding-left: 50px;
}
</style>
</head>
<body>
<jsp:include page="header.jsp"/>
<jsp:useBean id="collection" scope="request"
type="edu.umiacs.ace.monitor.access.CollectionSummaryBean"/>
<table id="summaryTable">
<tr>
<td class="lblTd">Active Files</td>
<td class="dataTd">
<h:DefaultValue test="${collection.activeFiles > -1}"
success="${collection.activeFiles}" failure="0"/></td>
<td class="lblTd">Corrupt Remote Files</td>
<td class="dataTd"><h:DefaultValue test="${collection.remoteCorrupt > -1}"
success="${collection.remoteCorrupt}" failure="0"/></td>
<td class="lblTd">Files without Tokens</td>
<td class="dataTd"><h:DefaultValue test="${collection.missingTokens > -1}"
success="${collection.missingTokens}" failure="0"/></td>
</tr>
<tr>
<td class="lblTd">Missing Files</td>
<td class="dataTd"><h:DefaultValue test="${collection.missingFiles > -1}"
success="${collection.missingFiles}" failure="0"/>
<c:if test="${collection.missingFiles > 0}"><a>Remove All</a></c:if>
</td>
<td class="lblTd">Missing Remote Files</td>
<td class="dataTd"><h:DefaultValue test="${collection.remoteMissing > -1}"
success="${collection.remoteMissing}" failure="0"/></td>
<td class="lblTd">Files with possible corrupt digests</td>
<td class="dataTd"><h:DefaultValue test="${collection.invalidDigests > -1}"
success="${collection.invalidDigests}" failure="0"/></td>
</tr>
<tr>
<td class="lblTd">Corrupt Files</td>
<td class="dataTd"><h:DefaultValue test="${collection.corruptFiles > -1}"
success="${collection.corruptFiles}" failure="0"/>
<c:if test="${collection.corruptFiles > 0}">
<a href="RemoveItem?redirect=Report&collectionid=${collection.collection.id}&type=corrupt">Remove All</a></c:if>
</td>
</tr>
<tr>
<td class="tblLinks">
<a href="Report?collectionid=${collection.collection.id}&amp;text=1&amp;count=-1">
Download List</a></td>
<td class="tblLinks">
<a href="StartSync?collectionid=${collection.collection.id}&amp;type=corrupt">
Audit Corrupt Files</a></td>
<td class="tblLinks"><a href="EventLog?sessionId=${session}">Recent Events</a></td>
</tr>
</table>
<form method="GET" action="RemoveItem">
<div class="container" style="width: 90%">
<span>Select All:&nbsp;</span> <input type="checkbox" id="selectall"/>
<input type="hidden" name="redirect" value="Report"/>
<input type="hidden" name="collectionid" value="${collection.collection.id}"/>
<table id="reportTable">
<thead>
<tr>
<td>Remove</td>
<td>State</td>
<td>Path</td>
<td>Last Seen</td>
<td></td>
</tr>
<tr>
<td>Remove</td>
<td>State</td>
<td>Path</td>
<td>Last Seen</td>
<td></td>
</tr>
</thead>
<c:forEach var="item" items="${items}">
<tr>
<td>
<input type="checkbox" name="removal" id="removal" value="${item.id}" />
<input type="checkbox" name="removal" id="removal" value="${item.id}"/>
</td>
<td>
<c:choose>
......@@ -141,35 +165,46 @@ on Libraries node in Projects view can be used to add the JSTL 1.1 library.
</c:choose>
</td>
<td>
<c:choose>
<c:when test="${item.directory}"><span><img src="images/folder.jpg"/>${item.path}</span></c:when>
<c:otherwise><span><img src="images/file.jpg" alt=""/>${fn:replace(item.path, "/", "&#x200B;/")}</span></c:otherwise>
<c:when test="${item.directory}"><span><img
src="images/folder.jpg"/>${item.path}</span></c:when>
<c:otherwise><span><img src="images/file.jpg"
alt=""/>${fn:replace(item.path, "/", "&#x200B;/")}</span></c:otherwise>
</c:choose>
</td>
<td class="datecol"><span>${fn:replace(item.lastSeen, " ", "&nbsp;")}</span></td>
<td><a href="EventLog?logpath=${item.path}">log</a> <a href="RemoveItem?itemid=${item.id}&amp;redirect=Report%3Fcollectionid=${collection.collection.id}">remove</a> </td>
<td class="datecol"><span>${fn:replace(item.lastSeen, " ", "&nbsp;")}</span>
</td>
<td><a href="EventLog?logpath=${item.path}">log</a> <a
href="RemoveItem?itemid=${item.id}&amp;redirect=Report%3Fcollectionid=${collection.collection.id}">remove</a>
</td>
</tr>
</c:forEach>
<tr><td colspan="5">
<tr>
<td colspan="5">
<table id="navtable">
<tr>
<td>
<a href="Report?top=${items[0].id}&collectionid=${collection.collection.id}&count=${count}">&lt;&lt;</a></td>
<a href="Report?top=${items[0].id}&collectionid=${collection.collection.id}&count=${count}">&lt;&lt;</a>
</td>
<td align="center">Only items with errors are listed</td>
<td align="right"><a href="Report?start=${next}&collectionid=${collection.collection.id}&count=${count}">&gt;&gt;</a></td>
<td align="right"><a
href="Report?start=${next}&collectionid=${collection.collection.id}&count=${count}">&gt;&gt;</a>
</td>
</tr>
</table>
</td></tr>
</td>
</tr>
</table>
<!-- Button trigger modal -->
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#exampleModal" style="width: 25%">
<!-- Button trigger -->
<button type="button" class="btn btn-primary" data-toggle="modal"
data-target="#exampleModal" style="width: 25%">
Remove Selected
</button>