From f7bd5e64a013bf8caebbe5d3495199fcab0b6945 Mon Sep 17 00:00:00 2001
From: brinn <brinn>
Date: Mon, 2 May 2011 18:42:34 +0000
Subject: [PATCH] fix: on startup, check for each item of the .faulty_paths
file whether it still exists and remove it, if it doesn't exist anymore
SVN: 21094
---
.../filesystem/DirectoryScannedStore.java | 6 ++
.../DirectoryScanningHandlerInterceptor.java | 9 ++-
.../DirectoryScanningTimerTask.java | 9 ++-
.../FaultyPathDirectoryScanningHandler.java | 73 ++++++++++++++-----
.../filesystem/IDirectoryScanningHandler.java | 7 +-
.../cisd/datamover/FileScannedStore.java | 7 +-
.../datamover/filesystem/intf/IFileStore.java | 6 ++
.../filesystem/store/FileStoreLocal.java | 6 ++
.../filesystem/store/FileStoreRemote.java | 10 +++
.../store/FileStoreRemoteMounted.java | 5 ++
10 files changed, 114 insertions(+), 24 deletions(-)
diff --git a/common/source/java/ch/systemsx/cisd/common/filesystem/DirectoryScannedStore.java b/common/source/java/ch/systemsx/cisd/common/filesystem/DirectoryScannedStore.java
index 52b68764ebd..099df325dd0 100644
--- a/common/source/java/ch/systemsx/cisd/common/filesystem/DirectoryScannedStore.java
+++ b/common/source/java/ch/systemsx/cisd/common/filesystem/DirectoryScannedStore.java
@@ -20,6 +20,7 @@ import java.io.FileFilter;
import java.util.ArrayList;
import java.util.List;
+import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang.StringUtils;
import ch.systemsx.cisd.common.filesystem.DirectoryScanningTimerTask.IScannedStore;
@@ -54,6 +55,11 @@ public final class DirectoryScannedStore implements IScannedStore
return StoreItem.asFile(directory, item).getPath();
}
+ public StoreItem asStoreItem(String locationDescription)
+ {
+ return new StoreItem(FilenameUtils.getName(locationDescription));
+ }
+
public final boolean existsOrError(final StoreItem item)
{
return StoreItem.asFile(directory, item).exists();
diff --git a/common/source/java/ch/systemsx/cisd/common/filesystem/DirectoryScanningHandlerInterceptor.java b/common/source/java/ch/systemsx/cisd/common/filesystem/DirectoryScanningHandlerInterceptor.java
index 381c439c1da..19cae99460d 100644
--- a/common/source/java/ch/systemsx/cisd/common/filesystem/DirectoryScanningHandlerInterceptor.java
+++ b/common/source/java/ch/systemsx/cisd/common/filesystem/DirectoryScanningHandlerInterceptor.java
@@ -40,9 +40,14 @@ public class DirectoryScanningHandlerInterceptor implements IDirectoryScanningHa
// IDirectoryScanningHandler
//
- public void beforeHandle()
+ public void init(IScannedStore scannedStore)
{
- directoryScanningHandler.beforeHandle();
+ directoryScanningHandler.init(scannedStore);
+ }
+
+ public void beforeHandle(IScannedStore scannedStore)
+ {
+ directoryScanningHandler.beforeHandle(scannedStore);
}
public HandleInstruction mayHandle(final IScannedStore scannedStore, final StoreItem storeItem)
diff --git a/common/source/java/ch/systemsx/cisd/common/filesystem/DirectoryScanningTimerTask.java b/common/source/java/ch/systemsx/cisd/common/filesystem/DirectoryScanningTimerTask.java
index ffac6f3b64b..bc69916863b 100644
--- a/common/source/java/ch/systemsx/cisd/common/filesystem/DirectoryScanningTimerTask.java
+++ b/common/source/java/ch/systemsx/cisd/common/filesystem/DirectoryScanningTimerTask.java
@@ -230,6 +230,7 @@ public final class DirectoryScanningTimerTask extends TimerTask implements ITime
{
didSomeWork = false;
int numberOfItemsProcessedInLastRound;
+ directoryScanningHandler.init(sourceDirectory);
do
{
numberOfItemsProcessedInLastRound = 0;
@@ -258,7 +259,7 @@ public final class DirectoryScanningTimerTask extends TimerTask implements ITime
break;
}
final int numberOfItems = storeItemsOrNull.length;
- directoryScanningHandler.beforeHandle();
+ directoryScanningHandler.beforeHandle(sourceDirectory);
if (operationLog.isTraceEnabled())
{
operationLog.trace("beforeHandle() completed.");
@@ -447,6 +448,12 @@ public final class DirectoryScanningTimerTask extends TimerTask implements ITime
* assumed that the result is the path which could be used in java.io.File constructor.
*/
String getLocationDescription(StoreItem item);
+
+ /**
+ * Constructs a {@link StoreItem} from a location description as created by
+ * {@link #getLocationDescription(StoreItem)}.
+ */
+ StoreItem asStoreItem(String locationDescription);
}
}
diff --git a/common/source/java/ch/systemsx/cisd/common/filesystem/FaultyPathDirectoryScanningHandler.java b/common/source/java/ch/systemsx/cisd/common/filesystem/FaultyPathDirectoryScanningHandler.java
index 2ddd91f9abe..724fc60582c 100644
--- a/common/source/java/ch/systemsx/cisd/common/filesystem/FaultyPathDirectoryScanningHandler.java
+++ b/common/source/java/ch/systemsx/cisd/common/filesystem/FaultyPathDirectoryScanningHandler.java
@@ -18,6 +18,7 @@ package ch.systemsx.cisd.common.filesystem;
import java.io.File;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.Set;
import org.apache.log4j.Logger;
@@ -46,7 +47,7 @@ public final class FaultyPathDirectoryScanningHandler implements IDirectoryScann
private static final Logger operationLog =
LogFactory.getLogger(LogCategory.OPERATION, FaultyPathDirectoryScanningHandler.class);
- private final Set<String> faultyPaths;
+ private final Set<StoreItem> faultyPaths;
private final File faultyPathsFile;
@@ -57,12 +58,31 @@ public final class FaultyPathDirectoryScanningHandler implements IDirectoryScann
public FaultyPathDirectoryScanningHandler(final File faultyPathDirectory,
final IStopSignaler stopSignaler)
{
- this.faultyPaths = new HashSet<String>();
+ this.faultyPaths = new HashSet<StoreItem>();
this.faultyPathsFile = new File(faultyPathDirectory, Constants.FAULTY_PATH_FILENAME);
this.stopSignaler = stopSignaler;
}
+
+ private Set<String> faultyPathsAsStrings(IScannedStore scannedStore)
+ {
+ final Set<String> faultyPathStrings = new HashSet<String>(faultyPaths.size());
+ for (StoreItem item : faultyPaths)
+ {
+ faultyPathStrings.add(scannedStore.getLocationDescription(item));
+ }
+ return faultyPathStrings;
+ }
+
+ private void setFaultyPathsFromStrings(IScannedStore scannedStore, Set<String> faultyPathStrings)
+ {
+ faultyPaths.clear();
+ for (String faultyPath : faultyPathStrings)
+ {
+ faultyPaths.add(scannedStore.asStoreItem(faultyPath));
+ }
+ }
- private final void checkForFaultyPathsFileChanged()
+ private final void checkForFaultyPathsFileChanged(IScannedStore scannedStore)
{
if (faultyPathsFile.exists())
{
@@ -70,7 +90,9 @@ public final class FaultyPathDirectoryScanningHandler implements IDirectoryScann
if (faultyPathsFile.lastModified() > faultyPathsLastChanged)
{
faultyPaths.clear();
- CollectionIO.readCollection(faultyPathsFile, faultyPaths);
+ final Set<String> faultyPathStrings = new HashSet<String>();
+ CollectionIO.readCollection(faultyPathsFile, faultyPathStrings);
+ setFaultyPathsFromStrings(scannedStore, faultyPathStrings);
faultyPathsLastChanged = faultyPathsFile.lastModified();
if (operationLog.isInfoEnabled())
{
@@ -92,12 +114,6 @@ public final class FaultyPathDirectoryScanningHandler implements IDirectoryScann
return file.getPath();
}
- private final boolean isFaultyPath(final IScannedStore scannedStore, final StoreItem storeItem)
- {
- final String path = scannedStore.getLocationDescription(storeItem);
- return faultyPaths.contains(path);
- }
-
private final boolean isFaultyPathsFile(final IScannedStore scannedStore,
final StoreItem storeItem)
{
@@ -108,20 +124,20 @@ public final class FaultyPathDirectoryScanningHandler implements IDirectoryScann
private final void addToFaultyPaths(final IScannedStore scannedStore, final StoreItem item)
{
- final String path = scannedStore.getLocationDescription(item);
if (operationLog.isDebugEnabled())
{
+ final String path = scannedStore.getLocationDescription(item);
operationLog.debug(String.format(
- "Following path '%s' has been added to faulty paths file '%s'", path,
+ "Path '%s' has been added to faulty paths file '%s'", path,
faultyPathsFile.getAbsolutePath()));
}
- faultyPaths.add(path);
- refreshFaultyPathsFile();
+ faultyPaths.add(item);
+ refreshFaultyPathsFile(scannedStore);
}
- private final void refreshFaultyPathsFile()
+ private final void refreshFaultyPathsFile(IScannedStore scannedStore)
{
- CollectionIO.writeIterable(faultyPathsFile, faultyPaths);
+ CollectionIO.writeIterable(faultyPathsFile, faultyPathsAsStrings(scannedStore));
faultyPathsLastChanged = faultyPathsFile.lastModified();
}
@@ -129,9 +145,28 @@ public final class FaultyPathDirectoryScanningHandler implements IDirectoryScann
// IDirectoryScanningHandler
//
- public final void beforeHandle()
+ public void init(IScannedStore scannedStore)
+ {
+ final Iterator<StoreItem> it = faultyPaths.iterator();
+ boolean changedFaultyPaths = false;
+ while (it.hasNext())
+ {
+ final StoreItem item = it.next();
+ if (scannedStore.existsOrError(item) == false)
+ {
+ it.remove();
+ changedFaultyPaths = true;
+ }
+ }
+ if (changedFaultyPaths)
+ {
+ refreshFaultyPathsFile(scannedStore);
+ }
+ }
+
+ public final void beforeHandle(IScannedStore scannedStore)
{
- checkForFaultyPathsFileChanged();
+ checkForFaultyPathsFileChanged(scannedStore);
}
public final HandleInstruction mayHandle(final IScannedStore scannedStore,
@@ -140,7 +175,7 @@ public final class FaultyPathDirectoryScanningHandler implements IDirectoryScann
if (isFaultyPathsFile(scannedStore, storeItem))
{
return HandleInstruction.IGNORE;
- } else if (isFaultyPath(scannedStore, storeItem))
+ } else if (faultyPaths.contains(storeItem))
{
return HandleInstruction.createError("Known bad item '%s'.", storeItem);
} else
diff --git a/common/source/java/ch/systemsx/cisd/common/filesystem/IDirectoryScanningHandler.java b/common/source/java/ch/systemsx/cisd/common/filesystem/IDirectoryScanningHandler.java
index c9ee9e90009..3f8c4f264d3 100644
--- a/common/source/java/ch/systemsx/cisd/common/filesystem/IDirectoryScanningHandler.java
+++ b/common/source/java/ch/systemsx/cisd/common/filesystem/IDirectoryScanningHandler.java
@@ -94,10 +94,15 @@ public interface IDirectoryScanningHandler
}
}
+ /**
+ * Run once before the handler is used.
+ */
+ public void init(IScannedStore scannedStore);
+
/**
* Is performed just before handling all the items contained in the store.
*/
- public void beforeHandle();
+ public void beforeHandle(IScannedStore scannedStore);
/**
* Whether given <code>storeItem</code> found in given <var>scannedStore</var> should be
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/FileScannedStore.java b/datamover/source/java/ch/systemsx/cisd/datamover/FileScannedStore.java
index 7088a2886a6..795e8002c41 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/FileScannedStore.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/FileScannedStore.java
@@ -22,9 +22,9 @@ import org.apache.commons.lang.StringUtils;
import ch.systemsx.cisd.common.Constants;
import ch.systemsx.cisd.common.filesystem.BooleanStatus;
+import ch.systemsx.cisd.common.filesystem.DirectoryScanningTimerTask.IScannedStore;
import ch.systemsx.cisd.common.filesystem.IStoreItemFilter;
import ch.systemsx.cisd.common.filesystem.StoreItem;
-import ch.systemsx.cisd.common.filesystem.DirectoryScanningTimerTask.IScannedStore;
import ch.systemsx.cisd.common.logging.ISimpleLogger;
import ch.systemsx.cisd.common.logging.LogLevel;
import ch.systemsx.cisd.datamover.filesystem.intf.IFileStore;
@@ -72,6 +72,11 @@ final class FileScannedStore implements IScannedStore
return fileStore.getLocationDescription(item);
}
+ public StoreItem asStoreItem(String locationDescription)
+ {
+ return fileStore.asStoreItem(locationDescription);
+ }
+
public StoreItem[] tryListSorted(ISimpleLogger loggerOrNull)
{
// Older items will be handled before newer items. This becomes important when doing online
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IFileStore.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IFileStore.java
index 96929a35727..89d0e45fedf 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IFileStore.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IFileStore.java
@@ -102,6 +102,12 @@ public interface IFileStore extends ISelfTestable, ILastModificationChecker
*/
public String getLocationDescription(StoreItem item);
+ /**
+ * Constructs a {@link StoreItem} from a location description as created by
+ * {@link #getLocationDescription(StoreItem)}.
+ */
+ StoreItem asStoreItem(String locationDescription);
+
/**
* Returns this file store as an extended file store if possible.
*
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreLocal.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreLocal.java
index b7e9fb40e67..4e457617ae7 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreLocal.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreLocal.java
@@ -20,6 +20,7 @@ import java.io.File;
import java.io.IOException;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang.time.DateUtils;
import org.apache.log4j.Logger;
@@ -185,6 +186,11 @@ public class FileStoreLocal extends AbstractFileStore implements IExtendedFileSt
return getChildFile(item).getPath();
}
+ public StoreItem asStoreItem(String locationDescription)
+ {
+ return new StoreItem(FilenameUtils.getName(locationDescription));
+ }
+
public final StoreItem[] tryListSortByLastModified(final ISimpleLogger loggerOrNull)
{
final File[] files = FileUtilities.tryListFiles(getPath(), loggerOrNull);
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreRemote.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreRemote.java
index 1370893d349..4df1e918012 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreRemote.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreRemote.java
@@ -19,6 +19,7 @@ package ch.systemsx.cisd.datamover.filesystem.store;
import java.io.File;
import java.util.List;
+import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
@@ -476,6 +477,15 @@ public class FileStoreRemote extends AbstractFileStore
return getHost() + ":" + getChildFile(item).getPath();
}
+ public StoreItem asStoreItem(String locationDescription)
+ {
+ final int beginIndex = locationDescription.indexOf(':');
+ final String path =
+ (beginIndex < 0) ? locationDescription : locationDescription
+ .substring(beginIndex + 1);
+ return new StoreItem(FilenameUtils.getName(path));
+ }
+
private String getHost()
{
final String host = tryGetHost();
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreRemoteMounted.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreRemoteMounted.java
index bb859d27e49..68137dca3bb 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreRemoteMounted.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreRemoteMounted.java
@@ -84,6 +84,11 @@ public final class FileStoreRemoteMounted extends AbstractFileStore
return localImpl.getLocationDescription(item);
}
+ public StoreItem asStoreItem(String locationDescription)
+ {
+ return localImpl.asStoreItem(locationDescription);
+ }
+
public final Status delete(final StoreItem item)
{
try
--
GitLab