From 030154a811fae0c855e206e4522f6ecd61214aa0 Mon Sep 17 00:00:00 2001
From: brinn <brinn>
Date: Tue, 15 Jul 2008 07:04:10 +0000
Subject: [PATCH] add: error status reporting [DMV-31] fix: ensure the error
marker files are removed when the error condition is finished
SVN: 7234
---
.../ch/systemsx/cisd/common/Constants.java | 8 +-
...HighwaterMarkDirectoryScanningHandler.java | 3 +-
.../utilities/DirectoryScannedStore.java | 20 +++-
.../DirectoryScanningHandlerInterceptor.java | 3 +-
.../utilities/DirectoryScanningTimerTask.java | 99 +++++++++++++++----
.../FaultyPathDirectoryScanningHandler.java | 17 ++--
.../utilities/IDirectoryScanningHandler.java | 67 ++++++++++++-
.../utilities/ITimerTaskStatusProvider.java | 22 ++++-
...waterMarkDirectoryScanningHandlerTest.java | 33 +++++--
.../DirectoryScanningTimerTaskTest.java | 3 +-
.../cisd/datamover/FileScannedStore.java | 30 +++---
.../cisd/datamover/IncomingProcessor.java | 5 +-
...imerTaskListenerForMarkerFileProtocol.java | 36 +++++--
.../cisd/datamover/common/MarkerFile.java | 8 ++
14 files changed, 282 insertions(+), 72 deletions(-)
diff --git a/common/source/java/ch/systemsx/cisd/common/Constants.java b/common/source/java/ch/systemsx/cisd/common/Constants.java
index a3df6c694a0..31b7c2cdec3 100644
--- a/common/source/java/ch/systemsx/cisd/common/Constants.java
+++ b/common/source/java/ch/systemsx/cisd/common/Constants.java
@@ -47,9 +47,13 @@ public final class Constants
/** The number of milliseconds to sleep before retrying (<i>3s</i>). */
public static final long MILLIS_TO_SLEEP_BEFORE_RETRYING = 3 * DateUtils.MILLIS_PER_SECOND;
- /** The number of retries when a given process failed. */
- public static final int MAXIMUM_RETRY_COUNT = 12;
+ /** The maximal number of invocations when a given process failed. */
+ public static final int MAXIMUM_INVOCATIONS_ON_FAILURE = 12;
/** The prefix of marker files that indicate that a directory is currently being processed. */
public static final String PROCESSING_PREFIX = MARKER_PREFIX + "processing_";
+
+ /** The file name of the file that contains file names which are known to be bad. */
+ public static final String FAULTY_PATH_FILENAME = ".faulty_paths";
+
}
diff --git a/common/source/java/ch/systemsx/cisd/common/highwatermark/HighwaterMarkDirectoryScanningHandler.java b/common/source/java/ch/systemsx/cisd/common/highwatermark/HighwaterMarkDirectoryScanningHandler.java
index f252714d3f4..99dc94461ba 100644
--- a/common/source/java/ch/systemsx/cisd/common/highwatermark/HighwaterMarkDirectoryScanningHandler.java
+++ b/common/source/java/ch/systemsx/cisd/common/highwatermark/HighwaterMarkDirectoryScanningHandler.java
@@ -88,7 +88,8 @@ public final class HighwaterMarkDirectoryScanningHandler extends
@Override
public HandleInstruction mayHandle(final IScannedStore scannedStore, final StoreItem storeItem)
{
- return mayHandle() == false ? HandleInstruction.ERROR : super.mayHandle(
+ return (mayHandle() == false) ? HandleInstruction.createError(
+ "Not enough disk space on store '%s'.", scannedStore) : super.mayHandle(
scannedStore, storeItem);
}
diff --git a/common/source/java/ch/systemsx/cisd/common/utilities/DirectoryScannedStore.java b/common/source/java/ch/systemsx/cisd/common/utilities/DirectoryScannedStore.java
index 349e05c7598..e87c0ebfad9 100644
--- a/common/source/java/ch/systemsx/cisd/common/utilities/DirectoryScannedStore.java
+++ b/common/source/java/ch/systemsx/cisd/common/utilities/DirectoryScannedStore.java
@@ -17,6 +17,8 @@ package ch.systemsx.cisd.common.utilities;
import java.io.File;
import java.io.FileFilter;
+import java.util.ArrayList;
+import java.util.List;
import ch.systemsx.cisd.common.logging.ISimpleLogger;
import ch.systemsx.cisd.common.utilities.DirectoryScanningTimerTask.IScannedStore;
@@ -54,9 +56,9 @@ public final class DirectoryScannedStore implements IScannedStore
return StoreItem.asFile(directory, item).exists();
}
- public final StoreItem[] tryListSortedReadyToProcess(final ISimpleLogger loggerOrNull)
+ public StoreItem[] tryListSorted(ISimpleLogger loggerOrNull)
{
- final File[] files = FileUtilities.tryListFiles(directory, filter, loggerOrNull);
+ final File[] files = FileUtilities.tryListFiles(directory, null, loggerOrNull);
if (files != null)
{
FileUtilities.sortByLastModified(files);
@@ -67,6 +69,19 @@ public final class DirectoryScannedStore implements IScannedStore
}
}
+ public StoreItem[] filterReadyToProcess(StoreItem[] items)
+ {
+ final List<StoreItem> result = new ArrayList<StoreItem>(items.length);
+ for (StoreItem item : items)
+ {
+ if (filter.accept(new File(directory, item.getName())))
+ {
+ result.add(item);
+ }
+ }
+ return result.toArray(new StoreItem[result.size()]);
+ }
+
//
// Object
//
@@ -76,4 +91,5 @@ public final class DirectoryScannedStore implements IScannedStore
{
return directory.toString();
}
+
}
\ No newline at end of file
diff --git a/common/source/java/ch/systemsx/cisd/common/utilities/DirectoryScanningHandlerInterceptor.java b/common/source/java/ch/systemsx/cisd/common/utilities/DirectoryScanningHandlerInterceptor.java
index 55205035ed2..0d93b38aa14 100644
--- a/common/source/java/ch/systemsx/cisd/common/utilities/DirectoryScanningHandlerInterceptor.java
+++ b/common/source/java/ch/systemsx/cisd/common/utilities/DirectoryScanningHandlerInterceptor.java
@@ -16,6 +16,7 @@
package ch.systemsx.cisd.common.utilities;
+import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.utilities.DirectoryScanningTimerTask.IScannedStore;
/**
@@ -49,7 +50,7 @@ public class DirectoryScanningHandlerInterceptor implements IDirectoryScanningHa
return directoryScanningHandler.mayHandle(scannedStore, storeItem);
}
- public boolean finishItemHandle(final IScannedStore scannedStore, final StoreItem storeItem)
+ public Status finishItemHandle(final IScannedStore scannedStore, final StoreItem storeItem)
{
return directoryScanningHandler.finishItemHandle(scannedStore, storeItem);
}
diff --git a/common/source/java/ch/systemsx/cisd/common/utilities/DirectoryScanningTimerTask.java b/common/source/java/ch/systemsx/cisd/common/utilities/DirectoryScanningTimerTask.java
index df5e3cf1574..4c1fe7f0af4 100644
--- a/common/source/java/ch/systemsx/cisd/common/utilities/DirectoryScanningTimerTask.java
+++ b/common/source/java/ch/systemsx/cisd/common/utilities/DirectoryScanningTimerTask.java
@@ -18,18 +18,26 @@ package ch.systemsx.cisd.common.utilities;
import java.io.File;
import java.io.FileFilter;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
import java.util.TimerTask;
import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import ch.systemsx.cisd.common.collections.CollectionUtils;
+import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.logging.ConditionalNotificationLogger;
import ch.systemsx.cisd.common.logging.ISimpleLogger;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.utilities.IDirectoryScanningHandler.HandleInstruction;
+import ch.systemsx.cisd.common.utilities.IDirectoryScanningHandler.HandleInstructionFlag;
/**
* A {@link TimerTask} that scans a source directory for entries that are accepted by some
@@ -60,9 +68,11 @@ public final class DirectoryScanningTimerTask extends TimerTask implements ITime
private final ConditionalNotificationLogger notificationLogger;
- private int numberOfProcessedItems;
+ private final Map<StoreItem, String> errorLog;
+
+ private boolean didSomeWork;
- private int numberOfErrorItems;
+ private String threadNameOrNull;
/**
* Indicates that we should try to exit the {@link #run()} method as soon as possible.
@@ -139,6 +149,7 @@ public final class DirectoryScanningTimerTask extends TimerTask implements ITime
this.notificationLogger =
new ConditionalNotificationLogger(operationLog, Level.WARN, notificationLog,
ignoredErrorCount);
+ this.errorLog = new LinkedHashMap<StoreItem, String>();
}
/**
@@ -172,7 +183,7 @@ public final class DirectoryScanningTimerTask extends TimerTask implements ITime
private final StoreItem[] listStoreItems()
{
final StoreItem[] storeItems =
- sourceDirectory.tryListSortedReadyToProcess(notificationLogger);
+ sourceDirectory.tryListSorted(notificationLogger);
if (storeItems != null)
{
notificationLogger.reset(String.format("Directory '%s' is available again.",
@@ -203,15 +214,17 @@ public final class DirectoryScanningTimerTask extends TimerTask implements ITime
{
operationLog.trace(String.format("Start scanning directory '%s'.", sourceDirectory));
}
+ threadNameOrNull = Thread.currentThread().getName();
try
{
- numberOfProcessedItems = 0;
- numberOfErrorItems = 0;
+ didSomeWork = false;
int numberOfItemsProcessedInLastRound;
do
{
numberOfItemsProcessedInLastRound = 0;
- final StoreItem[] storeItems = listStoreItems();
+ final StoreItem[] allStoreItems = listStoreItems();
+ cleanseErrorLog(allStoreItems);
+ final StoreItem[] storeItems = sourceDirectory.filterReadyToProcess(allStoreItems);
final int numberOfItems = storeItems.length;
directoryScanningHandler.beforeHandle();
for (int i = 0; i < numberOfItems; i++)
@@ -231,7 +244,7 @@ public final class DirectoryScanningTimerTask extends TimerTask implements ITime
}
final HandleInstruction instruction =
directoryScanningHandler.mayHandle(sourceDirectory, storeItem);
- if (HandleInstruction.PROCESS.equals(instruction))
+ if (HandleInstructionFlag.PROCESS.equals(instruction.getFlag()))
{
try
{
@@ -239,31 +252,38 @@ public final class DirectoryScanningTimerTask extends TimerTask implements ITime
if (operationLog.isTraceEnabled())
{
operationLog.trace(String.format(
- "Following store item '%s' has been handled.", storeItem));
+ "Store item '%s' has been handled.", storeItem));
}
- ++numberOfProcessedItems;
+ didSomeWork = true;
++numberOfItemsProcessedInLastRound;
} catch (final Exception ex)
{
// Do not stop when processing of one file has failed,
// continue with other files.
- ++numberOfErrorItems;
+ errorLog.put(storeItem, String.format(
+ "Exception when processing item '%s': %s (%s)", storeItem, ex
+ .getClass().getSimpleName(), StringUtils.defaultString(
+ ex.getMessage(), "no message")));
printNotification(ex);
} finally
{
- final boolean ok =
+ final Status status =
directoryScanningHandler.finishItemHandle(sourceDirectory,
storeItem);
- if (ok == false)
+ if (status.isError())
{
- ++numberOfErrorItems;
+ final String msgOrNull = status.tryGetErrorMessage();
+ errorLog.put(storeItem, StringUtils.defaultIfEmpty(msgOrNull,
+ getDefaultErrorMessage(storeItem)));
}
}
} else
{
- if (HandleInstruction.ERROR.equals(instruction))
+ if (HandleInstructionFlag.ERROR.equals(instruction.getFlag()))
{
- ++numberOfErrorItems;
+ final String msgOrNull = instruction.tryGetMessage();
+ errorLog.put(storeItem, StringUtils.defaultIfEmpty(msgOrNull,
+ getDefaultErrorMessage(storeItem)));
}
if (operationLog.isTraceEnabled())
{
@@ -284,18 +304,47 @@ public final class DirectoryScanningTimerTask extends TimerTask implements ITime
}
}
+ private void cleanseErrorLog(StoreItem[] allStoreItems)
+ {
+ final Set<StoreItem> itemSet = new HashSet<StoreItem>(Arrays.asList(allStoreItems));
+ for (StoreItem errorItem : errorLog.keySet())
+ {
+ if (itemSet.contains(errorItem) == false)
+ {
+ errorLog.remove(errorItem);
+ }
+ }
+ }
+
+ private String getDefaultErrorMessage(final StoreItem storeItem)
+ {
+ return String.format("Error processing item '%s'.", storeItem);
+ }
+
//
// ITimerTaskStatusProvider
//
public boolean hasErrors()
{
- return (numberOfErrorItems > 0);
+ return errorLog.size() > 0;
}
public boolean hasPerformedMeaningfulWork()
{
- return (numberOfProcessedItems > 0);
+ return didSomeWork;
+ }
+
+ public String tryGetErrorLog()
+ {
+ if (hasErrors())
+ {
+ return String.format(" [%s]\n %s", StringUtils.defaultIfEmpty(threadNameOrNull,
+ "UNKNOWN"), StringUtils.join(errorLog.values(), "\n "));
+ } else
+ {
+ return null;
+ }
}
//
@@ -305,12 +354,24 @@ public final class DirectoryScanningTimerTask extends TimerTask implements ITime
public static interface IScannedStore
{
/**
- * List items in the scanned store in order in which they should be handled.
+ * List <i>all</i> items (not just the ones who are ready to be processed) in the scanned
+ * store in order in which they should be handled.
*
* @return <code>null</code> if it was no able to access the items of this scanned store.
*/
- StoreItem[] tryListSortedReadyToProcess(ISimpleLogger loggerOrNull);
+ StoreItem[] tryListSorted(ISimpleLogger loggerOrNull);
+
+ /**
+ * Performs a filtering on the items.
+ *
+ * @returns Only those <var>items</var> which are ready to be processed right now.
+ */
+ StoreItem[] filterReadyToProcess(final StoreItem[] items);
+ /**
+ * Returns <code>true</code>, if the <var>item</var> either still exists or is in an
+ * error state.
+ */
boolean existsOrError(StoreItem item);
/**
diff --git a/common/source/java/ch/systemsx/cisd/common/utilities/FaultyPathDirectoryScanningHandler.java b/common/source/java/ch/systemsx/cisd/common/utilities/FaultyPathDirectoryScanningHandler.java
index d4c62e3586d..b7e6c5d78b0 100644
--- a/common/source/java/ch/systemsx/cisd/common/utilities/FaultyPathDirectoryScanningHandler.java
+++ b/common/source/java/ch/systemsx/cisd/common/utilities/FaultyPathDirectoryScanningHandler.java
@@ -22,8 +22,10 @@ import java.util.Set;
import org.apache.log4j.Logger;
+import ch.systemsx.cisd.common.Constants;
import ch.systemsx.cisd.common.collections.CollectionIO;
import ch.systemsx.cisd.common.collections.CollectionUtils;
+import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.utilities.DirectoryScanningTimerTask.IScannedStore;
@@ -49,12 +51,10 @@ public final class FaultyPathDirectoryScanningHandler implements IDirectoryScann
private long faultyPathsLastChanged;
- static final String FAULTY_PATH_FILENAME = ".faulty_paths";
-
public FaultyPathDirectoryScanningHandler(final File faultyPathDirectory)
{
this.faultyPaths = new HashSet<String>();
- this.faultyPathsFile = new File(faultyPathDirectory, FAULTY_PATH_FILENAME);
+ this.faultyPathsFile = new File(faultyPathDirectory, Constants.FAULTY_PATH_FILENAME);
}
private final void checkForFaultyPathsFileChanged()
@@ -129,31 +129,32 @@ public final class FaultyPathDirectoryScanningHandler implements IDirectoryScann
checkForFaultyPathsFileChanged();
}
- public final HandleInstruction mayHandle(final IScannedStore scannedStore, final StoreItem storeItem)
+ public final HandleInstruction mayHandle(final IScannedStore scannedStore,
+ final StoreItem storeItem)
{
if (isFaultyPathsFile(scannedStore, storeItem))
{
return HandleInstruction.IGNORE;
} else if (isFaultyPath(scannedStore, storeItem))
{
- return HandleInstruction.ERROR;
+ return HandleInstruction.createError("Known bad item '%s'.", storeItem);
} else
{
return HandleInstruction.PROCESS;
}
}
- public final boolean finishItemHandle(final IScannedStore scannedStore, final StoreItem storeItem)
+ public final Status finishItemHandle(final IScannedStore scannedStore, final StoreItem storeItem)
{
// If the item still exists, we assume that it has not been handled. So it
// should be added to the faulty paths.
if (scannedStore.existsOrError(storeItem))
{
addToFaultyPaths(scannedStore, storeItem);
- return false;
+ return Status.createError("Failed to move item '%s'.", storeItem);
} else
{
- return true;
+ return Status.OK;
}
}
diff --git a/common/source/java/ch/systemsx/cisd/common/utilities/IDirectoryScanningHandler.java b/common/source/java/ch/systemsx/cisd/common/utilities/IDirectoryScanningHandler.java
index 3c794f1a46a..54c1001499e 100644
--- a/common/source/java/ch/systemsx/cisd/common/utilities/IDirectoryScanningHandler.java
+++ b/common/source/java/ch/systemsx/cisd/common/utilities/IDirectoryScanningHandler.java
@@ -16,6 +16,7 @@
package ch.systemsx.cisd.common.utilities;
+import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.utilities.DirectoryScanningTimerTask.IScannedStore;
/**
@@ -29,13 +30,70 @@ public interface IDirectoryScanningHandler
{
/**
- * The instruction of whether to process an item or not.
+ * The instruction flag of whether to process an item or not.
*/
- public enum HandleInstruction
+ public enum HandleInstructionFlag
{
PROCESS, IGNORE, ERROR
}
+ /**
+ * The instruction of whether to process an item or not, possibly including a message on why not
+ * to process it.
+ *
+ * @author Bernd Rinn
+ */
+ public static class HandleInstruction
+ {
+ public static HandleInstruction PROCESS =
+ new HandleInstruction(HandleInstructionFlag.PROCESS, null);
+
+ public static HandleInstruction IGNORE =
+ new HandleInstruction(HandleInstructionFlag.IGNORE, null);
+
+ private final HandleInstructionFlag flag;
+
+ private final String messageOrNull;
+
+ public static HandleInstruction createError()
+ {
+ return new HandleInstruction(HandleInstructionFlag.ERROR, null);
+ }
+
+ public static HandleInstruction createError(String message)
+ {
+ assert message != null;
+
+ return new HandleInstruction(HandleInstructionFlag.ERROR, message);
+ }
+
+ public static HandleInstruction createError(String messageTemplate, Object... args)
+ {
+ assert messageTemplate != null;
+
+ return new HandleInstruction(HandleInstructionFlag.ERROR, String.format(
+ messageTemplate, args));
+ }
+
+ private HandleInstruction(HandleInstructionFlag flag, String messageOrNull)
+ {
+ assert flag != null;
+
+ this.flag = flag;
+ this.messageOrNull = messageOrNull;
+ }
+
+ public final HandleInstructionFlag getFlag()
+ {
+ return flag;
+ }
+
+ public final String tryGetMessage()
+ {
+ return messageOrNull;
+ }
+ }
+
/**
* Is performed just before handling all the items contained in the store.
*/
@@ -50,8 +108,7 @@ public interface IDirectoryScanningHandler
/**
* Finishes and closes the handling of given <var>storeItem</var>.
*
- * @returns <code>true</code>, if the item has been handled correctly and <code>false</code>
- * if an error occurred.
+ * @returns A status of handling the <var>storeItem</var>.
*/
- public boolean finishItemHandle(IScannedStore scannedStore, StoreItem storeItem);
+ public Status finishItemHandle(IScannedStore scannedStore, StoreItem storeItem);
}
diff --git a/common/source/java/ch/systemsx/cisd/common/utilities/ITimerTaskStatusProvider.java b/common/source/java/ch/systemsx/cisd/common/utilities/ITimerTaskStatusProvider.java
index 380baa0a096..768b9c4b026 100644
--- a/common/source/java/ch/systemsx/cisd/common/utilities/ITimerTaskStatusProvider.java
+++ b/common/source/java/ch/systemsx/cisd/common/utilities/ITimerTaskStatusProvider.java
@@ -17,15 +17,29 @@
package ch.systemsx.cisd.common.utilities;
/**
- * A provider for status information about the last run of a {@link java.util.TimerTask}.
- *
+ * A provider for status information about the last run of a {@link java.util.TimerTask}.
+ *
* @author Bernd Rinn
*/
public interface ITimerTaskStatusProvider
{
-
+
+ /**
+ * Returns <code>true</code> if the last execution of the timer task found some useful work to
+ * do.
+ */
public boolean hasPerformedMeaningfulWork();
-
+
+ /**
+ * Returns <code>true</code> if during the last execution of the timer task errors have
+ * occurred.
+ */
public boolean hasErrors();
+ /**
+ * Returns the error log of the last execution, or <code>null</code>, if no errors have
+ * occurred.
+ */
+ public String tryGetErrorLog();
+
}
diff --git a/common/sourceTest/java/ch/systemsx/cisd/common/highwatermark/HighwaterMarkDirectoryScanningHandlerTest.java b/common/sourceTest/java/ch/systemsx/cisd/common/highwatermark/HighwaterMarkDirectoryScanningHandlerTest.java
index be0762458c5..340ccdcb83c 100644
--- a/common/sourceTest/java/ch/systemsx/cisd/common/highwatermark/HighwaterMarkDirectoryScanningHandlerTest.java
+++ b/common/sourceTest/java/ch/systemsx/cisd/common/highwatermark/HighwaterMarkDirectoryScanningHandlerTest.java
@@ -18,6 +18,7 @@ package ch.systemsx.cisd.common.highwatermark;
import static org.testng.AssertJUnit.assertEquals;
import static org.testng.AssertJUnit.assertFalse;
+import static org.testng.AssertJUnit.assertNull;
import java.io.File;
import java.io.IOException;
@@ -34,6 +35,7 @@ import ch.systemsx.cisd.common.utilities.IDirectoryScanningHandler;
import ch.systemsx.cisd.common.utilities.StoreItem;
import ch.systemsx.cisd.common.utilities.DirectoryScanningTimerTask.IScannedStore;
import ch.systemsx.cisd.common.utilities.IDirectoryScanningHandler.HandleInstruction;
+import ch.systemsx.cisd.common.utilities.IDirectoryScanningHandler.HandleInstructionFlag;
/**
* Test cases for the {@link HighwaterMarkDirectoryScanningHandler}.
@@ -62,6 +64,7 @@ public final class HighwaterMarkDirectoryScanningHandlerTest
context = new Mockery();
directoryScanningHandler = context.mock(IDirectoryScanningHandler.class);
scannedStore = context.mock(IScannedStore.class);
+
freeSpaceProvider = context.mock(IFreeSpaceProvider.class);
highwaterMarkWatcher = new HighwaterMarkWatcher(HIGHWATER_MARK, freeSpaceProvider);
}
@@ -135,9 +138,18 @@ public final class HighwaterMarkDirectoryScanningHandlerTest
}
}
});
- boolean mayHandle =
- HandleInstruction.PROCESS.equals(scanningHandler.mayHandle(scannedStore, storeItem));
- assertEquals(freeSpace > HIGHWATER_MARK, mayHandle);
+ final HandleInstruction instruction = scanningHandler.mayHandle(scannedStore, storeItem);
+ boolean mayHandleExpected = (freeSpace > HIGHWATER_MARK);
+ boolean mayHandleObserved = HandleInstructionFlag.PROCESS.equals(instruction.getFlag());
+ assertEquals(mayHandleExpected, mayHandleObserved);
+ if (mayHandleExpected)
+ {
+ assertNull(instruction.tryGetMessage());
+ } else
+ {
+ assertEquals("Not enough disk space on store 'iScannedStore'.", instruction
+ .tryGetMessage());
+ }
context.assertIsSatisfied();
}
@@ -165,9 +177,18 @@ public final class HighwaterMarkDirectoryScanningHandlerTest
}
}
});
- boolean mayHandle =
- HandleInstruction.PROCESS.equals(scanningHandler.mayHandle(scannedStore, storeItem));
- assertEquals(freeSpace > HIGHWATER_MARK, mayHandle);
+ final HandleInstruction instruction = scanningHandler.mayHandle(scannedStore, storeItem);
+ boolean mayHandleExpected = (freeSpace > HIGHWATER_MARK);
+ boolean mayHandleObserved = HandleInstructionFlag.PROCESS.equals(instruction.getFlag());
+ assertEquals(mayHandleExpected, mayHandleObserved);
+ if (mayHandleExpected)
+ {
+ assertNull(instruction.tryGetMessage());
+ } else
+ {
+ assertEquals("Not enough disk space on store 'iScannedStore'.", instruction
+ .tryGetMessage());
+ }
context.assertIsSatisfied();
}
}
\ No newline at end of file
diff --git a/common/sourceTest/java/ch/systemsx/cisd/common/utilities/DirectoryScanningTimerTaskTest.java b/common/sourceTest/java/ch/systemsx/cisd/common/utilities/DirectoryScanningTimerTaskTest.java
index e5ffbfe3a98..60b8dabecbb 100644
--- a/common/sourceTest/java/ch/systemsx/cisd/common/utilities/DirectoryScanningTimerTaskTest.java
+++ b/common/sourceTest/java/ch/systemsx/cisd/common/utilities/DirectoryScanningTimerTaskTest.java
@@ -30,6 +30,7 @@ import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
+import ch.systemsx.cisd.common.Constants;
import ch.systemsx.cisd.common.collections.CollectionIO;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogInitializer;
@@ -113,7 +114,7 @@ public class DirectoryScanningTimerTaskTest
private final static File getFaultyPathFile()
{
- return new File(workingDirectory, FaultyPathDirectoryScanningHandler.FAULTY_PATH_FILENAME);
+ return new File(workingDirectory, Constants.FAULTY_PATH_FILENAME);
}
private final static void createNewFile(final File someFile) throws IOException
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/FileScannedStore.java b/datamover/source/java/ch/systemsx/cisd/datamover/FileScannedStore.java
index 9dc1a4602f5..cace1e6369d 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/FileScannedStore.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/FileScannedStore.java
@@ -44,19 +44,6 @@ final class FileScannedStore implements IScannedStore
this.storeItemFilter = storeItemFilter;
}
- private final StoreItem[] filterReadyToProcess(final StoreItem[] items)
- {
- final Vector<StoreItem> result = new Vector<StoreItem>();
- for (final StoreItem item : items)
- {
- if (isReadyToProcess(item))
- {
- result.add(item);
- }
- }
- return result.toArray(StoreItem.EMPTY_ARRAY);
- }
-
private final boolean isReadyToProcess(final StoreItem item)
{
if (item.getName().startsWith(Constants.DELETION_IN_PROGRESS_PREFIX))
@@ -81,7 +68,7 @@ final class FileScannedStore implements IScannedStore
return fileStore.getLocationDescription(item);
}
- public final StoreItem[] tryListSortedReadyToProcess(final ISimpleLogger loggerOrNull)
+ public StoreItem[] tryListSorted(ISimpleLogger loggerOrNull)
{
// Older items will be handled before newer items. This becomes important when doing online
// quality control of measurements.
@@ -90,7 +77,20 @@ final class FileScannedStore implements IScannedStore
{
return null;
}
- return filterReadyToProcess(items);
+ return items;
+ }
+
+ public final StoreItem[] filterReadyToProcess(final StoreItem[] items)
+ {
+ final Vector<StoreItem> result = new Vector<StoreItem>();
+ for (final StoreItem item : items)
+ {
+ if (isReadyToProcess(item))
+ {
+ result.add(item);
+ }
+ }
+ return result.toArray(StoreItem.EMPTY_ARRAY);
}
//
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/IncomingProcessor.java b/datamover/source/java/ch/systemsx/cisd/datamover/IncomingProcessor.java
index 28519602540..b572a34de52 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/IncomingProcessor.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/IncomingProcessor.java
@@ -84,7 +84,7 @@ public class IncomingProcessor implements IRecoverableTimerTaskFactory
private final IStoreItemFilter storeItemFilter;
private final String markerFileName;
-
+
private final String errorMarkerFileName;
private final String successorMarkerFileName;
@@ -284,7 +284,8 @@ public class IncomingProcessor implements IRecoverableTimerTaskFactory
for (final File file : files)
{
- if (MarkerFile.isDeletionInProgressMarker(file))
+ if (MarkerFile.isDeletionInProgressMarker(file)
+ || MarkerFile.isFaultyPathsFile(file))
{
continue;
}
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/TimerTaskListenerForMarkerFileProtocol.java b/datamover/source/java/ch/systemsx/cisd/datamover/TimerTaskListenerForMarkerFileProtocol.java
index 5f1e838f7e4..f0f35891045 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/TimerTaskListenerForMarkerFileProtocol.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/TimerTaskListenerForMarkerFileProtocol.java
@@ -20,6 +20,7 @@ import java.io.File;
import java.io.IOException;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
import ch.systemsx.cisd.common.concurrent.DummyTimerTaskListener;
import ch.systemsx.cisd.common.concurrent.ITimerTaskListener;
@@ -89,7 +90,7 @@ public class TimerTaskListenerForMarkerFileProtocol extends DummyTimerTaskListen
@Override
public void startRunning()
{
- touch(markerFile);
+ touch(markerFile, StringUtils.EMPTY);
}
/**
@@ -100,11 +101,17 @@ public class TimerTaskListenerForMarkerFileProtocol extends DummyTimerTaskListen
{
if (successorMarkerFileOrNull != null && hasPerformedMeaningfullWork(statusProviderOrNull))
{
- touch(successorMarkerFileOrNull);
+ touch(successorMarkerFileOrNull, StringUtils.EMPTY);
}
- if (errorMarkerFileOrNull != null && hasErrors(statusProviderOrNull))
+ if (errorMarkerFileOrNull != null)
{
- touch(errorMarkerFileOrNull);
+ if (hasErrors(statusProviderOrNull))
+ {
+ touch(errorMarkerFileOrNull, getErrorLog(statusProviderOrNull));
+ } else
+ {
+ errorMarkerFileOrNull.delete();
+ }
}
// Avoid deleting the marker file when it is used as error marker file, too, and an error
// occurred.
@@ -125,6 +132,22 @@ public class TimerTaskListenerForMarkerFileProtocol extends DummyTimerTaskListen
return (statusProviderOrNull != null) && statusProviderOrNull.hasErrors();
}
+ private String getErrorLog(ITimerTaskStatusProvider statusProviderOrNull)
+ {
+ if (statusProviderOrNull == null)
+ {
+ return StringUtils.EMPTY;
+ }
+ final String errorLogOrNull = statusProviderOrNull.tryGetErrorLog();
+ if (errorLogOrNull == null)
+ {
+ return StringUtils.EMPTY;
+ } else
+ {
+ return errorLogOrNull;
+ }
+ }
+
private static void failIfDirectory(File markerFile)
{
if (markerFile.isDirectory())
@@ -134,11 +157,12 @@ public class TimerTaskListenerForMarkerFileProtocol extends DummyTimerTaskListen
}
}
- private static void touch(final File markerFile) throws EnvironmentFailureException
+ private static void touch(final File markerFile, String message)
+ throws EnvironmentFailureException
{
try
{
- FileUtils.touch(markerFile);
+ FileUtils.writeStringToFile(markerFile, message);
} catch (IOException ex)
{
throw new EnvironmentFailureException("Can not create marker file '"
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/common/MarkerFile.java b/datamover/source/java/ch/systemsx/cisd/datamover/common/MarkerFile.java
index 3258cf92a05..b53fdea9df5 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/common/MarkerFile.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/common/MarkerFile.java
@@ -55,6 +55,13 @@ public class MarkerFile
return file.getName().startsWith(Constants.DELETION_IN_PROGRESS_PREFIX);
}
+ public static boolean isFaultyPathsFile(File file)
+ {
+ assert file != null;
+
+ return Constants.FAULTY_PATH_FILENAME.equals(file.getName());
+ }
+
public static File extractOriginalFromCopyFinishedMarker(File markerFile)
{
assert isCopyFinishedMarker(markerFile);
@@ -80,4 +87,5 @@ public class MarkerFile
{
return new StoreItem(".requiresDeletionBeforeCreation");
}
+
}
--
GitLab