From 7bdabad5e47300948c930c1fd8eda210c60c0877 Mon Sep 17 00:00:00 2001
From: ribeaudc <ribeaudc>
Date: Wed, 14 May 2008 19:09:08 +0000
Subject: [PATCH] [DMV-12] add: - 'DirectoryScanningChangeListener'. -
'highwater-mark' value in integration tests for both datamovers. change: -
'Main.UnhandledPathsChangeListener' renamed to 'PathHandlerInterceptor' and
moved to 'common'. - 'LocalProcessor' constructor and 'LocalProcessor.create'
method removed. - 'IFileStore' extends 'ISelfTestable' and default 'check'
method implementation in 'FileStore'. minor: - Javadoc improved in
'IPathMover' and 'FileStoreRemoteMounted'. - final keyword added when
possible.
SVN: 6050
---
.../ch/systemsx/cisd/datamover/DataMover.java | 11 ++--
.../cisd/datamover/LocalProcessor.java | 64 ++++++++-----------
.../systemsx/cisd/datamover/Parameters.java | 49 +++++++-------
.../ch/systemsx/cisd/datamover/SelfTest.java | 18 ++----
.../RemoteMonitoredMoverFactory.java | 12 +++-
.../datamover/filesystem/intf/FileStore.java | 56 ++++++++++------
.../datamover/filesystem/intf/IFileStore.java | 4 +-
.../datamover/filesystem/intf/IPathMover.java | 4 ++
.../filesystem/remote/RemotePathMover.java | 9 ++-
.../filesystem/store/FileStoreLocal.java | 46 ++++++-------
.../filesystem/store/FileStoreRemote.java | 52 ++++++++-------
.../store/FileStoreRemoteMounted.java | 36 +++++++----
.../cisd/datamover/utils/LocalBufferDirs.java | 2 +-
.../cisd/datamover/ParametersTest.java | 2 +-
.../systemsx/cisd/datamover/SelfTestTest.java | 9 +--
15 files changed, 201 insertions(+), 173 deletions(-)
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/DataMover.java b/datamover/source/java/ch/systemsx/cisd/datamover/DataMover.java
index 996e5d95c18..bcf6b5b99a3 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/DataMover.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/DataMover.java
@@ -20,6 +20,7 @@ import java.io.File;
import java.util.Timer;
import ch.systemsx.cisd.common.Constants;
+import ch.systemsx.cisd.common.highwatermark.PathHandlerInterceptor;
import ch.systemsx.cisd.common.utilities.DirectoryScanningTimerTask;
import ch.systemsx.cisd.common.utilities.FileUtilities;
import ch.systemsx.cisd.common.utilities.IStoreHandler;
@@ -134,12 +135,14 @@ public class DataMover
private DataMoverProcess createLocalProcessor()
{
final LocalProcessor localProcessor =
- LocalProcessor.create(parameters, bufferDirs.getCopyCompleteDir(), bufferDirs
- .getReadyToMoveDir(), bufferDirs.getTempDir(), factory, bufferDirs
- .getBufferDirHighwaterMark());
+ new LocalProcessor(parameters, bufferDirs, factory.getImmutableCopier(), factory
+ .getMover());
+ final PathHandlerInterceptor pathHandlerInterceptor =
+ new PathHandlerInterceptor(localProcessor);
final DirectoryScanningTimerTask localProcessingTask =
new DirectoryScanningTimerTask(bufferDirs.getCopyCompleteDir(),
- FileUtilities.ACCEPT_ALL_FILTER, localProcessor);
+ FileUtilities.ACCEPT_ALL_FILTER, pathHandlerInterceptor);
+ pathHandlerInterceptor.setDirectoryScanning(localProcessingTask);
return new DataMoverProcess(localProcessingTask, "Local Processor", localProcessor);
}
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/LocalProcessor.java b/datamover/source/java/ch/systemsx/cisd/datamover/LocalProcessor.java
index e04c7b5907b..f3a3b61a9c4 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/LocalProcessor.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/LocalProcessor.java
@@ -23,7 +23,7 @@ import java.util.regex.Pattern;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
-import ch.systemsx.cisd.common.highwatermark.FileWithHighwaterMark;
+import ch.systemsx.cisd.common.highwatermark.HighwaterMarkWatcher;
import ch.systemsx.cisd.common.logging.ISimpleLogger;
import ch.systemsx.cisd.common.logging.Log4jSimpleLogger;
import ch.systemsx.cisd.common.logging.LogCategory;
@@ -33,9 +33,9 @@ import ch.systemsx.cisd.common.utilities.IPathHandler;
import ch.systemsx.cisd.common.utilities.IPathImmutableCopier;
import ch.systemsx.cisd.common.utilities.RegexFileFilter;
import ch.systemsx.cisd.common.utilities.RegexFileFilter.PathType;
-import ch.systemsx.cisd.datamover.filesystem.intf.IFileSysOperationsFactory;
import ch.systemsx.cisd.datamover.filesystem.intf.IPathMover;
import ch.systemsx.cisd.datamover.filesystem.intf.IRecoverableTimerTaskFactory;
+import ch.systemsx.cisd.datamover.utils.LocalBufferDirs;
/**
* Processing of the files on the local machine. This class does not scan its input directory, all
@@ -76,33 +76,21 @@ public class LocalProcessor implements IPathHandler, IRecoverableTimerTaskFactor
// destination as soon as they appear there.
private final File tempDir;
- private final FileWithHighwaterMark extraCopyDirOrNull;
+ private final File extraCopyDirOrNull;
- @SuppressWarnings("unused")
- private final long bufferDirWatermark;
+ private final HighwaterMarkWatcher highwaterMarkWatcher;
- private LocalProcessor(final Parameters parameters, final File inputDir, final File outputDir,
- final File tempDir, final IFileSysOperationsFactory factory,
- final long bufferDirHighwaterMark)
+ LocalProcessor(final Parameters parameters, final LocalBufferDirs bufferDirs,
+ final IPathImmutableCopier copier, final IPathMover mover)
{
this.parameters = parameters;
- this.inputDir = inputDir;
- this.outputDir = outputDir;
- this.tempDir = tempDir;
- this.bufferDirWatermark = bufferDirHighwaterMark;
+ this.inputDir = bufferDirs.getCopyCompleteDir();
+ this.outputDir = bufferDirs.getReadyToMoveDir();
+ this.tempDir = bufferDirs.getTempDir();
+ highwaterMarkWatcher = new HighwaterMarkWatcher(bufferDirs.getBufferDirHighwaterMark());
this.extraCopyDirOrNull = parameters.tryGetExtraCopyDir();
- this.copier = factory.getImmutableCopier();
- this.mover = factory.getMover();
- }
-
- public static final LocalProcessor create(final Parameters parameters, final File inputDir,
- final File outputDir, final File tempDir, final IFileSysOperationsFactory factory,
- final long bufferDirHighwaterMark)
- {
- final LocalProcessor handlerAndRecoverable =
- new LocalProcessor(parameters, inputDir, outputDir, tempDir, factory,
- bufferDirHighwaterMark);
- return handlerAndRecoverable;
+ this.copier = copier;
+ this.mover = mover;
}
// ----------------
@@ -156,9 +144,7 @@ public class LocalProcessor implements IPathHandler, IRecoverableTimerTaskFactor
// errors.
if (extraCopyDirOrNull != null)
{
- // TODO 2008-05-14, Christian Ribeaud: We should work with the high water mark
- // here.
- mover.tryMove(file, extraCopyDirOrNull.getFile());
+ mover.tryMove(file, extraCopyDirOrNull);
}
}
}
@@ -210,15 +196,9 @@ public class LocalProcessor implements IPathHandler, IRecoverableTimerTaskFactor
return pathDeleted;
}
- private enum EFileManipResult
- {
- CONTINUE, FAILURE, STOP
- }
-
private EFileManipResult doManualIntervention(final File resource)
{
- // TODO 2008-05-14, Christian Ribeaud: We should work with the high water mark here.
- final File manualInterventionDir = parameters.tryGetManualInterventionDir().getFile();
+ final File manualInterventionDir = parameters.tryGetManualInterventionDir();
if (manualInterventionDir == null)
{
return EFileManipResult.CONTINUE;
@@ -310,8 +290,7 @@ public class LocalProcessor implements IPathHandler, IRecoverableTimerTaskFactor
if (extraTmpCopy != null)
{
assert extraCopyDirOrNull != null;
- // TODO 2008-05-14, Christian Ribeaud: We should work with the high water mark here.
- final File extraCopy = mover.tryMove(extraTmpCopy, extraCopyDirOrNull.getFile());
+ final File extraCopy = mover.tryMove(extraTmpCopy, extraCopyDirOrNull);
if (extraCopy == null)
{
notificationLog.error(String.format(
@@ -323,6 +302,17 @@ public class LocalProcessor implements IPathHandler, IRecoverableTimerTaskFactor
public final boolean mayHandle(final File path)
{
- return true;
+ highwaterMarkWatcher.setPathAndRun(outputDir);
+ return highwaterMarkWatcher.isBelow() == false;
}
+
+ //
+ // Helper classes
+ //
+
+ private enum EFileManipResult
+ {
+ CONTINUE, FAILURE, STOP
+ }
+
}
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/Parameters.java b/datamover/source/java/ch/systemsx/cisd/datamover/Parameters.java
index f99b22f1685..a591c2dd4b1 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/Parameters.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/Parameters.java
@@ -212,8 +212,8 @@ public class Parameters implements ITimingParameters, IFileSysParameters
* the same file system than {@link #bufferDirectory}.
*/
@Option(longName = PropertyNames.MANUAL_INTERVENTION_DIR, metaVar = "DIR", usage = "The local directory to "
- + "store paths that need manual intervention.", handler = FileWithHighwaterMarkHandler.class)
- private FileWithHighwaterMark manualInterventionDirectoryOrNull = null;
+ + "store paths that need manual intervention.")
+ private File manualInterventionDirectoryOrNull = null;
/**
* The directory on the remote side to move the paths to from the buffer directory.
@@ -233,8 +233,8 @@ public class Parameters implements ITimingParameters, IFileSysParameters
* directory is specified)
*/
@Option(longName = PropertyNames.EXTRA_COPY_DIR, metaVar = "DIR", usage = "The local directory where we create additional "
- + "copy of the incoming data.", handler = FileWithHighwaterMarkHandler.class)
- private FileWithHighwaterMark extraCopyDirectory = null;
+ + "copy of the incoming data.")
+ private File extraCopyDirectory = null;
/**
* The regular expression to use for cleansing on the incoming directory before moving it to the
@@ -408,11 +408,7 @@ public class Parameters implements ITimingParameters, IFileSysParameters
Boolean.parseBoolean(serviceProperties.getProperty("treat-incoming-as-remote",
Boolean.toString(DEFAULT_TREAT_INCOMING_AS_REMOTE)).trim());
prefixForIncoming = serviceProperties.getProperty("prefix-for-incoming", "").trim();
- if (serviceProperties.getProperty(PropertyNames.INCOMING_DIR) != null)
- {
- incomingDirectory =
- new File(serviceProperties.getProperty(PropertyNames.INCOMING_DIR).trim());
- }
+ incomingDirectory = tryCreateFile(serviceProperties, PropertyNames.INCOMING_DIR);
incomingHost = serviceProperties.getProperty("incoming-host");
if (serviceProperties.getProperty(PropertyNames.BUFFER_DIR) != null)
{
@@ -420,12 +416,8 @@ public class Parameters implements ITimingParameters, IFileSysParameters
FileWithHighwaterMark.fromProperties(serviceProperties,
PropertyNames.BUFFER_DIR);
}
- if (serviceProperties.getProperty(PropertyNames.MANUAL_INTERVENTION_DIR) != null)
- {
- manualInterventionDirectoryOrNull =
- FileWithHighwaterMark.fromProperties(serviceProperties,
- PropertyNames.MANUAL_INTERVENTION_DIR);
- }
+ manualInterventionDirectoryOrNull =
+ tryCreateFile(serviceProperties, PropertyNames.MANUAL_INTERVENTION_DIR);
if (serviceProperties.getProperty(PropertyNames.OUTGOING_DIR) != null)
{
outgoingDirectory =
@@ -433,12 +425,7 @@ public class Parameters implements ITimingParameters, IFileSysParameters
PropertyNames.OUTGOING_DIR);
}
outgoingHost = serviceProperties.getProperty("outgoing-host");
- if (serviceProperties.getProperty(PropertyNames.EXTRA_COPY_DIR) != null)
- {
- extraCopyDirectory =
- FileWithHighwaterMark.fromProperties(serviceProperties,
- PropertyNames.EXTRA_COPY_DIR);
- }
+ extraCopyDirectory = tryCreateFile(serviceProperties, PropertyNames.EXTRA_COPY_DIR);
if (serviceProperties.getProperty("cleansing-regex") != null)
{
cleansingRegex = Pattern.compile(serviceProperties.getProperty("cleansing-regex"));
@@ -450,6 +437,18 @@ public class Parameters implements ITimingParameters, IFileSysParameters
}
}
+ private final File tryCreateFile(final Properties serviceProperties, final String propertyKey)
+ {
+ final String propertyValue = serviceProperties.getProperty(propertyKey);
+ if (propertyValue != null)
+ {
+ return new File(propertyValue.trim());
+ } else
+ {
+ return null;
+ }
+ }
+
/**
* Returns the service property.
*
@@ -600,7 +599,7 @@ public class Parameters implements ITimingParameters, IFileSysParameters
* directory for long enough and that need manual intervention. Note that this directory
* needs to be on the same file system as {@link #getBufferDirectoryPath}.
*/
- public FileWithHighwaterMark tryGetManualInterventionDir()
+ public File tryGetManualInterventionDir()
{
return manualInterventionDirectoryOrNull;
}
@@ -610,7 +609,7 @@ public class Parameters implements ITimingParameters, IFileSysParameters
* <code>null</code> if it is not specified. Note that this directory needs to be on
* the same file system as {@link #getBufferDirectoryPath}.
*/
- public FileWithHighwaterMark tryGetExtraCopyDir()
+ public File tryGetExtraCopyDir()
{
return extraCopyDirectory;
}
@@ -670,12 +669,12 @@ public class Parameters implements ITimingParameters, IFileSysParameters
if (null != tryGetManualInterventionDir())
{
operationLog.info(String.format("Manual interventions directory: '%s'.",
- tryGetManualInterventionDir().getCanonicalPath()));
+ tryGetManualInterventionDir().getAbsolutePath()));
}
if (null != extraCopyDirectory)
{
operationLog.info(String.format("Extra copy directory: '%s'.", extraCopyDirectory
- .getCanonicalPath()));
+ .getAbsolutePath()));
}
operationLog.info(String.format("Check intervall (external): %d s.",
getCheckIntervalMillis() / 1000));
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/SelfTest.java b/datamover/source/java/ch/systemsx/cisd/datamover/SelfTest.java
index 97514c8cc59..7bf04ccd6d9 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/SelfTest.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/SelfTest.java
@@ -24,6 +24,7 @@ import ch.systemsx.cisd.common.exceptions.HighLevelException;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.logging.LogInitializer;
+import ch.systemsx.cisd.common.utilities.ISelfTestable;
import ch.systemsx.cisd.datamover.filesystem.intf.IFileStore;
import ch.systemsx.cisd.datamover.filesystem.intf.IPathCopier;
@@ -34,8 +35,6 @@ import ch.systemsx.cisd.datamover.filesystem.intf.IPathCopier;
*/
public class SelfTest
{
- private static final long TIMEOUT_MILLIS = 3000L;
-
private static final Logger operationLog =
LogFactory.getLogger(LogCategory.OPERATION, SelfTest.class);
@@ -44,19 +43,14 @@ public class SelfTest
LogInitializer.init();
}
- private static void checkPathRecords(final IFileStore[] pathRecords)
+ private static void checkPathRecords(final IFileStore[] fileStores)
{
- assert pathRecords != null;
-
- checkPathRecordsContainEachOther(pathRecords);
- for (final IFileStore pathRecord : pathRecords)
+ assert fileStores != null : "Unspecified file stores";
+ for (final ISelfTestable fileStore : fileStores)
{
- final String errorMessage = pathRecord.tryCheckDirectoryFullyAccessible(TIMEOUT_MILLIS);
- if (errorMessage != null)
- {
- throw new ConfigurationFailureException(errorMessage);
- }
+ fileStore.check();
}
+ checkPathRecordsContainEachOther(fileStores);
}
private static void checkPathRecordsContainEachOther(final IFileStore[] store)
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/RemoteMonitoredMoverFactory.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/RemoteMonitoredMoverFactory.java
index 206da3562dc..0f2ce786b02 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/RemoteMonitoredMoverFactory.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/RemoteMonitoredMoverFactory.java
@@ -27,8 +27,14 @@ import ch.systemsx.cisd.datamover.intf.ITimingParameters;
/**
* @author Tomasz Pylak
*/
-public class RemoteMonitoredMoverFactory
+public final class RemoteMonitoredMoverFactory
{
+
+ private RemoteMonitoredMoverFactory()
+ {
+ // Can not be instantiated.
+ }
+
/**
* Creates a handler to move files remotely from source to destination and monitor the progress
*
@@ -36,8 +42,8 @@ public class RemoteMonitoredMoverFactory
* @param destinationDirectory The directory to move paths to.
* @param parameters The timing parameters used for monitoring and reporting stall situations.
*/
- public static final IStoreHandler create(IFileStore sourceDirectory,
- FileStore destinationDirectory, ITimingParameters parameters)
+ public static final IStoreHandler create(final IFileStore sourceDirectory,
+ final FileStore destinationDirectory, final ITimingParameters parameters)
{
final IStoreCopier copier = sourceDirectory.getCopier(destinationDirectory);
final CopyActivityMonitor monitor =
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/FileStore.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/FileStore.java
index ffc81df8838..9fec5a272b8 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/FileStore.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/FileStore.java
@@ -23,11 +23,15 @@ import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.lang.builder.HashCodeBuilder;
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.highwatermark.FileWithHighwaterMark;
+import ch.systemsx.cisd.common.highwatermark.HighwaterMarkSelfTestable;
+import ch.systemsx.cisd.common.highwatermark.HighwaterMarkWatcher;
import ch.systemsx.cisd.common.logging.ISimpleLogger;
import ch.systemsx.cisd.common.utilities.StoreItem;
+import ch.systemsx.cisd.datamover.filesystem.remote.RemotePathMover;
/**
* The abstract super-class of classes that represent a file store.
@@ -37,13 +41,13 @@ import ch.systemsx.cisd.common.utilities.StoreItem;
*/
public abstract class FileStore implements IFileStore
{
- private final FileWithHighwaterMark path;
+ private final FileWithHighwaterMark fileWithHighwaterMark;
- protected final String hostOrNull;
+ private final String hostOrNull;
- protected final String kind;
+ private final String kind;
- protected final boolean remote;
+ private final boolean remote;
protected final IFileSysOperationsFactory factory;
@@ -52,16 +56,32 @@ public abstract class FileStore implements IFileStore
{
assert path != null;
assert kind != null;
- this.path = path;
+ this.fileWithHighwaterMark = path;
this.kind = kind;
this.hostOrNull = hostOrNull;
this.remote = remote;
this.factory = factory;
}
+ private final String getCanonicalPath(final File file)
+ {
+ if (hostOrNull != null)
+ {
+ return file.getPath();
+ }
+ try
+ {
+ return file.getCanonicalPath() + File.separator;
+ } catch (final IOException e)
+ {
+ throw EnvironmentFailureException.fromTemplate(e,
+ "Cannot determine canonical form of path '%s'", file.getPath());
+ }
+ }
+
protected final File getPath()
{
- return path.getFile();
+ return fileWithHighwaterMark.getFile();
}
protected final String tryGetHost()
@@ -129,7 +149,7 @@ public abstract class FileStore implements IFileStore
return remote;
}
- public boolean isParentDirectory(final IFileStore child)
+ public final boolean isParentDirectory(final IFileStore child)
{
if (child instanceof FileStore == false)
{
@@ -141,20 +161,16 @@ public abstract class FileStore implements IFileStore
getCanonicalPath(getPath()));
}
- private String getCanonicalPath(final File file)
+ public void check() throws EnvironmentFailureException, ConfigurationFailureException
{
- if (hostOrNull != null)
- {
- return file.getPath();
- }
- try
- {
- return file.getCanonicalPath() + File.separator;
- } catch (final IOException e)
+ final String errorMessage =
+ tryCheckDirectoryFullyAccessible(RemotePathMover.DIRECTORY_ACCESSIBLE_TIMEOUT_MILLIS);
+ if (errorMessage != null)
{
- throw EnvironmentFailureException.fromTemplate(e,
- "Cannot determine canonical form of path '%s'", file.getPath());
+ throw new ConfigurationFailureException(errorMessage);
}
+ new HighwaterMarkSelfTestable(fileWithHighwaterMark.getFile(), new HighwaterMarkWatcher(
+ fileWithHighwaterMark.getHighwaterMark())).check();
}
//
@@ -176,7 +192,7 @@ public abstract class FileStore implements IFileStore
final EqualsBuilder equalsBuilder = new EqualsBuilder();
equalsBuilder.append(hostOrNull, that.hostOrNull);
equalsBuilder.append(kind, that.kind);
- equalsBuilder.append(path, that.path);
+ equalsBuilder.append(fileWithHighwaterMark, that.fileWithHighwaterMark);
return equalsBuilder.isEquals();
}
@@ -186,7 +202,7 @@ public abstract class FileStore implements IFileStore
final HashCodeBuilder builder = new HashCodeBuilder();
builder.append(hostOrNull);
builder.append(kind);
- builder.append(path);
+ builder.append(fileWithHighwaterMark);
return builder.toHashCode();
}
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IFileStore.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IFileStore.java
index f304ac567db..211e844ad9f 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IFileStore.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IFileStore.java
@@ -18,6 +18,7 @@ package ch.systemsx.cisd.datamover.filesystem.intf;
import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.logging.ISimpleLogger;
+import ch.systemsx.cisd.common.utilities.ISelfTestable;
import ch.systemsx.cisd.common.utilities.StoreItem;
/**
@@ -27,7 +28,7 @@ import ch.systemsx.cisd.common.utilities.StoreItem;
*/
// TODO 2008-05-13, Christian Ribeaud: concrete classes seem to be more widely used than this
// interface.
-public interface IFileStore
+public interface IFileStore extends ISelfTestable
{
/**
@@ -101,5 +102,4 @@ public interface IFileStore
public String getLocationDescription(StoreItem item);
public IExtendedFileStore tryAsExtended();
-
}
\ No newline at end of file
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IPathMover.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IPathMover.java
index b97fd758c61..6744bd895ca 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IPathMover.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IPathMover.java
@@ -30,11 +30,15 @@ public interface IPathMover
* Moves source path (file or directory) to destination directory, putting <var>prefixTemplate</var>
* in front of its name, where any occurrence of '%t' in the template is replaced with the
* current time stamp.
+ *
+ * @return the new path created, or <code>null</code> if the operation fails.
*/
public File tryMove(File sourcePath, File destinationDirectory, String prefixTemplate);
/**
* Moves source path (file or directory) to destination directory.
+ *
+ * @return the new path created, or <code>null</code> if the operation fails.
*/
public File tryMove(File sourcePath, File destinationDir);
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/remote/RemotePathMover.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/remote/RemotePathMover.java
index 541c268f88d..42f6573a19b 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/remote/RemotePathMover.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/remote/RemotePathMover.java
@@ -41,7 +41,10 @@ import ch.systemsx.cisd.datamover.intf.ITimingParameters;
public final class RemotePathMover implements IStoreHandler
{
- private static final long TIMEOUT_DESTINATION_MILLIS = 3000L;
+ /**
+ * Number of milliseconds to wait before considering that the directory is not fully accessible.
+ */
+ public static final long DIRECTORY_ACCESSIBLE_TIMEOUT_MILLIS = 3000L;
private static final String START_COPYING_PATH_TEMPLATE = "Start copying path '%s' to '%s'.";
@@ -112,7 +115,7 @@ public final class RemotePathMover implements IStoreHandler
|| destinationDirectory.tryAsExtended() != null;
final String errorMsg =
- destinationDirectory.tryCheckDirectoryFullyAccessible(TIMEOUT_DESTINATION_MILLIS);
+ destinationDirectory.tryCheckDirectoryFullyAccessible(DIRECTORY_ACCESSIBLE_TIMEOUT_MILLIS);
if (StringUtils.isNotBlank(errorMsg))
{
throw new ConfigurationFailureException(errorMsg);
@@ -225,7 +228,7 @@ public final class RemotePathMover implements IStoreHandler
private boolean checkTargetAvailable()
{
final String msg =
- destinationDirectory.tryCheckDirectoryFullyAccessible(TIMEOUT_DESTINATION_MILLIS);
+ destinationDirectory.tryCheckDirectoryFullyAccessible(DIRECTORY_ACCESSIBLE_TIMEOUT_MILLIS);
if (msg != null)
{
machineLog.error(msg);
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreLocal.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreLocal.java
index 738e5f84650..7a3aa1aa390 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreLocal.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreLocal.java
@@ -26,6 +26,7 @@ import ch.systemsx.cisd.common.highwatermark.FileWithHighwaterMark;
import ch.systemsx.cisd.common.logging.ISimpleLogger;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.utilities.DirectoryScannedStore;
import ch.systemsx.cisd.common.utilities.FileUtilities;
import ch.systemsx.cisd.common.utilities.StoreItem;
import ch.systemsx.cisd.datamover.common.MarkerFile;
@@ -39,6 +40,8 @@ import ch.systemsx.cisd.datamover.filesystem.intf.IStoreCopier;
import ch.systemsx.cisd.datamover.filesystem.intf.FileStore.ExtendedFileStore;
/**
+ * An {@link IFileStore} implementation for local stores.
+ *
* @author Tomasz Pylak
*/
public class FileStoreLocal extends ExtendedFileStore
@@ -63,32 +66,33 @@ public class FileStoreLocal extends ExtendedFileStore
//
@Override
- public Status delete(final StoreItem item)
+ public final Status delete(final StoreItem item)
{
return remover.remove(getChildFile(item));
}
@Override
- public boolean exists(final StoreItem item)
+ public final boolean exists(final StoreItem item)
{
return getChildFile(item).exists();
}
@Override
- public long lastChanged(final StoreItem item, final long stopWhenFindYounger)
+ public final long lastChanged(final StoreItem item, final long stopWhenFindYounger)
{
return FileUtilities.lastChanged(getChildFile(item), true, stopWhenFindYounger);
}
@Override
- public long lastChangedRelative(final StoreItem item, final long stopWhenFindYoungerRelative)
+ public final long lastChangedRelative(final StoreItem item,
+ final long stopWhenFindYoungerRelative)
{
return FileUtilities.lastChangedRelative(getChildFile(item), true,
stopWhenFindYoungerRelative);
}
@Override
- public String tryCheckDirectoryFullyAccessible(final long timeOutMillis)
+ public final String tryCheckDirectoryFullyAccessible(final long timeOutMillis)
{
final boolean available = FileUtilities.isAvailable(getPath(), timeOutMillis);
if (available == false)
@@ -101,7 +105,7 @@ public class FileStoreLocal extends ExtendedFileStore
}
@Override
- public IStoreCopier getCopier(final FileStore destinationDirectory)
+ public final IStoreCopier getCopier(final FileStore destinationDirectory)
{
boolean requiresDeletion = false;
final IStoreCopier simpleCopier =
@@ -117,13 +121,13 @@ public class FileStoreLocal extends ExtendedFileStore
}
@Override
- public IExtendedFileStore tryAsExtended()
+ public final IExtendedFileStore tryAsExtended()
{
return this;
}
@Override
- public boolean createNewFile(final StoreItem item)
+ public final boolean createNewFile(final StoreItem item)
{
try
{
@@ -138,49 +142,39 @@ public class FileStoreLocal extends ExtendedFileStore
}
@Override
- public File tryMoveLocal(final StoreItem sourceItem, final File destinationDir,
+ public final File tryMoveLocal(final StoreItem sourceItem, final File destinationDir,
final String newFilePrefix)
{
return mover.tryMove(getChildFile(sourceItem), destinationDir, newFilePrefix);
}
@Override
- public String toString()
+ public final String toString()
{
final String pathStr = getPath().getPath();
return "[local fs]" + pathStr;
}
@Override
- public String getLocationDescription(final StoreItem item)
+ public final String getLocationDescription(final StoreItem item)
{
return getChildFile(item).getPath();
}
@Override
- public StoreItem[] tryListSortByLastModified(final ISimpleLogger loggerOrNull)
+ public final StoreItem[] tryListSortByLastModified(final ISimpleLogger loggerOrNull)
{
final File[] files = FileUtilities.tryListFiles(getPath(), loggerOrNull);
if (files != null)
{
FileUtilities.sortByLastModified(files);
- return asItems(files);
+ return DirectoryScannedStore.asItems(files);
} else
{
return null;
}
}
- private static StoreItem[] asItems(final File[] files)
- {
- final StoreItem[] items = new StoreItem[files.length];
- for (int i = 0; i < items.length; i++)
- {
- items[i] = new StoreItem(files[i].getName());
- }
- return items;
- }
-
// ------
/**
@@ -188,7 +182,7 @@ public class FileStoreLocal extends ExtendedFileStore
* <var>destinationDirectory</var> resides requires deleting an existing file before it
* can be overwritten.
*/
- protected boolean requiresDeletionBeforeCreation(final IFileStore destinationDirectory,
+ protected final boolean requiresDeletionBeforeCreation(final IFileStore destinationDirectory,
final IStoreCopier simpleCopier)
{
final StoreItem item = MarkerFile.createRequiresDeletionBeforeCreationMarker();
@@ -213,7 +207,7 @@ public class FileStoreLocal extends ExtendedFileStore
return requiresDeletion;
}
- private static void logFIleSystemNeedsOverwrite(final IFileStore destinationDirectory)
+ private final static void logFIleSystemNeedsOverwrite(final IFileStore destinationDirectory)
{
if (machineLog.isInfoEnabled())
{
@@ -223,7 +217,7 @@ public class FileStoreLocal extends ExtendedFileStore
}
}
- private static void logCopierOverwriteState(final IFileStore destinationDirectory,
+ private final static void logCopierOverwriteState(final IFileStore destinationDirectory,
final boolean requiresDeletion)
{
if (machineLog.isInfoEnabled())
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreRemote.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreRemote.java
index 6a99391e626..f95c41c2820 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreRemote.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreRemote.java
@@ -16,6 +16,9 @@
package ch.systemsx.cisd.datamover.filesystem.store;
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.common.exceptions.NotImplementedException;
import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.highwatermark.FileWithHighwaterMark;
import ch.systemsx.cisd.common.logging.ISimpleLogger;
@@ -30,7 +33,6 @@ import ch.systemsx.cisd.datamover.filesystem.intf.IStoreCopier;
*/
// TODO 2007-10-09, Tomasz Pylak: Ssh tunneling mode should be implemented here. This class is a
// dummy implementation.
-// Methods with NOTE should be written.
public class FileStoreRemote extends FileStore
{
public FileStoreRemote(final FileWithHighwaterMark path, final String host, final String kind,
@@ -44,69 +46,71 @@ public class FileStoreRemote extends FileStore
//
@Override
- public IExtendedFileStore tryAsExtended()
+ public final IExtendedFileStore tryAsExtended()
{
return null;
}
@Override
- public Status delete(final StoreItem item)
+ public final Status delete(final StoreItem item)
{
- // NOTE: implement this
- return Status.OK;
+ throw new NotImplementedException();
}
@Override
- public boolean exists(final StoreItem item)
+ public final boolean exists(final StoreItem item)
{
- return factory.getCopier(false).existsRemotely(getPath(), hostOrNull);
+ return factory.getCopier(false).existsRemotely(getPath(), tryGetHost());
}
@Override
- public IStoreCopier getCopier(final FileStore destinationDirectory)
+ public final IStoreCopier getCopier(final FileStore destinationDirectory)
{
final boolean requiresDeletion = false;
return constructStoreCopier(destinationDirectory, requiresDeletion);
}
@Override
- public long lastChanged(final StoreItem item, final long stopWhenFindYounger)
+ public final long lastChanged(final StoreItem item, final long stopWhenFindYounger)
{
- // NOTE: implement this
- return 0;
+ throw new NotImplementedException();
}
@Override
- public long lastChangedRelative(final StoreItem item, final long stopWhenFindYoungerRelative)
+ public final long lastChangedRelative(final StoreItem item,
+ final long stopWhenFindYoungerRelative)
{
- // NOTE: implement this
- return 0;
+ throw new NotImplementedException();
}
@Override
- public String tryCheckDirectoryFullyAccessible(final long timeOutMillis)
+ public final String tryCheckDirectoryFullyAccessible(final long timeOutMillis)
{
- // NOTE: implement this
- return null;
+ throw new NotImplementedException();
}
@Override
- public String toString()
+ public final String toString()
{
final String pathStr = getPath().getPath();
- return "[remote fs]" + hostOrNull + ":" + pathStr;
+ return "[remote fs]" + tryGetHost() + ":" + pathStr;
}
@Override
- public String getLocationDescription(final StoreItem item)
+ public final String getLocationDescription(final StoreItem item)
{
- return hostOrNull + ":" + getChildFile(item).getPath();
+ return tryGetHost() + ":" + getChildFile(item).getPath();
}
@Override
- public StoreItem[] tryListSortByLastModified(final ISimpleLogger loggerOrNull)
+ public final StoreItem[] tryListSortByLastModified(final ISimpleLogger loggerOrNull)
{
- // NOTE: implement this
- return null;
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public final void check() throws EnvironmentFailureException, ConfigurationFailureException
+ {
+ throw new NotImplementedException();
}
}
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreRemoteMounted.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreRemoteMounted.java
index ef67ac2b6aa..259871471f8 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreRemoteMounted.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreRemoteMounted.java
@@ -16,6 +16,8 @@
package ch.systemsx.cisd.datamover.filesystem.store;
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.highwatermark.FileWithHighwaterMark;
import ch.systemsx.cisd.common.logging.ISimpleLogger;
@@ -26,9 +28,14 @@ import ch.systemsx.cisd.datamover.filesystem.intf.IFileSysOperationsFactory;
import ch.systemsx.cisd.datamover.filesystem.intf.IStoreCopier;
/**
+ * A <code>FileStore</code> extension for remote paths mounted.
+ * <p>
+ * The works is mainly delegated to an internal {@link FileStoreLocal}.
+ * </p>
+ *
* @author Tomasz Pylak
*/
-public class FileStoreRemoteMounted extends FileStore
+public final class FileStoreRemoteMounted extends FileStore
{
private final FileStoreLocal localImpl;
@@ -44,64 +51,71 @@ public class FileStoreRemoteMounted extends FileStore
//
@Override
- public IExtendedFileStore tryAsExtended()
+ public final IExtendedFileStore tryAsExtended()
{
return null;
}
@Override
- public IStoreCopier getCopier(final FileStore destinationDirectory)
+ public final IStoreCopier getCopier(final FileStore destinationDirectory)
{
final boolean requiresDeletion = false;
return constructStoreCopier(destinationDirectory, requiresDeletion);
}
@Override
- public String toString()
+ public final String toString()
{
final String pathStr = getPath().getPath();
return "[mounted remote fs]" + pathStr;
}
@Override
- public String getLocationDescription(final StoreItem item)
+ public final String getLocationDescription(final StoreItem item)
{
return localImpl.getLocationDescription(item);
}
@Override
- public Status delete(final StoreItem item)
+ public final Status delete(final StoreItem item)
{
return localImpl.delete(item);
}
@Override
- public boolean exists(final StoreItem item)
+ public final boolean exists(final StoreItem item)
{
return localImpl.exists(item);
}
@Override
- public long lastChanged(final StoreItem item, final long stopWhenFindYounger)
+ public final long lastChanged(final StoreItem item, final long stopWhenFindYounger)
{
return localImpl.lastChanged(item, stopWhenFindYounger);
}
@Override
- public long lastChangedRelative(final StoreItem item, final long stopWhenFindYoungerRelative)
+ public final long lastChangedRelative(final StoreItem item,
+ final long stopWhenFindYoungerRelative)
{
return localImpl.lastChangedRelative(item, stopWhenFindYoungerRelative);
}
@Override
- public String tryCheckDirectoryFullyAccessible(final long timeOutMillis)
+ public final String tryCheckDirectoryFullyAccessible(final long timeOutMillis)
{
return localImpl.tryCheckDirectoryFullyAccessible(timeOutMillis);
}
@Override
- public StoreItem[] tryListSortByLastModified(final ISimpleLogger loggerOrNull)
+ public final StoreItem[] tryListSortByLastModified(final ISimpleLogger loggerOrNull)
{
return localImpl.tryListSortByLastModified(loggerOrNull);
}
+
+ @Override
+ public void check() throws EnvironmentFailureException, ConfigurationFailureException
+ {
+ localImpl.check();
+ }
}
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/utils/LocalBufferDirs.java b/datamover/source/java/ch/systemsx/cisd/datamover/utils/LocalBufferDirs.java
index 7650ed3f763..2ee5be795de 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/utils/LocalBufferDirs.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/utils/LocalBufferDirs.java
@@ -30,7 +30,7 @@ import ch.systemsx.cisd.common.highwatermark.FileWithHighwaterMark;
*
* @author Tomasz Pylak
*/
-public class LocalBufferDirs
+public final class LocalBufferDirs
{
private final File copyInProgressDir;
diff --git a/datamover/sourceTest/java/ch/systemsx/cisd/datamover/ParametersTest.java b/datamover/sourceTest/java/ch/systemsx/cisd/datamover/ParametersTest.java
index a83de600ad5..18b1b74d730 100644
--- a/datamover/sourceTest/java/ch/systemsx/cisd/datamover/ParametersTest.java
+++ b/datamover/sourceTest/java/ch/systemsx/cisd/datamover/ParametersTest.java
@@ -274,7 +274,7 @@ public class ParametersTest
assertEquals(incomingStoreExpected, incomingStore);
assertEquals(localTempDir, parameters.getBufferDirectoryPath().getFile().getPath());
assertEquals(outgoingStoreExpected, outgoingStore);
- assertEquals(extraCopyDir, parameters.tryGetExtraCopyDir().getFile().getPath());
+ assertEquals(extraCopyDir, parameters.tryGetExtraCopyDir().getPath());
assertEquals(1000 * checkIntervall, parameters.getCheckIntervalMillis());
assertEquals(1000 * quietPeriod, parameters.getQuietPeriodMillis());
assertTrue(incomingStore.isRemote());
diff --git a/datamover/sourceTest/java/ch/systemsx/cisd/datamover/SelfTestTest.java b/datamover/sourceTest/java/ch/systemsx/cisd/datamover/SelfTestTest.java
index 246c795d07e..aade926ab3a 100644
--- a/datamover/sourceTest/java/ch/systemsx/cisd/datamover/SelfTestTest.java
+++ b/datamover/sourceTest/java/ch/systemsx/cisd/datamover/SelfTestTest.java
@@ -24,6 +24,7 @@ import org.testng.annotations.Test;
import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.common.exceptions.NotImplementedException;
import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.logging.LogInitializer;
import ch.systemsx.cisd.common.utilities.FileUtilities;
@@ -60,7 +61,7 @@ public class SelfTestTest
private static final IPathCopier mockCopier = createMockCopier();
- // ////////////////////////////////////////
+ //
// Initialization methods.
//
@@ -86,7 +87,7 @@ public class SelfTestTest
outgoingDirectory.deleteOnExit();
}
- // ////////////////////////////////////////
+ //
// Mocks.
//
@@ -129,11 +130,11 @@ public class SelfTestTest
};
}
- // ////////////////////////////////////////
+ //
// Test cases.
//
- @Test
+ @Test(expectedExceptions = NotImplementedException.class)
public void testHappyCaseWithRemoteHost()
{
final String outgoingHost = "some_remote_host";
--
GitLab