diff --git a/datamover/resource/dependency-structure.ddf b/datamover/resource/dependency-structure.ddf
index 96dfeacc71e6bd1f15fc12888648248582c0d80e..e7f44ccb374f7ac57394539a167c1c783e35d5d9 100644
--- a/datamover/resource/dependency-structure.ddf
+++ b/datamover/resource/dependency-structure.ddf
@@ -8,16 +8,18 @@ check absenceOfPackageCycles > 1 in {package}.*
[fs.common] = ${fs}.common.*
[fs.impl] = ${fs}.impl.*
+[fs.store] = ${fs}.store.*
[fs.intf] = ${fs}.intf.*
[fs.remote] = ${fs}.remote.*
[fs.main] = ${fs}.* excluding ${fs}.*.*
-layer fs.layer0 = [fs.intf]
-layer fs.layer1 = [fs.common]
-layer fs.layer2 = [fs.impl] [fs.remote]
-layer fs.layer3 = [fs.main]
+layer fs.intf = [fs.intf]
+layer fs.common = [fs.common]
+layer fs.store = [fs.store]
+layer fs.impl = [fs.impl] [fs.remote]
+layer fs.main = [fs.main]
-check layeringOf fs.layer0 fs.layer1 fs.layer2 fs.layer3
+check layeringOf fs.intf fs.common fs.store fs.impl fs.main
#-------------------------------
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/DataMover.java b/datamover/source/java/ch/systemsx/cisd/datamover/DataMover.java
index 8f67b6a9cc21009aba7e4c674f7471b2375c172c..57e087fff2df1037b8b140da46a10c8c53ff2e52 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/DataMover.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/DataMover.java
@@ -27,9 +27,12 @@ import ch.systemsx.cisd.common.utilities.ITerminable;
import ch.systemsx.cisd.common.utilities.ITriggerable;
import ch.systemsx.cisd.common.utilities.TimerHelper;
import ch.systemsx.cisd.common.utilities.TriggeringTimerTask;
+import ch.systemsx.cisd.datamover.common.StoreItem;
+import ch.systemsx.cisd.datamover.filesystem.FileStoreFactory;
import ch.systemsx.cisd.datamover.filesystem.RemoteMonitoredMoverFactory;
+import ch.systemsx.cisd.datamover.filesystem.intf.FileStore;
import ch.systemsx.cisd.datamover.filesystem.intf.IFileSysOperationsFactory;
-import ch.systemsx.cisd.datamover.utils.FileStore;
+import ch.systemsx.cisd.datamover.filesystem.intf.IStoreHandler;
import ch.systemsx.cisd.datamover.utils.LocalBufferDirs;
/**
@@ -69,7 +72,7 @@ public class DataMover
private static LocalBufferDirs createLocalBufferDirs(Parameters parameters)
{
- return new LocalBufferDirs(parameters.getBufferStore().getPath(), LOCAL_COPY_IN_PROGRESS_DIR,
+ return new LocalBufferDirs(parameters.getBufferDirectoryPath(), LOCAL_COPY_IN_PROGRESS_DIR,
LOCAL_COPY_COMPLETE_DIR, LOCAL_READY_TO_MOVE_DIR, LOCAL_TEMP_DIR);
}
@@ -138,18 +141,33 @@ public class DataMover
private DataMoverProcess createOutgoingMovingProcess()
{
- final FileStore outgoingStore = parameters.getOutgoingStore();
- final IPathHandler remoteMover = createRemotePathMover(null, outgoingStore.getPath(), outgoingStore.getHost());
+ final FileStore outgoingStore = parameters.getOutgoingStore(factory);
+ final File readyToMoveDir = bufferDirs.getReadyToMoveDir();
+ final FileStore readyToMoveStore = FileStoreFactory.createLocal(readyToMoveDir, "ready-to-move", factory);
+ final IStoreHandler remoteStoreMover = createRemotePathMover(readyToMoveStore, outgoingStore);
+
final DirectoryScanningTimerTask outgoingMovingTask =
- new DirectoryScanningTimerTask(bufferDirs.getReadyToMoveDir(), FileUtilities.ACCEPT_ALL_FILTER,
- remoteMover);
+ new DirectoryScanningTimerTask(readyToMoveDir, FileUtilities.ACCEPT_ALL_FILTER,
+ asPathHandler(remoteStoreMover));
return new DataMoverProcess(outgoingMovingTask, "Final Destination Mover");
}
- private IPathHandler createRemotePathMover(String sourceHost, File destinationDirectory, String destinationHost)
+ // TODO 2007-10-10 Tomasz Pylak: remove this when DirectoryScanningTimerTask will work with IStoreHandler. This is a
+ // quick hack.
+ private static IPathHandler asPathHandler(final IStoreHandler storeHandler)
+ {
+ return new IPathHandler()
+ {
+ public void handle(File path)
+ {
+ storeHandler.handle(new StoreItem(path.getName()));
+ }
+ };
+ }
+
+ private IStoreHandler createRemotePathMover(FileStore source, FileStore destination)
{
- return RemoteMonitoredMoverFactory.create(sourceHost, destinationDirectory, destinationHost, factory,
- parameters);
+ return RemoteMonitoredMoverFactory.create(source, destination, parameters);
}
private static ITerminable createCompoundTerminable(final ITerminable... terminables)
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/IncomingProcessor.java b/datamover/source/java/ch/systemsx/cisd/datamover/IncomingProcessor.java
index 527fdd901d6232453e6636145d10dd6c314d133d..e64049b119a1723978bb81920f9ee0076fd686a4 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/IncomingProcessor.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/IncomingProcessor.java
@@ -29,15 +29,19 @@ import ch.systemsx.cisd.common.logging.Log4jSimpleLogger;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.utilities.DirectoryScanningTimerTask;
+import ch.systemsx.cisd.common.utilities.FileUtilities;
import ch.systemsx.cisd.common.utilities.IPathHandler;
import ch.systemsx.cisd.common.utilities.NamePrefixFileFilter;
import ch.systemsx.cisd.datamover.common.MarkerFile;
+import ch.systemsx.cisd.datamover.common.StoreItem;
+import ch.systemsx.cisd.datamover.filesystem.FileStoreFactory;
import ch.systemsx.cisd.datamover.filesystem.RemoteMonitoredMoverFactory;
+import ch.systemsx.cisd.datamover.filesystem.intf.FileStore;
import ch.systemsx.cisd.datamover.filesystem.intf.IFileSysOperationsFactory;
import ch.systemsx.cisd.datamover.filesystem.intf.IPathMover;
-import ch.systemsx.cisd.datamover.filesystem.intf.IReadPathOperations;
import ch.systemsx.cisd.datamover.filesystem.intf.IRecoverableTimerTaskFactory;
-import ch.systemsx.cisd.datamover.utils.FileStore;
+import ch.systemsx.cisd.datamover.filesystem.intf.IStoreHandler;
+import ch.systemsx.cisd.datamover.filesystem.intf.FileStore.ExtendedFileStore;
import ch.systemsx.cisd.datamover.utils.LocalBufferDirs;
import ch.systemsx.cisd.datamover.utils.QuietPeriodFileFilter;
@@ -60,14 +64,10 @@ public class IncomingProcessor implements IRecoverableTimerTaskFactory
private final IFileSysOperationsFactory factory;
- private final IReadPathOperations incomingReadOperations;
-
private final IPathMover pathMover;
private final LocalBufferDirs bufferDirs;
- private final boolean isIncomingRemote;
-
private final FileStore incomingStore;
private final String prefixForIncoming;
@@ -84,9 +84,7 @@ public class IncomingProcessor implements IRecoverableTimerTaskFactory
{
this.parameters = parameters;
this.prefixForIncoming = parameters.getPrefixForIncoming();
- this.isIncomingRemote = parameters.getTreatIncomingAsRemote();
- this.incomingStore = parameters.getIncomingStore();
- this.incomingReadOperations = factory.getReadPathOperations();
+ this.incomingStore = parameters.getIncomingStore(factory);
this.pathMover = factory.getMover();
this.factory = factory;
this.bufferDirs = bufferDirs;
@@ -99,18 +97,32 @@ public class IncomingProcessor implements IRecoverableTimerTaskFactory
private DataMoverProcess create()
{
- final IPathHandler pathHandler = createIncomingMovingPathHandler(incomingStore.getHost());
+ final IStoreHandler pathHandler = createIncomingMovingPathHandler();
final FileFilter filter = createQuietPeriodFilter();
+ // TODO 2007-10-10 Tomasz Pylak: refactor not to use incomingStore.getPath()
final DirectoryScanningTimerTask movingTask =
- new DirectoryScanningTimerTask(incomingStore.getPath(), filter, pathHandler,
+ new DirectoryScanningTimerTask(incomingStore.getPath(), filter, asPathHandler(pathHandler),
NUMBER_OF_ERRORS_IN_LISTING_IGNORED);
return new DataMoverProcess(movingTask, "Mover of Incoming Data", this);
}
+ // TODO 2007-10-10 Tomasz Pylak: remove this when DirectoryScanningTimerTask will work with IStoreHandler. This is a
+ // quick hack.
+ private static IPathHandler asPathHandler(final IStoreHandler storeHandler)
+ {
+ return new IPathHandler()
+ {
+ public void handle(File path)
+ {
+ storeHandler.handle(new StoreItem(path.getName()));
+ }
+ };
+ }
+
private FileFilter createQuietPeriodFilter()
{
- FileFilter quitePeriodFilter = new QuietPeriodFileFilter(parameters, incomingReadOperations);
+ FileFilter quitePeriodFilter = new QuietPeriodFileFilter(parameters);
FileFilter filterDeletionMarkers = new NamePrefixFileFilter(Constants.DELETION_IN_PROGRESS_PREFIX, false);
FileFilter filter = combineFilters(filterDeletionMarkers, quitePeriodFilter);
return filter;
@@ -127,53 +139,43 @@ public class IncomingProcessor implements IRecoverableTimerTaskFactory
};
}
- private IPathHandler createIncomingMovingPathHandler(final String sourceHostOrNull)
+ private IStoreHandler createIncomingMovingPathHandler()
{
- return new IPathHandler()
+ return new IStoreHandler()
{
- public void handle(File sourceFile)
+ public void handle(StoreItem sourceItem)
{
- if (isIncomingRemote)
+ ExtendedFileStore extendedFileStore = incomingStore.tryAsExtended();
+ if (extendedFileStore == null)
{
- moveFromRemoteIncoming(sourceFile, sourceHostOrNull);
+ moveFromRemoteIncoming(sourceItem);
} else
{
- moveFromLocalIncoming(sourceFile);
+ moveFromLocalIncoming(extendedFileStore, sourceItem);
}
}
};
}
- private void moveFromLocalIncoming(File source)
+ private void moveFromLocalIncoming(ExtendedFileStore sourceStore, StoreItem sourceItem)
{
- final File finalFile = tryMoveLocal(source, bufferDirs.getCopyCompleteDir(), parameters.getPrefixForIncoming());
- if (finalFile == null)
- {
- return;
- }
+ sourceStore.tryMoveLocal(sourceItem, bufferDirs.getCopyCompleteDir(), parameters.getPrefixForIncoming());
}
- private void moveFromRemoteIncoming(File source, String sourceHostOrNull)
+ private void moveFromRemoteIncoming(StoreItem sourceItem)
{
// 1. move from incoming: copy, delete, create copy-finished-marker
final File copyInProgressDir = bufferDirs.getCopyInProgressDir();
- moveFromRemoteToLocal(source, sourceHostOrNull, copyInProgressDir);
- final File destFile = new File(copyInProgressDir, source.getName());
- if (destFile.exists() == false)
+ moveFromRemoteToLocal(sourceItem, incomingStore, copyInProgressDir);
+ final File copiedFile = new File(copyInProgressDir, sourceItem.getName());
+ if (copiedFile.exists() == false)
{
return;
}
- final File copiedFile = new File(copyInProgressDir, source.getName());
- assert copiedFile.exists() : copiedFile.getAbsolutePath();
- final File markerFile = MarkerFile.createCopyFinishedMarker(copiedFile);
- assert markerFile.exists() : markerFile.getAbsolutePath();
// 2. Move to final directory, delete marker
- final File finalFile = tryMoveFromInProgressToFinished(copiedFile, markerFile, bufferDirs.getCopyCompleteDir());
- if (finalFile == null)
- {
- return;
- }
+ final File markerFile = MarkerFile.createCopyFinishedMarker(copiedFile);
+ tryMoveFromInProgressToFinished(copiedFile, markerFile, bufferDirs.getCopyCompleteDir());
}
private File tryMoveFromInProgressToFinished(File copiedFile, File markerFileOrNull, File copyCompleteDir)
@@ -183,7 +185,14 @@ public class IncomingProcessor implements IRecoverableTimerTaskFactory
{
if (markerFileOrNull != null)
{
- markerFileOrNull.delete(); // process even if marker file could not be deleted
+ if (markerFileOrNull.exists() == false)
+ {
+ operationLog.error("Could not find expected copy-finished-mrker file "
+ + markerFileOrNull.getAbsolutePath());
+ } else
+ {
+ markerFileOrNull.delete(); // process even if marker file could not be deleted
+ }
}
return finalFile;
} else
@@ -192,15 +201,15 @@ public class IncomingProcessor implements IRecoverableTimerTaskFactory
}
}
- private void moveFromRemoteToLocal(File source, String sourceHostOrNull, File localDestDir)
+ private void moveFromRemoteToLocal(StoreItem sourceItem, FileStore sourceStore, File localDestDir)
{
- createRemotePathMover(sourceHostOrNull, localDestDir, null).handle(source);
+ createRemotePathMover(sourceStore, FileStoreFactory.createLocal(localDestDir, "local", factory)).handle(
+ sourceItem);
}
- private IPathHandler createRemotePathMover(String sourceHost, File destinationDirectory, String destinationHost)
+ private IStoreHandler createRemotePathMover(FileStore sourceDirectory, FileStore destinationDirectory)
{
- return RemoteMonitoredMoverFactory.create(sourceHost, destinationDirectory, destinationHost, factory,
- parameters);
+ return RemoteMonitoredMoverFactory.create(sourceDirectory, destinationDirectory, parameters);
}
private File tryMoveLocal(File sourceFile, File destinationDir, String prefixTemplate)
@@ -219,7 +228,7 @@ public class IncomingProcessor implements IRecoverableTimerTaskFactory
{
operationLog.debug("Recovery starts.");
}
- if (isIncomingRemote)
+ if (incomingStore.isRemote())
{
recoverIncomingInProgress(bufferDirs.getCopyInProgressDir(), bufferDirs.getCopyCompleteDir());
}
@@ -231,7 +240,7 @@ public class IncomingProcessor implements IRecoverableTimerTaskFactory
private void recoverIncomingInProgress(File copyInProgressDir, File copyCompleteDir)
{
- final File[] files = incomingReadOperations.tryListFiles(copyInProgressDir, errorLog);
+ final File[] files = FileUtilities.tryListFiles(copyInProgressDir, errorLog);
if (files == null || files.length == 0)
{
return; // directory is empty, no recovery is needed
@@ -273,9 +282,7 @@ public class IncomingProcessor implements IRecoverableTimerTaskFactory
} else
// no marker
{
- File incomingDir = incomingStore.getPath();
- File originalInIncoming = new File(incomingDir, localCopy.getName());
- if (incomingReadOperations.exists(originalInIncoming))
+ if (incomingStore.exists(new StoreItem(localCopy.getName())))
{
// partial copy - nothing to do, will be copied again
} else
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/LocalProcessor.java b/datamover/source/java/ch/systemsx/cisd/datamover/LocalProcessor.java
index 7fb81ec43e2f6e6fee9d9ec591a06ea9f1031688..359e2f477e3f941097d80791153805876106065e 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/LocalProcessor.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/LocalProcessor.java
@@ -34,7 +34,6 @@ import ch.systemsx.cisd.common.utilities.RegexFileFilter.PathType;
import ch.systemsx.cisd.datamover.filesystem.intf.IFileSysOperationsFactory;
import ch.systemsx.cisd.datamover.filesystem.intf.IPathImmutableCopier;
import ch.systemsx.cisd.datamover.filesystem.intf.IPathMover;
-import ch.systemsx.cisd.datamover.filesystem.intf.IReadPathOperations;
import ch.systemsx.cisd.datamover.filesystem.intf.IRecoverableTimerTaskFactory;
/**
@@ -57,8 +56,6 @@ public class LocalProcessor implements IPathHandler, IRecoverableTimerTaskFactor
private final IPathMover mover;
- private final IReadPathOperations readOperations;
-
// input: where the data are moved from (for recovery).
private final File inputDir;
@@ -83,7 +80,6 @@ public class LocalProcessor implements IPathHandler, IRecoverableTimerTaskFactor
this.extraCopyDirOrNull = parameters.tryGetExtraCopyDir();
this.copier = factory.getImmutableCopier();
this.mover = factory.getMover();
- this.readOperations = factory.getReadPathOperations();
}
public static final LocalProcessor create(Parameters parameters, File inputDir, File outputDir, File bufferDir,
@@ -123,7 +119,7 @@ public class LocalProcessor implements IPathHandler, IRecoverableTimerTaskFactor
private void recoverTemporaryExtraCopy()
{
- final File[] files = readOperations.tryListFiles(tempDir, errorLog);
+ final File[] files = FileUtilities.tryListFiles(tempDir, errorLog);
if (files == null || files.length == 0)
{
return; // directory is empty, no recovery is needed
@@ -233,7 +229,7 @@ public class LocalProcessor implements IPathHandler, IRecoverableTimerTaskFactor
private boolean doCleansing(File resource)
{
final RegexFileFilter cleansingFilter = new RegexFileFilter();
- final Pattern cleansingRegex = parameters.getCleansingRegex();
+ final Pattern cleansingRegex = parameters.tryGetCleansingRegex();
if (cleansingRegex != null)
{
log(resource, "Doing cleansing");
@@ -252,8 +248,13 @@ public class LocalProcessor implements IPathHandler, IRecoverableTimerTaskFactor
private EFileManipResult doManualIntervention(File resource)
{
+ final File manualInterventionDir = parameters.tryGetManualInterventionDir();
+ if (manualInterventionDir == null)
+ {
+ return EFileManipResult.CONTINUE;
+ }
final RegexFileFilter manualInterventionFilter = new RegexFileFilter();
- final Pattern manualInterventionRegex = parameters.getManualInterventionRegex();
+ final Pattern manualInterventionRegex = parameters.tryGetManualInterventionRegex();
if (manualInterventionRegex != null)
{
manualInterventionFilter.add(PathType.ALL, manualInterventionRegex);
@@ -263,7 +264,6 @@ public class LocalProcessor implements IPathHandler, IRecoverableTimerTaskFactor
if (filtered)
{
log(resource, "Moving to manual intervention directory");
- File manualInterventionDir = parameters.getManualInterventionDirectory();
File movedFile = mover.tryMove(resource, manualInterventionDir);
return (movedFile != null) ? EFileManipResult.STOP : EFileManipResult.FAILURE;
} else
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/Main.java b/datamover/source/java/ch/systemsx/cisd/datamover/Main.java
index a0d5854a8faee93572b8c963df202e88c9ac210d..b19f412d653314aaff8cf72972146d18565bac89 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/Main.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/Main.java
@@ -27,14 +27,15 @@ import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.logging.LogInitializer;
import ch.systemsx.cisd.common.utilities.BuildAndEnvironmentInfo;
import ch.systemsx.cisd.common.utilities.ITerminable;
+import ch.systemsx.cisd.datamover.filesystem.FileStoreFactory;
import ch.systemsx.cisd.datamover.filesystem.FileSysOperationsFactory;
+import ch.systemsx.cisd.datamover.filesystem.intf.FileStore;
import ch.systemsx.cisd.datamover.filesystem.intf.IFileSysOperationsFactory;
import ch.systemsx.cisd.datamover.filesystem.intf.IPathCopier;
-import ch.systemsx.cisd.datamover.utils.FileStore;
import ch.systemsx.cisd.datamover.utils.LocalBufferDirs;
/**
- * The main class of the datamover.
+ * The main class of the Datamover.
*
* @author Bernd Rinn
*/
@@ -54,16 +55,16 @@ public class Main
};
private static final Runnable loggingShutdownHook = new Runnable()
- {
- public void run()
{
- if (operationLog.isInfoEnabled())
+ public void run()
{
- operationLog.info("Datamover is shutting down.");
+ if (operationLog.isInfoEnabled())
+ {
+ operationLog.info("Datamover is shutting down.");
+ }
}
- }
- };
-
+ };
+
private static void initLog()
{
LogInitializer.init();
@@ -91,17 +92,28 @@ public class Main
{
try
{
- IPathCopier copyProcess = new FileSysOperationsFactory(parameters).getCopierNoDeletionRequired();
ArrayList<FileStore> stores = new ArrayList<FileStore>();
- stores.add(parameters.getIncomingStore());
- stores.add(parameters.getBufferStore());
- stores.add(parameters.getOutgoingStore());
- stores.add(parameters.getManualInterventionStore());
+ FileSysOperationsFactory factory = new FileSysOperationsFactory(parameters);
+ stores.add(parameters.getIncomingStore(factory));
+ FileStore buferStore =
+ FileStoreFactory.createLocal(parameters.getBufferDirectoryPath(), Parameters.BUFFER_KIND_DESC,
+ factory);
+ stores.add(buferStore);
+ stores.add(parameters.getOutgoingStore(factory));
+ if (parameters.tryGetManualInterventionDir() != null)
+ {
+ FileStore dummyStore =
+ FileStoreFactory.createLocal(parameters.tryGetManualInterventionDir(), "manual intervention",
+ factory);
+ stores.add(dummyStore);
+ }
if (parameters.tryGetExtraCopyDir() != null)
{
- FileStore dummyStore = new FileStore(parameters.tryGetExtraCopyDir(), "extra-copy", null, false);
+ FileStore dummyStore =
+ FileStoreFactory.createLocal(parameters.tryGetExtraCopyDir(), "extra-copy", factory);
stores.add(dummyStore);
}
+ IPathCopier copyProcess = factory.getCopier(false);
SelfTest.check(copyProcess, stores.toArray(new FileStore[] {}));
} catch (HighLevelException e)
{
@@ -135,7 +147,7 @@ public class Main
printInitialLogMessage(parameters);
selfTest(parameters);
startupServer(parameters);
- operationLog.info("datamover ready and waiting for data.");
+ operationLog.info("Datamover ready and waiting for data.");
}
}
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/Parameters.java b/datamover/source/java/ch/systemsx/cisd/datamover/Parameters.java
index 2ad120462c738810e404a828c0c09eb46eb14d1f..bb54617d265ca18011aa01d4bc3df32a9800477e 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/Parameters.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/Parameters.java
@@ -38,9 +38,11 @@ import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.utilities.BuildAndEnvironmentInfo;
import ch.systemsx.cisd.common.utilities.IExitHandler;
import ch.systemsx.cisd.common.utilities.SystemExit;
+import ch.systemsx.cisd.datamover.filesystem.FileStoreFactory;
+import ch.systemsx.cisd.datamover.filesystem.intf.FileStore;
+import ch.systemsx.cisd.datamover.filesystem.intf.IFileSysOperationsFactory;
import ch.systemsx.cisd.datamover.intf.IFileSysParameters;
import ch.systemsx.cisd.datamover.intf.ITimingParameters;
-import ch.systemsx.cisd.datamover.utils.FileStore;
/**
* The class to process the command line parameters.
@@ -49,7 +51,6 @@ import ch.systemsx.cisd.datamover.utils.FileStore;
*/
public class Parameters implements ITimingParameters, IFileSysParameters
{
-
private static final String SERVICE_PROPERTIES_FILE = "etc/service.properties";
private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION, Parameters.class);
@@ -197,7 +198,7 @@ public class Parameters implements ITimingParameters, IFileSysParameters
*/
@Option(longName = "manual-intervention-dir", metaVar = "DIR", usage = "The local directory to "
+ "store paths that need manual intervention.")
- private File manualInterventionDirectory = null;
+ private File manualInterventionDirectoryOrNull = null;
/**
* The directory on the remote side to move the paths to from the buffer directory.
@@ -227,26 +228,6 @@ public class Parameters implements ITimingParameters, IFileSysParameters
+ "moving to outgoing.")
private Pattern cleansingRegex = null;
- /**
- * The store where the files come in.
- */
- private final FileStore incomingStore;
-
- /**
- * The store to buffer the files before copying to outgoing.
- */
- private final FileStore bufferStore;
-
- /**
- * The store to copy the files to.
- */
- private final FileStore outgoingStore;
-
- /**
- * The store to copy files to that need manual intervention.
- */
- private final FileStore manualInterventionStore;
-
/**
* The regular expression to use for deciding whether a path in the incoming directory needs manual intervention.
*/
@@ -316,6 +297,14 @@ public class Parameters implements ITimingParameters, IFileSysParameters
}
+ static final String INCOMING_KIND_DESC = "incoming";
+
+ static final String MANUAL_INTERVENTION_KIND_DESC = "manual intervention";
+
+ static final String BUFFER_KIND_DESC = "buffer";
+
+ static final String OUTGOING_KIND_DESC = "outgoing";
+
Parameters(String[] args)
{
this(args, SystemExit.SYSTEM_EXIT);
@@ -339,15 +328,11 @@ public class Parameters implements ITimingParameters, IFileSysParameters
{
throw new ConfigurationFailureException("No 'outgoing-dir' defined.");
}
- if (manualInterventionDirectory == null && manualInterventionRegex != null)
+ if (manualInterventionDirectoryOrNull == null && manualInterventionRegex != null)
{
throw new ConfigurationFailureException(
"No 'manual-intervention-dir' defined, but 'manual-intervention-regex'.");
}
- incomingStore = new FileStore(incomingDirectory, "incoming", incomingHost, treatIncomingAsRemote);
- bufferStore = new FileStore(bufferDirectory, "buffer", null, false);
- manualInterventionStore = new FileStore(manualInterventionDirectory, "manual intervention", null, false);
- outgoingStore = new FileStore(outgoingDirectory, "outgoing", outgoingHost, true);
} catch (Exception ex)
{
outputException(ex);
@@ -357,6 +342,25 @@ public class Parameters implements ITimingParameters, IFileSysParameters
}
}
+ private static FileStore createStore(File directory, String kind, String hostOrNull, boolean isRemote,
+ IFileSysOperationsFactory factory)
+ {
+ if (hostOrNull != null)
+ {
+ assert isRemote == true;
+ return FileStoreFactory.createRemoteHost(directory, hostOrNull, kind, factory);
+ } else
+ {
+ if (isRemote)
+ {
+ return FileStoreFactory.createRemoteShare(directory, kind, factory);
+ } else
+ {
+ return FileStoreFactory.createLocal(directory, kind, factory);
+ }
+ }
+ }
+
private void outputException(Exception ex)
{
if (ex instanceof HighLevelException || ex instanceof CmdLineException)
@@ -412,7 +416,7 @@ public class Parameters implements ITimingParameters, IFileSysParameters
}
if (serviceProperties.getProperty("manual-intervention-dir") != null)
{
- manualInterventionDirectory = new File(serviceProperties.getProperty("manual-intervention-dir"));
+ manualInterventionDirectoryOrNull = new File(serviceProperties.getProperty("manual-intervention-dir"));
}
if (serviceProperties.getProperty("outgoing-dir") != null)
{
@@ -548,59 +552,41 @@ public class Parameters implements ITimingParameters, IFileSysParameters
/**
* @return The store to monitor for new files and directories to move to the buffer.
*/
- public FileStore getIncomingStore()
+ public FileStore getIncomingStore(IFileSysOperationsFactory factory)
{
- return incomingStore;
- }
-
- /**
- * @return true if directory with incoming data is supposed to be on a remote share. It implies that a special care
- * will be taken when coping is performed from that directory.
- */
- public boolean getTreatIncomingAsRemote()
- {
- return treatIncomingAsRemote;
+ return createStore(incomingDirectory, INCOMING_KIND_DESC, incomingHost, treatIncomingAsRemote, factory);
}
/**
* @return The directory for local files and directories manipulations.
*/
- public FileStore getBufferStore()
+ public File getBufferDirectoryPath()
{
- return bufferStore;
+ return bufferDirectory;
}
/**
* @return The store to copy the data to.
*/
- public FileStore getOutgoingStore()
+ public FileStore getOutgoingStore(IFileSysOperationsFactory factory)
{
- return outgoingStore;
+ return createStore(outgoingDirectory, OUTGOING_KIND_DESC, outgoingHost, true, factory);
}
/**
* @return The directory to move files and directories to that have been quiet in the local data directory for long
* enough and that need manual intervention. Note that this directory needs to be on the same file system as
- * {@link #getBufferStore}.
- */
- public File getManualInterventionDirectory()
- {
- return manualInterventionDirectory;
- }
-
- /**
- * @return The store to move files and directories to that have been quiet in the local data directory for long
- * enough and that need manual intervention. Note that this directory needs to be on the same file system as
- * {@link #getBufferStore}.
+ * {@link #getBufferDirectoryPath}.
*/
- public FileStore getManualInterventionStore()
+ public File tryGetManualInterventionDir()
{
- return manualInterventionStore;
+ return manualInterventionDirectoryOrNull;
}
/**
* @return The directory where we create an additional copy of incoming data or <code>null</code> if it is not
- * specified. Note that this directory needs to be on the same file system as {@link #getBufferStore}.
+ * specified. Note that this directory needs to be on the same file system as
+ * {@link #getBufferDirectoryPath}.
*/
public File tryGetExtraCopyDir()
{
@@ -611,7 +597,7 @@ public class Parameters implements ITimingParameters, IFileSysParameters
* @return The regular expression to use for cleansing on the incoming directory before moving it to the buffer or
* <code>null</code>, if no regular expression for cleansing has been provided.
*/
- public Pattern getCleansingRegex()
+ public Pattern tryGetCleansingRegex()
{
return cleansingRegex;
}
@@ -621,7 +607,7 @@ public class Parameters implements ITimingParameters, IFileSysParameters
* intervention or <code>null</code>, if no regular expression for manual interventionpaths has been
* provided.
*/
- public Pattern getManualInterventionRegex()
+ public Pattern tryGetManualInterventionRegex()
{
return manualInterventionRegex;
}
@@ -654,9 +640,9 @@ public class Parameters implements ITimingParameters, IFileSysParameters
{
operationLog.info(String.format("Outgoing host: '%s'.", outgoingHost));
}
- if (null != getManualInterventionDirectory())
+ if (null != tryGetManualInterventionDir())
{
- operationLog.info(String.format("Manual interventions directory: '%s'.", manualInterventionDirectory
+ operationLog.info(String.format("Manual interventions directory: '%s'.", tryGetManualInterventionDir()
.getAbsolutePath()));
}
if (null != extraCopyDirectory)
@@ -671,16 +657,16 @@ public class Parameters implements ITimingParameters, IFileSysParameters
operationLog.info(String.format("Intervall to wait after failure: %d s.",
getIntervalToWaitAfterFailure() / 1000));
operationLog.info(String.format("Maximum number of retries: %d.", getMaximalNumberOfRetries()));
- if (getCleansingRegex() != null)
+ if (tryGetCleansingRegex() != null)
{
operationLog.info(String.format("Regular expression used for cleansing before moving: '%s'",
- getCleansingRegex().pattern()));
+ tryGetCleansingRegex().pattern()));
}
- if (getManualInterventionRegex() != null)
+ if (tryGetManualInterventionRegex() != null)
{
operationLog.info(String.format(
"Regular expression used for deciding whether a path needs manual intervention: '%s'",
- getManualInterventionRegex().pattern()));
+ tryGetManualInterventionRegex().pattern()));
}
}
}
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/SelfTest.java b/datamover/source/java/ch/systemsx/cisd/datamover/SelfTest.java
index 2144ec374dcdc722e60130969255790cd56fa5a7..3159930379b21373499cea42f7a11b4454939c89 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/SelfTest.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/SelfTest.java
@@ -16,7 +16,6 @@
package ch.systemsx.cisd.datamover;
-import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
@@ -25,9 +24,8 @@ import ch.systemsx.cisd.common.exceptions.HighLevelException;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.logging.LogInitializer;
-import ch.systemsx.cisd.common.utilities.FileUtilities;
+import ch.systemsx.cisd.datamover.filesystem.intf.FileStore;
import ch.systemsx.cisd.datamover.filesystem.intf.IPathCopier;
-import ch.systemsx.cisd.datamover.utils.FileStore;
/**
* A class that can perform a self test of the data mover.
@@ -43,23 +41,17 @@ public class SelfTest
LogInitializer.init();
}
- private static void checkPathRecords(FileStore[] pathRecords, IPathCopier copier)
+ private static void checkPathRecords(FileStore[] pathRecords)
{
assert pathRecords != null;
checkPathRecordsContainEachOther(pathRecords);
for (FileStore pathRecord : pathRecords)
{
- if (pathRecord.getPath() == null)
+ String errorMessage = pathRecord.tryCheckDirectoryFullyAccessible();
+ if (errorMessage != null)
{
- continue;
- }
- if (pathRecord.getHost() == null)
- {
- checkDirectoryOnLocalHost(pathRecord);
- } else
- {
- checkDirectoryOnRemoteHost(pathRecord, copier);
+ throw new ConfigurationFailureException(errorMessage);
}
}
}
@@ -70,52 +62,15 @@ public class SelfTest
{
for (int j = 0; j < i; ++j)
{
- if (StringUtils.equals(store[i].getHost(), store[j].getHost())
- && containOneAnother(store[i].getCanonicalPath(), store[j].getCanonicalPath()))
+ if (store[i].isParentDirectory(store[j]) || store[j].isParentDirectory(store[i]))
{
throw ConfigurationFailureException.fromTemplate("Directory '%s' and '%s' contain each other",
- store[i].getCanonicalPath(), store[j].getCanonicalPath());
+ store[i], store[j]);
}
}
}
}
- private static void checkDirectoryOnRemoteHost(FileStore pathRecord, IPathCopier copier)
- throws ConfigurationFailureException
- {
- if (false == copier.exists(pathRecord.getPath(), pathRecord.getHost()))
-
- {
- throw ConfigurationFailureException.fromTemplate("Cannot access %s directory '%s' on host '%s'", pathRecord
- .getKind(), pathRecord.getCanonicalPath(), pathRecord.getHost());
- }
- }
-
- private static void checkDirectoryOnLocalHost(FileStore pathRecord)
- {
- String errorMessage = FileUtilities.checkDirectoryFullyAccessible(pathRecord.getPath(), pathRecord.getKind());
- if (errorMessage != null)
- {
- throw new ConfigurationFailureException(errorMessage);
- }
-
- }
-
- private static boolean containOneAnother(String directory1, String directory2)
- {
- if (directory1 == null || directory2 == null)
- {
- return false;
- }
- if (directory1.length() < directory2.length())
- {
- return directory2.startsWith(directory1);
- } else
- {
- return directory1.startsWith(directory2);
- }
- }
-
/**
* Will perform all checks of the self-test. If the method returns without exception, the self-test can be
* considered "past", otherwise the exception will have more information on what went wrong. This method performs
@@ -126,7 +81,7 @@ public class SelfTest
try
{
copier.check();
- checkPathRecords(stores, copier);
+ checkPathRecords(stores);
if (operationLog.isInfoEnabled())
{
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/common/MarkerFile.java b/datamover/source/java/ch/systemsx/cisd/datamover/common/MarkerFile.java
index f934a77b143a244e3f16f179c5ef26e6238f4ba7..6579c609223978d752a1ff41fae98d775dad5a19 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/common/MarkerFile.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/common/MarkerFile.java
@@ -34,9 +34,9 @@ public class MarkerFile
return Constants.IS_FINISHED_PREFIX + originalFileName;
}
- public static File createDeletionInProgressMarker(File parent, String originalFileName)
+ public static StoreItem createDeletionInProgressMarker(StoreItem originalItem)
{
- return new File(parent, getDeletionInProgressMarkerName(originalFileName));
+ return new StoreItem(getDeletionInProgressMarkerName(originalItem.getName()));
}
private static String getDeletionInProgressMarkerName(String originalFileName)
@@ -69,4 +69,14 @@ public class MarkerFile
{
return new File(parent, getCopyFinishedMarkerName(originalFileName));
}
+
+ public static StoreItem createCopyFinishedMarker(StoreItem originalItem)
+ {
+ return new StoreItem(getCopyFinishedMarkerName(originalItem.getName()));
+ }
+
+ public static StoreItem createRequiresDeletionBeforeCreationMarker()
+ {
+ return new StoreItem(".requiresDeletionBeforeCreation");
+ }
}
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/common/StoreItem.java b/datamover/source/java/ch/systemsx/cisd/datamover/common/StoreItem.java
new file mode 100644
index 0000000000000000000000000000000000000000..160420d17e7d1114a13d4509094787f7a5f47d0d
--- /dev/null
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/common/StoreItem.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.datamover.common;
+
+/**
+ * @author Tomasz Pylak on Oct 8, 2007
+ */
+public class StoreItem
+{
+ private final String name;
+
+ public StoreItem(String name)
+ {
+ this.name = name;
+ }
+
+ /** Should not be used for logging. Use toString() instead. */
+ public String getName()
+ {
+ return name;
+ }
+
+ @Override
+ public String toString()
+ {
+ return name;
+ }
+}
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/FileStoreFactory.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/FileStoreFactory.java
new file mode 100644
index 0000000000000000000000000000000000000000..1e608bb546523a2ba7260a4052186770b5eed105
--- /dev/null
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/FileStoreFactory.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.datamover.filesystem;
+
+import java.io.File;
+
+import ch.systemsx.cisd.datamover.filesystem.intf.FileStore;
+import ch.systemsx.cisd.datamover.filesystem.intf.IFileSysOperationsFactory;
+import ch.systemsx.cisd.datamover.filesystem.store.FileStoreLocal;
+import ch.systemsx.cisd.datamover.filesystem.store.FileStoreRemote;
+import ch.systemsx.cisd.datamover.filesystem.store.FileStoreRemoteMounted;
+
+/**
+ * @author Tomasz Pylak on Oct 11, 2007
+ */
+public class FileStoreFactory
+{
+ /** use when file store is on a local host */
+ public static final FileStore createLocal(File path, String kind, IFileSysOperationsFactory factory)
+ {
+ return new FileStoreLocal(path, kind, factory);
+ }
+
+ /** use when file store is on a remote share mounted on local host */
+ public static final FileStore createRemoteShare(File path, String kind, IFileSysOperationsFactory factory)
+ {
+ return new FileStoreRemoteMounted(path, kind, factory);
+ }
+
+ /**
+ * use when file store is on a remote share mounted on local host
+ *
+ * @param factory
+ */
+ public static final FileStore createRemoteHost(File path, String host, String kind,
+ IFileSysOperationsFactory factory)
+ {
+ return new FileStoreRemote(path, host, kind, factory);
+ }
+}
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/FileSysOperationsFactory.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/FileSysOperationsFactory.java
index cd51c56a8501d7f6cfc8f4361296d17f8e98b8f8..a4cb8c7bbc73ef2b2a6451b50f791b50e7da29fc 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/FileSysOperationsFactory.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/FileSysOperationsFactory.java
@@ -17,18 +17,14 @@
package ch.systemsx.cisd.datamover.filesystem;
import java.io.File;
-import java.io.FileFilter;
-import java.io.IOException;
import org.apache.log4j.Logger;
import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.exceptions.StatusFlag;
-import ch.systemsx.cisd.common.logging.ISimpleLogger;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
-import ch.systemsx.cisd.common.utilities.FileUtilities;
import ch.systemsx.cisd.common.utilities.OSUtilities;
import ch.systemsx.cisd.datamover.filesystem.impl.RecursiveHardLinkMaker;
import ch.systemsx.cisd.datamover.filesystem.intf.IFileSysOperationsFactory;
@@ -36,7 +32,6 @@ import ch.systemsx.cisd.datamover.filesystem.intf.IPathCopier;
import ch.systemsx.cisd.datamover.filesystem.intf.IPathImmutableCopier;
import ch.systemsx.cisd.datamover.filesystem.intf.IPathMover;
import ch.systemsx.cisd.datamover.filesystem.intf.IPathRemover;
-import ch.systemsx.cisd.datamover.filesystem.intf.IReadPathOperations;
import ch.systemsx.cisd.datamover.filesystem.remote.rsync.RsyncCopier;
import ch.systemsx.cisd.datamover.intf.IFileSysParameters;
@@ -54,8 +49,6 @@ public class FileSysOperationsFactory implements IFileSysOperationsFactory
private static final Logger notificationLog =
LogFactory.getLogger(LogCategory.NOTIFY, FileSysOperationsFactory.class);
- private static final Logger machineLog = LogFactory.getLogger(LogCategory.MACHINE, FileSysOperationsFactory.class);
-
final private IFileSysParameters parameters;
public FileSysOperationsFactory(IFileSysParameters parameters)
@@ -70,33 +63,6 @@ public class FileSysOperationsFactory implements IFileSysOperationsFactory
return new RetryingPathRemover(MAX_RETRIES_ON_FAILURE, MILLIS_TO_SLEEP_ON_FAILURE);
}
- public IReadPathOperations getReadPathOperations()
- {
- return new IReadPathOperations()
- {
-
- public boolean exists(File file)
- {
- return file.exists();
- }
-
- public long lastChanged(File path)
- {
- return FileUtilities.lastChanged(path);
- }
-
- public File[] tryListFiles(File directory, FileFilter filter, ISimpleLogger loggerOrNull)
- {
- return FileUtilities.tryListFiles(directory, filter, loggerOrNull);
- }
-
- public File[] tryListFiles(File directory, ISimpleLogger loggerOrNull)
- {
- return FileUtilities.tryListFiles(directory, FileUtilities.ACCEPT_ALL_FILTER, loggerOrNull);
- }
- };
- }
-
public IPathImmutableCopier getImmutableCopier()
{
String lnExec = parameters.getHardLinkExecutable();
@@ -119,7 +85,7 @@ public class FileSysOperationsFactory implements IFileSysOperationsFactory
private IPathImmutableCopier createFakedImmCopier()
{
- final IPathCopier normalCopier = suggestPathCopier(false);
+ final IPathCopier normalCopier = getCopier(false);
return new IPathImmutableCopier()
{
public File tryCopy(File file, File destinationDirectory)
@@ -138,7 +104,7 @@ public class FileSysOperationsFactory implements IFileSysOperationsFactory
};
}
- private IPathCopier suggestPathCopier(boolean requiresDeletionBeforeCreation)
+ public IPathCopier getCopier(boolean requiresDeletionBeforeCreation)
{
final File rsyncExecutable = findRsyncExecutable(parameters.getRsyncExecutable());
final File sshExecutable = findSshExecutable(parameters.getSshExecutable());
@@ -198,76 +164,6 @@ public class FileSysOperationsFactory implements IFileSysOperationsFactory
return sshExecutable;
}
- /**
- * @return <code>true</code> if the <var>copyProcess</var> on the file system where the <var>destinationDirectory</var>
- * resides requires deleting an existing file before it can be overwritten.
- */
- private static boolean requiresDeletionBeforeCreation(IPathCopier copyProcess, File destinationDirectory)
- {
- assert copyProcess != null;
- assert destinationDirectory != null;
- assert destinationDirectory.isDirectory();
-
- String fileName = ".requiresDeletionBeforeCreation";
- final File destinationFile = new File(destinationDirectory, fileName);
- final File tmpSourceDir = new File(destinationDirectory, ".DataMover-OverrideTest");
- final File sourceFile = new File(tmpSourceDir, fileName);
- try
- {
- tmpSourceDir.mkdir();
- sourceFile.createNewFile();
- destinationFile.createNewFile();
- // If we have e.g. a Cellera NAS server, the next call will raise an IOException.
- final boolean OK = Status.OK.equals(copyProcess.copy(sourceFile, destinationDirectory));
- if (machineLog.isInfoEnabled())
- {
- if (OK)
- {
- machineLog.info(String.format("Copier %s on directory '%s' works with overwriting existing files.",
- copyProcess.getClass().getSimpleName(), destinationDirectory.getAbsolutePath()));
- } else
- {
- machineLog.info(String.format(
- "Copier %s on directory '%s' requires deletion before creation of existing files.",
- copyProcess.getClass().getSimpleName(), destinationDirectory.getAbsolutePath()));
- }
- }
- return (OK == false);
- } catch (IOException e)
- {
- if (machineLog.isInfoEnabled())
- {
- machineLog.info(String.format(
- "The file system on '%s' requires deletion before creation of existing files.",
- destinationDirectory.getAbsolutePath()));
- }
- return true;
- } finally
- {
- // We don't check for success because there is nothing we can do if we fail.
- sourceFile.delete();
- tmpSourceDir.delete();
- destinationFile.delete();
- }
- }
-
- public IPathCopier getCopierNoDeletionRequired()
- {
- return suggestPathCopier(false);
- }
-
- public IPathCopier getCopier(File destinationDirectory)
- {
- IPathCopier copyProcess = suggestPathCopier(false);
- if (requiresDeletionBeforeCreation(copyProcess, destinationDirectory))
- {
- return suggestPathCopier(true);
- } else
- {
- return copyProcess;
- }
- }
-
public IPathMover getMover()
{
return new RetryingPathMover(MAX_RETRIES_ON_FAILURE, MILLIS_TO_SLEEP_ON_FAILURE);
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/RemoteMonitoredMoverFactory.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/RemoteMonitoredMoverFactory.java
index b6e16c22ca6330d3f8535c59b31d130046cac7ca..a221dbd60f15ad123e914ed629870988f62b1f2e 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/RemoteMonitoredMoverFactory.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/RemoteMonitoredMoverFactory.java
@@ -16,12 +16,9 @@
package ch.systemsx.cisd.datamover.filesystem;
-import java.io.File;
-
-import ch.systemsx.cisd.common.utilities.IPathHandler;
-import ch.systemsx.cisd.datamover.filesystem.intf.IFileSysOperationsFactory;
-import ch.systemsx.cisd.datamover.filesystem.intf.IPathCopier;
-import ch.systemsx.cisd.datamover.filesystem.intf.IPathRemover;
+import ch.systemsx.cisd.datamover.filesystem.intf.FileStore;
+import ch.systemsx.cisd.datamover.filesystem.intf.IStoreCopier;
+import ch.systemsx.cisd.datamover.filesystem.intf.IStoreHandler;
import ch.systemsx.cisd.datamover.filesystem.remote.CopyActivityMonitor;
import ch.systemsx.cisd.datamover.filesystem.remote.RemotePathMover;
import ch.systemsx.cisd.datamover.intf.ITimingParameters;
@@ -32,24 +29,17 @@ import ch.systemsx.cisd.datamover.intf.ITimingParameters;
public class RemoteMonitoredMoverFactory
{
/**
- * Creates a handler to move files remotely and monitor the progress
+ * Creates a handler to move files remotely from source to destination and monitor the progress
*
- * @param sourceHost The host to move paths from, or <code>null</code>, if data will be moved from the local file
- * system
+ * @param sourceDirectory The directory to move paths from
* @param destinationDirectory The directory to move paths to.
- * @param destinationHost The host to move paths to, or <code>null</code>, if <var>destinationDirectory</var> is
- * a remote share.
- * @param fsFactory operations on (remote) file system
* @param parameters The timing parameters used for monitoring and reporting stall situations.
*/
- public static final IPathHandler create(String sourceHost, File destinationDirectory, String destinationHost,
- IFileSysOperationsFactory fsFactory, ITimingParameters parameters)
+ public static final IStoreHandler create(FileStore sourceDirectory, FileStore destinationDirectory,
+ ITimingParameters parameters)
{
- final IPathCopier copier = fsFactory.getCopier(destinationDirectory);
- final CopyActivityMonitor monitor =
- new CopyActivityMonitor(destinationDirectory, fsFactory.getReadPathOperations(), copier, parameters);
- final IPathRemover remover = fsFactory.getRemover();
- return new RemotePathMover(destinationDirectory, destinationHost, monitor, remover, copier, sourceHost,
- parameters);
+ final IStoreCopier copier = sourceDirectory.getCopier(destinationDirectory);
+ final CopyActivityMonitor monitor = new CopyActivityMonitor(destinationDirectory, copier, parameters);
+ return new RemotePathMover(sourceDirectory, destinationDirectory, copier, monitor, parameters);
}
}
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/RetryingPathMover.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/RetryingPathMover.java
index 01eb0d11254deb590e534a5bea257caf13bdfe8b..edcc3d6f6499f5227501ed6fdb6260e267c217be 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/RetryingPathMover.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/RetryingPathMover.java
@@ -37,9 +37,9 @@ class RetryingPathMover implements IPathMover
private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION, RetryingPathMover.class);
private static final Logger notificationLog = LogFactory.getLogger(LogCategory.NOTIFY, RetryingPathMover.class);
-
+
private final int maxRetriesOnFailure;
-
+
private final long millisToSleepOnFailure;
RetryingPathMover(int maxRetriesOnFailure, long millisToSleepOnFailure)
@@ -47,7 +47,7 @@ class RetryingPathMover implements IPathMover
this.maxRetriesOnFailure = maxRetriesOnFailure;
this.millisToSleepOnFailure = millisToSleepOnFailure;
}
-
+
public File tryMove(File sourceFile, File destinationDir)
{
return tryMove(sourceFile, destinationDir, "");
@@ -69,13 +69,14 @@ class RetryingPathMover implements IPathMover
File destFile = new File(destinationPath);
int failures = 0;
boolean movedOK = false;
- while(true)
+ while (true)
{
movedOK = sourcePath.renameTo(destFile);
if (movedOK)
{
break;
- } else {
+ } else
+ {
if (sourcePath.exists() == false)
{
operationLog.error(String.format("Path '%s' doesn't exist, so it can't be moved to '%s'.",
@@ -138,5 +139,4 @@ class RetryingPathMover implements IPathMover
return StringUtils.replace(prefixTemplate, "%t", DateFormatUtils.format(System.currentTimeMillis(),
"yyyyMMddHHmmss"));
}
-
}
\ No newline at end of file
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/RetryingPathRemover.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/RetryingPathRemover.java
index b90373164e2056d09767c1353a37a9394bcff480..584f5a83a02afa31426a2f97dc11cbd310e0aded 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/RetryingPathRemover.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/RetryingPathRemover.java
@@ -46,13 +46,14 @@ final class RetryingPathRemover implements IPathRemover
}
int failures = 0;
boolean deletionOK = false;
- while(true)
+ while (true)
{
deletionOK = FileUtilities.deleteRecursively(path);
if (deletionOK)
{
break;
- } else {
+ } else
+ {
if (path.exists() == false)
{
operationLog.warn(String.format("Path '%s' doesn't exist, so it can't be removed.", path));
@@ -73,14 +74,14 @@ final class RetryingPathRemover implements IPathRemover
}
}
}
-
+
if (deletionOK == false)
{
notificationLog.error(String.format("Removing path '%s' failed, giving up.", path));
return STATUS_FAILED_DELETION;
} else
{
- return Status.OK;
+ return Status.OK;
}
}
}
\ No newline at end of file
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/FileStore.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/FileStore.java
new file mode 100644
index 0000000000000000000000000000000000000000..34cde46930de3db12c2260723b4acdf37193021e
--- /dev/null
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/FileStore.java
@@ -0,0 +1,206 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.datamover.filesystem.intf;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.commons.lang.StringUtils;
+
+import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.common.exceptions.Status;
+import ch.systemsx.cisd.datamover.common.StoreItem;
+
+/**
+ * A class to holds the information about a file store.
+ *
+ * @author Bernd Rinn
+ * @author Tomasz Pylak
+ */
+public abstract class FileStore
+{
+ protected final File path;
+
+ protected final String hostOrNull;
+
+ protected final String kind;
+
+ protected final boolean remote;
+
+ protected final IFileSysOperationsFactory factory;
+
+ protected FileStore(File path, String hostOrNull, boolean remote, String kind, IFileSysOperationsFactory factory)
+ {
+ assert path != null;
+ assert kind != null;
+ this.path = path;
+ this.kind = kind;
+ this.hostOrNull = hostOrNull;
+ this.remote = remote;
+ this.factory = factory;
+ }
+
+ // TODO 2007-10-10 Tomasz Pylak: change visibility to protected after changing DirectoryScanningTimerTask
+ public final File getPath()
+ {
+ return path;
+ }
+
+ protected final String tryGetHost()
+ {
+ return hostOrNull;
+ }
+
+ protected final String getDescription()
+ {
+ return kind;
+ }
+
+ protected final File getChildFile(StoreItem item)
+ {
+ return new File(path, item.getName());
+ }
+
+ // does not take into account the fact, that the destination cannot be overwritten and must be deleted beforehand
+ protected final IStoreCopier constructStoreCopier(FileStore destinationDirectory,
+ boolean requiresDeletionBeforeCreation)
+ {
+ final IPathCopier copier = factory.getCopier(requiresDeletionBeforeCreation);
+ final String srcHostOrNull = hostOrNull;
+ final String destHostOrNull = destinationDirectory.hostOrNull;
+ final File destPath = destinationDirectory.path;
+ return new IStoreCopier()
+ {
+ public Status copy(StoreItem item)
+ {
+ File srcItem = getChildFile(item);
+ if (srcHostOrNull == null)
+ {
+ if (destHostOrNull == null)
+ {
+ return copier.copy(srcItem, destPath);
+ } else
+ {
+ return copier.copyToRemote(srcItem, destPath, destHostOrNull);
+ }
+ } else
+ {
+ assert destHostOrNull == null;
+ return copier.copyFromRemote(srcItem, srcHostOrNull, destPath);
+ }
+ }
+
+ public boolean terminate()
+ {
+ return copier.terminate();
+ }
+ };
+ }
+
+ /**
+ * Returns <code>true</code>, if the file store resides on a remote computer and <code>false</code> otherwise.
+ * <p>
+ * Note that the resource does not have to be on a remote host. Instead it can reside on a remote share mounted on
+ * local host via NFS or CIFS.
+ */
+ public final boolean isRemote()
+ {
+ return remote;
+ }
+
+ public boolean isParentDirectory(FileStore child)
+ {
+ return StringUtils.equals(hostOrNull, child.hostOrNull)
+ && getCanonicalPath(child.path).startsWith(getCanonicalPath(path));
+ }
+
+ private String getCanonicalPath(File file)
+ {
+ if (hostOrNull != null)
+ {
+ return file.getPath();
+ }
+ try
+ {
+ return file.getCanonicalPath() + File.separator;
+ } catch (IOException e)
+ {
+ throw EnvironmentFailureException.fromTemplate(e, "Cannot determine canonical form of path '%s'", file
+ .getPath());
+ }
+ }
+
+ @Override
+ public boolean equals(Object obj)
+ {
+ if (obj instanceof FileStore)
+ {
+ FileStore fileStore = (FileStore) obj;
+ boolean sameHost =
+ (hostOrNull == null ? fileStore.hostOrNull == null : fileStore.hostOrNull != null
+ && hostOrNull.equals(fileStore.hostOrNull));
+ return sameHost && kind.equals(fileStore.kind) && path.equals(fileStore.path);
+ } else
+ {
+ return false;
+ }
+ }
+
+ // -------------------
+
+ /**
+ * Checks whether this store is a directory and is fully accessible to the program.
+ *
+ * @return <code>null</code> if the <var>directory</var> is fully accessible and an error message describing the
+ * problem with the <var>directory</var> otherwise.
+ */
+ public abstract String tryCheckDirectoryFullyAccessible();
+
+ public abstract boolean exists(StoreItem item);
+
+ /**
+ * Returns the last time when there was a write access to <var>resource</var>.
+ *
+ * @return The time (in milliseconds since the start of the epoch) when <var>resource</var> was last changed.
+ */
+ public abstract long lastChanged(StoreItem item);
+
+ public abstract Status delete(StoreItem item);
+
+ /**
+ * @param destinationDirectory The directory to use as a destination in the copy operation. It must be readable and
+ * writable. Copier will override the destination item if it already exists.
+ */
+ public abstract IStoreCopier getCopier(FileStore destinationDirectory);
+
+ // public boolean isParentDirectory(FileStoreIntf child);
+
+ public abstract ExtendedFileStore tryAsExtended();
+
+ public static abstract class ExtendedFileStore extends FileStore
+ {
+ protected ExtendedFileStore(File path, String hostOrNull, boolean remote, String kind,
+ IFileSysOperationsFactory factory)
+ {
+ super(path, hostOrNull, remote, kind, factory);
+ }
+
+ public abstract boolean createNewFile(StoreItem item);
+
+ public abstract File tryMoveLocal(StoreItem sourceItem, File destinationDir, String newFilePrefix);
+ }
+}
\ No newline at end of file
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IFileSysOperationsFactory.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IFileSysOperationsFactory.java
index 884995dbd44a430f7e85ba495e6bc9f5bfbdb610..e59f627170e19a5b5fdffb30eac9c9c1c65109d7 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IFileSysOperationsFactory.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IFileSysOperationsFactory.java
@@ -15,7 +15,6 @@
*/
package ch.systemsx.cisd.datamover.filesystem.intf;
-import java.io.File;
/**
* A role that provides access to the roles which perform file system operations.
@@ -24,13 +23,11 @@ import java.io.File;
*/
public interface IFileSysOperationsFactory
{
- public IPathCopier getCopier(File destinationDirectory);
+ public IPathCopier getCopier(boolean requiresDeletionBeforeCreation);
public IPathImmutableCopier getImmutableCopier();
public IPathRemover getRemover();
-
- public IPathMover getMover();
- public IReadPathOperations getReadPathOperations();
+ public IPathMover getMover();
}
\ No newline at end of file
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IPathCopier.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IPathCopier.java
index 8f67a852b41ac054529951a6ed056b71c3b8e06a..36abcb3c54b9ace47372d59df1de5c86eb22f7a2 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IPathCopier.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IPathCopier.java
@@ -29,35 +29,45 @@ import ch.systemsx.cisd.common.utilities.ITerminable;
* <i>Note: If the copier is terminated, the <var>destinationDirectory</var> is in an undefined state afterwards.</i>
*
* @author Bernd Rinn
+ * @author Tomasz Pylak
*/
public interface IPathCopier extends ITerminable, ISelfTestable
{
/**
- * @return <code>true</code> iff <var>destinationDirectory</var> on host <var>destinationHost</var> exists.
+ * Copies <var>sourcePath</var> to <var>destinationDir</var>.
+ *
+ * @param sourcePath The source to copy. Can be a file or a directory. It needs to exist and be readable.
+ * @param destinationDirectory The directory to use as a destination in the copy operation. It must be readable and
+ * writable. If <var>destinationDir/sourcePath</var> exists, it will be overwritten.
+ * @return The status of the operation, {@link Status#OK} if everything went OK.
*/
- public boolean exists(File destinationDirectory, String destinationHost);
+ Status copy(File sourcePath, File destinationDirectory);
/**
- * Copies <var>sourcePath</var> to <var>destinationDir</var>.
+ * Copies <var>sourcePath</var> to <var>destinationDir</var> on <var>destinationHost</var>.
*
* @param sourcePath The source to copy. Can be a file or a directory. It needs to exist and be readable.
* @param destinationDirectory The directory to use as a destination in the copy operation. It must be readable and
* writable. If <var>destinationDir/sourcePath</var> exists, it will be overwritten.
+ * @param destinationHost The host where the <var>destinationDirectory</var> resides or null if it is local.
* @return The status of the operation, {@link Status#OK} if everything went OK.
*/
- public Status copy(File sourcePath, File destinationDirectory);
+ Status copyToRemote(File sourcePath, File destinationDirectory, String destinationHost);
/**
- * Copies <var>sourcePath</var> to <var>destinationDir</var> on <var>destinationHost</var>.
+ * Copies <var>sourcePath</var> on <var>sourceHost</var> to <var>destinationDir</var>.
*
* @param sourcePath The source to copy. Can be a file or a directory. It needs to exist and be readable.
- * @param sourceHost The host where the <var>sourcePath</var> resides or null if it is local.
+ * @param sourceHost The host where the <var>sourcePath</var> resides
* @param destinationDirectory The directory to use as a destination in the copy operation. It must be readable and
* writable. If <var>destinationDir/sourcePath</var> exists, it will be overwritten.
- * @param destinationHost The host where the <var>destinationDirectory</var> resides or null if it is local.
* @return The status of the operation, {@link Status#OK} if everything went OK.
*/
- public Status copy(File sourcePath, String sourceHost, File destinationDirectory, String destinationHost);
+ Status copyFromRemote(File sourcePath, String sourceHost, File destinationDirectory);
+ /**
+ * @return <code>true</code> iff <var>destinationDirectory</var> on host <var>destinationHost</var> exists.
+ */
+ public boolean existsRemotely(File destinationDirectory, String destinationHost);
}
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IReadPathOperations.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IReadPathOperations.java
deleted file mode 100644
index e4f12636828dbaf26d7ad59ff92321806223f40a..0000000000000000000000000000000000000000
--- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IReadPathOperations.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright 2007 ETH Zuerich, CISD
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package ch.systemsx.cisd.datamover.filesystem.intf;
-
-import java.io.File;
-import java.io.FileFilter;
-
-import ch.systemsx.cisd.common.logging.ISimpleLogger;
-
-/**
- * Allows for checking the status of the file system. Performs only read-only operations.
- *
- * @author Tomasz Pylak on Aug 27, 2007
- */
-public interface IReadPathOperations
-{
- /**
- * lists files in a given directory matching a given filter
- *
- * @param loggerOrNull logger, if <code>null</code> than no logging occurs
- */
- File[] tryListFiles(File directory, FileFilter filter, ISimpleLogger loggerOrNull);
-
- /** lists all files in a given directory, logs errors */
- File[] tryListFiles(File directory, ISimpleLogger logger);
-
- /** checks if a file exists */
- boolean exists(File file);
-
- /**
- * Returns the last time when there was a write access to <var>resource</var>.
- *
- * @param resource The path to check for last write activity.
- * @return The time (in milliseconds since the start of the epoch) when <var>resource</var> was last changed.
- */
- long lastChanged(File resource);
-
-}
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IStoreCopier.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IStoreCopier.java
new file mode 100644
index 0000000000000000000000000000000000000000..4d289a364d67a9a4b24f6a225c94d710882f0be7
--- /dev/null
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IStoreCopier.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.datamover.filesystem.intf;
+
+import ch.systemsx.cisd.common.exceptions.Status;
+import ch.systemsx.cisd.common.utilities.ITerminable;
+import ch.systemsx.cisd.datamover.common.StoreItem;
+
+/**
+ * @author Tomasz Pylak on Oct 10, 2007
+ */
+public interface IStoreCopier extends ITerminable
+{
+ Status copy(StoreItem item);
+}
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IStoreHandler.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IStoreHandler.java
new file mode 100644
index 0000000000000000000000000000000000000000..1870ed04f029cf958da58fbccb94199f5a9a4874
--- /dev/null
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IStoreHandler.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.datamover.filesystem.intf;
+
+import ch.systemsx.cisd.datamover.common.StoreItem;
+
+
+/**
+ * @author Tomasz Pylak on Oct 9, 2007
+ */
+public interface IStoreHandler
+{
+ void handle(StoreItem item);
+}
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/remote/CopyActivityMonitor.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/remote/CopyActivityMonitor.java
index 1c465603619d91c9e5e0dd14a460754762c5b337..672a61b1220279ccc0a12f408f5a30aa6d7d16e1 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/remote/CopyActivityMonitor.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/remote/CopyActivityMonitor.java
@@ -16,7 +16,6 @@
package ch.systemsx.cisd.datamover.filesystem.remote;
-import java.io.File;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -28,7 +27,8 @@ import org.apache.log4j.Logger;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.utilities.ITerminable;
-import ch.systemsx.cisd.datamover.filesystem.intf.IReadPathOperations;
+import ch.systemsx.cisd.datamover.common.StoreItem;
+import ch.systemsx.cisd.datamover.filesystem.intf.FileStore;
import ch.systemsx.cisd.datamover.intf.ITimingParameters;
/**
@@ -44,9 +44,7 @@ public class CopyActivityMonitor
private static final Logger machineLog = LogFactory.getLogger(LogCategory.MACHINE, CopyActivityMonitor.class);
- private final File destinationDirectory;
-
- private final IReadPathOperations readOperations;
+ private final FileStore destinationDirectory;
private final long checkIntervallMillis;
@@ -78,7 +76,7 @@ public class CopyActivityMonitor
/**
* A <code>null</code> reference means: no monitoring.
*/
- private final AtomicReference<File> pathToBeCopied;
+ private final AtomicReference<StoreItem> pathToBeCopied;
/**
* The time in milliseconds since start of the epoch when the monitored path has last been changed.
@@ -94,25 +92,22 @@ public class CopyActivityMonitor
* Creates a monitor.
*
* @param destinationDirectory The directory to monitor for write access.
- * @param readOperations Provides read-only access to the file system.
* @param copyProcess The {@link ITerminable} representing the copy process. This will get terminated if the copy
* process gets stuck.
* @param timingParameters The {@link ITimingParameters} to get the check interval and the inactivity period from.
*/
- public CopyActivityMonitor(File destinationDirectory, IReadPathOperations readOperations, ITerminable copyProcess,
+ public CopyActivityMonitor(FileStore destinationDirectory, ITerminable copyProcess,
ITimingParameters timingParameters)
{
this.monitoredPathLastChecked = new AtomicLong(0);
this.monitoredPathLastChanged = new AtomicLong(0);
- this.pathToBeCopied = new AtomicReference<File>(null);
+ this.pathToBeCopied = new AtomicReference<StoreItem>(null);
assert destinationDirectory != null;
- assert readOperations != null;
assert copyProcess != null;
assert timingParameters != null;
this.destinationDirectory = destinationDirectory;
- this.readOperations = readOperations;
this.checkIntervallMillis = timingParameters.getCheckIntervalMillis();
assert this.checkIntervallMillis > 0;
@@ -158,7 +153,7 @@ public class CopyActivityMonitor
* @param newPathToBeCopied The path that will be copied to the destination directory and whose write progress
* should be monitored.
*/
- public void start(File newPathToBeCopied)
+ public void start(StoreItem newPathToBeCopied)
{
assert newPathToBeCopied != null;
@@ -187,7 +182,6 @@ public class CopyActivityMonitor
private ActivityMonitoringTimerTask()
{
- assert readOperations != null;
assert pathToBeCopied != null;
assert monitoredPathLastChanged != null;
assert destinationDirectory != null;
@@ -196,8 +190,8 @@ public class CopyActivityMonitor
@Override
public void run()
{
- final File path = pathToBeCopied.get();
- if (path == null)
+ final StoreItem item = pathToBeCopied.get();
+ if (item == null)
{
return;
}
@@ -209,24 +203,24 @@ public class CopyActivityMonitor
try
{
- final File pathToCheck = new File(destinationDirectory, path.getName());
if (operationLog.isTraceEnabled())
{
- operationLog.trace(String.format("Asking checker %s for last change time of path '%s'.",
- readOperations.getClass().getName(), pathToCheck));
+ operationLog.trace(String.format("Asking for last change time of '%s' inside '%s'.", item,
+ destinationDirectory));
}
- if (readOperations.exists(pathToCheck) == false)
+ if (destinationDirectory.exists(item) == false)
{
- operationLog.warn(String.format("File or directory '%s' does not (yet?) exist.", pathToCheck));
+ operationLog.warn(String.format("File or directory '%s' inside '%s' does not (yet?) exist.", item,
+ destinationDirectory));
monitoredPathLastChecked.set(System.currentTimeMillis());
return;
}
- final long lastChangedAsFoundByPathChecker = readOperations.lastChanged(pathToCheck);
+ final long lastChangedAsFoundByPathChecker = destinationDirectory.lastChanged(item);
if (operationLog.isTraceEnabled())
{
operationLog.trace(String.format(
- "Checker %s reported last changed time of path '%s' to be %3$tF %3$tT.", readOperations
- .getClass().getName(), pathToCheck.getPath(), lastChangedAsFoundByPathChecker));
+ "Reported last changed time of '%s' inside '%s' to be %3$tF %3$tT.", item,
+ destinationDirectory, lastChangedAsFoundByPathChecker));
}
if (terminated.get()) // Don't modify the time variables any more if we got terminated.
{
@@ -303,7 +297,7 @@ public class CopyActivityMonitor
@Override
public void run()
{
- final File path = pathToBeCopied.get();
+ final StoreItem path = pathToBeCopied.get();
if (path == null)
{
return;
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/remote/RemotePathMover.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/remote/RemotePathMover.java
index aaec1d8915ecea68c7e997c7d6a33d7345f09422..d62cde273ae2c2657985af9ecdf64bfd6549defc 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/remote/RemotePathMover.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/remote/RemotePathMover.java
@@ -16,20 +16,18 @@
package ch.systemsx.cisd.datamover.filesystem.remote;
-import java.io.File;
-import java.io.IOException;
-
import org.apache.log4j.Logger;
import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.exceptions.StatusFlag;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
-import ch.systemsx.cisd.common.utilities.FileUtilities;
-import ch.systemsx.cisd.common.utilities.IPathHandler;
import ch.systemsx.cisd.datamover.common.MarkerFile;
-import ch.systemsx.cisd.datamover.filesystem.intf.IPathCopier;
-import ch.systemsx.cisd.datamover.filesystem.intf.IPathRemover;
+import ch.systemsx.cisd.datamover.common.StoreItem;
+import ch.systemsx.cisd.datamover.filesystem.intf.FileStore;
+import ch.systemsx.cisd.datamover.filesystem.intf.IStoreCopier;
+import ch.systemsx.cisd.datamover.filesystem.intf.IStoreHandler;
+import ch.systemsx.cisd.datamover.filesystem.intf.FileStore.ExtendedFileStore;
import ch.systemsx.cisd.datamover.intf.ITimingParameters;
/**
@@ -38,7 +36,7 @@ import ch.systemsx.cisd.datamover.intf.ITimingParameters;
*
* @author Bernd Rinn
*/
-public final class RemotePathMover implements IPathHandler
+public final class RemotePathMover implements IStoreHandler
{
private static final String START_COPYING_PATH_TEMPLATE = "Start copying path '%s' to '%s'.";
@@ -56,10 +54,10 @@ public final class RemotePathMover implements IPathHandler
private static final String REMOVING_LOCAL_PATH_FAILED_TEMPLATE = "Removing local path '%s' failed (%s).";
- private static final String FAILED_TO_CREATE_MARK_FILE_TEMPLATE = "Failed to create mark file '%s'";
+ private static final String FAILED_TO_CREATE_FILE_TEMPLATE = "Failed to create file '%s' in '%s'";
- private static final String FAILED_TO_COPY_MARK_FILE_TO_REMOTE_TEMPLATE =
- "Failed to copy mark file '%s' to remote (%s)";
+ private static final String FAILED_TO_COPY_FILE_TO_REMOTE_TEMPLATE =
+ "Failed to copy file '%s' from '%s' to remote (%s)";
private static final Logger machineLog = LogFactory.getLogger(LogCategory.MACHINE, RemotePathMover.class);
@@ -67,15 +65,11 @@ public final class RemotePathMover implements IPathHandler
private static final Logger notificationLog = LogFactory.getLogger(LogCategory.NOTIFY, RemotePathMover.class);
- private final File destinationDirectory;
-
- private final String destinationHost;
-
- private final IPathCopier copier;
+ private final FileStore sourceDirectory;
- private final IPathRemover remover;
+ private final FileStore destinationDirectory;
- private final String sourceHost;
+ private final IStoreCopier copier;
private final CopyActivityMonitor monitor;
@@ -86,33 +80,27 @@ public final class RemotePathMover implements IPathHandler
/**
* Creates a <var>PathRemoteMover</var>.
*
+ * @param sourceDirectory The directory to move paths from.
* @param destinationDirectory The directory to move paths to.
- * @param destinationHost The host to move paths to, or <code>null</code>, if <var>destinationDirectory</var> is
- * a remote share.
+ * @param copier Copies items from source to destination
* @param monitor The activity monitor to inform about actions.
- * @param remover Allows to remove files.
- * @param copier Allows to copy files
- * @param sourceHost The host to move paths from, or <code>null</code>, if data will be moved from the local file
- * system
* @param timingParameters The timing parametes used for monitoring and reporting stall situations.
*/
- public RemotePathMover(File destinationDirectory, String destinationHost, CopyActivityMonitor monitor,
- IPathRemover remover, IPathCopier copier, String sourceHost, ITimingParameters timingParameters)
+ public RemotePathMover(FileStore sourceDirectory, FileStore destinationDirectory, IStoreCopier copier,
+ CopyActivityMonitor monitor, ITimingParameters timingParameters)
{
+ assert sourceDirectory != null;
assert destinationDirectory != null;
assert monitor != null;
- assert remover != null;
- assert copier != null;
assert timingParameters != null;
- assert FileUtilities.checkDirectoryFullyAccessible(destinationDirectory, "destination") == null : FileUtilities
- .checkDirectoryFullyAccessible(destinationDirectory, "destination");
+ String errorMsg = destinationDirectory.tryCheckDirectoryFullyAccessible();
+ assert errorMsg == null : errorMsg;
+ assert sourceDirectory.tryAsExtended() != null || destinationDirectory.tryAsExtended() != null;
+ this.sourceDirectory = sourceDirectory;
this.destinationDirectory = destinationDirectory;
- this.destinationHost = destinationHost;
- this.monitor = monitor;
this.copier = copier;
- this.remover = remover;
- this.sourceHost = sourceHost;
+ this.monitor = monitor;
this.intervallToWaitAfterFailure = timingParameters.getIntervalToWaitAfterFailure();
this.maximalNumberOfRetries = timingParameters.getMaximalNumberOfRetries();
@@ -120,19 +108,18 @@ public final class RemotePathMover implements IPathHandler
assert maximalNumberOfRetries >= 0;
}
- public void handle(File path)
+ public void handle(StoreItem item)
{
- if (isDeletionInProgress(path))
+ if (isDeletionInProgress(item))
{
// This is a recovery situation: we have been interrupted removing the path and now finish the job.
if (operationLog.isInfoEnabled())
{
operationLog.info(String.format(
- "Detected recovery situation: '%s' has been interrupted in deletion phase, finishing up.", path
- .getAbsolutePath()));
+ "Detected recovery situation: '%s' has been interrupted in deletion phase, finishing up.",
+ getSrcPath(item)));
}
- remove(path);
- markAsFinished(path);
+ removeAndMark(item);
return;
}
int tryCount = 0;
@@ -142,33 +129,30 @@ public final class RemotePathMover implements IPathHandler
{
if (tryCount > 0) // This is a retry
{
- operationLog.info(String.format(START_COPYING_PATH_RETRY_TEMPLATE, path.getPath(),
- destinationDirectory.getPath(), tryCount));
+ operationLog.info(String.format(START_COPYING_PATH_RETRY_TEMPLATE, getSrcPath(item),
+ destinationDirectory, tryCount));
} else
{
- operationLog.info(String.format(START_COPYING_PATH_TEMPLATE, path.getPath(), destinationDirectory
- .getPath()));
+ operationLog.info(String
+ .format(START_COPYING_PATH_TEMPLATE, getSrcPath(item), destinationDirectory));
}
}
final long startTime = System.currentTimeMillis();
- monitor.start(path);
- final Status copyStatus = copier.copy(path, sourceHost, destinationDirectory, destinationHost);
- monitor.stop();
+ final Status copyStatus = copyAndMonitor(item);
if (StatusFlag.OK.equals(copyStatus.getFlag()))
{
if (operationLog.isInfoEnabled())
{
final long endTime = System.currentTimeMillis();
- operationLog.info(String.format(FINISH_COPYING_PATH_TEMPLATE, path.getPath(), destinationDirectory
- .getPath(), (endTime - startTime) / 1000.0));
+ operationLog.info(String.format(FINISH_COPYING_PATH_TEMPLATE, getSrcPath(item),
+ destinationDirectory, (endTime - startTime) / 1000.0));
}
- remove(path);
- markAsFinished(path);
+ removeAndMark(item);
return;
} else
{
- operationLog.warn(String.format(COPYING_PATH_TO_REMOTE_FAILED, path.getPath(), destinationDirectory
- .getPath(), copyStatus));
+ operationLog.warn(String.format(COPYING_PATH_TO_REMOTE_FAILED, getSrcPath(item), destinationDirectory,
+ copyStatus));
if (StatusFlag.FATAL_ERROR.equals(copyStatus.getFlag()))
{
break;
@@ -189,122 +173,143 @@ public final class RemotePathMover implements IPathHandler
}
} while (true);
- notificationLog.error(String.format(MOVING_PATH_TO_REMOTE_FAILED_TEMPLATE, path, destinationDirectory));
+ notificationLog.error(String.format(MOVING_PATH_TO_REMOTE_FAILED_TEMPLATE, getSrcPath(item),
+ destinationDirectory));
+ }
+
+ private Status copyAndMonitor(StoreItem item)
+ {
+ monitor.start(item);
+ final Status copyStatus = copier.copy(item);
+ monitor.stop();
+ return copyStatus;
+ }
+
+ private void removeAndMark(StoreItem item)
+ {
+ remove(item);
+ markAsFinished(item);
}
- private void remove(File path)
+ private void remove(StoreItem sourceItem)
{
- final File removalInProgressMarkerFile = tryMarkAsDeletionInProgress(path);
- final Status removalStatus = remover.remove(path);
- removeMarkerFile(removalInProgressMarkerFile);
+ final StoreItem removalInProgressMarkerFile = tryMarkAsDeletionInProgress(sourceItem);
+ final Status removalStatus = sourceDirectory.delete(sourceItem);
+ removeDeletionMarkerFile(removalInProgressMarkerFile);
+
if (Status.OK.equals(removalStatus) == false)
{
- notificationLog.error(String.format(REMOVING_LOCAL_PATH_FAILED_TEMPLATE, path, removalStatus));
+ notificationLog.error(String.format(REMOVING_LOCAL_PATH_FAILED_TEMPLATE, getSrcPath(sourceItem),
+ removalStatus));
} else if (operationLog.isInfoEnabled())
{
- operationLog.info(String.format(REMOVED_PATH_TEMPLATE, path.getPath()));
+ operationLog.info(String.format(REMOVED_PATH_TEMPLATE, getSrcPath(sourceItem)));
}
}
- private boolean isDeletionInProgress(File path)
+ private boolean isDeletionInProgress(StoreItem item)
{
- final File markDeletionInProgressMarkerFile = getDeletionInProgressMarkerFile(path);
- return markDeletionInProgressMarkerFile.exists();
+ StoreItem markDeletionInProgressMarkerFile = MarkerFile.createDeletionInProgressMarker(item);
+ return getDeletionMarkerStore().exists(markDeletionInProgressMarkerFile);
}
- private File tryMarkAsDeletionInProgress(File path)
+ private StoreItem tryMarkAsDeletionInProgress(StoreItem item)
{
- final File markDeletionInProgressMarkerFile = getDeletionInProgressMarkerFile(path);
- if (markLocal(markDeletionInProgressMarkerFile))
+ final StoreItem markDeletionInProgressMarkerFile = MarkerFile.createDeletionInProgressMarker(item);
+ if (createFileInside(getDeletionMarkerStore(), markDeletionInProgressMarkerFile))
{
return markDeletionInProgressMarkerFile;
} else
{
- machineLog.error(String.format("Cannot create deletion-in-progress marker file for path '%s' [%s]", path
- .getAbsolutePath(), markDeletionInProgressMarkerFile.getAbsolutePath()));
+ machineLog.error(String.format("Cannot create deletion-in-progress marker file for path '%s' [%s]", item,
+ markDeletionInProgressMarkerFile));
return null;
}
}
- private File getDeletionInProgressMarkerFile(File path)
+ private void removeDeletionMarkerFile(StoreItem markerOrNull)
{
- // When destinationHost == null, we put the marker directory in the destination directory, otherwise in
- // the source directory
- final File markerParentDirectory = (destinationHost == null) ? destinationDirectory : path.getParentFile();
- final File markDeletionInProgressMarkerFile =
- MarkerFile.createDeletionInProgressMarker(markerParentDirectory, path.getName());
- return markDeletionInProgressMarkerFile;
- }
-
- private void removeMarkerFile(File markerFileOrNull)
- {
- if (markerFileOrNull != null)
+ if (markerOrNull != null)
{
- final boolean removalOK = markerFileOrNull.delete();
- if (removalOK == false)
+ final Status status = getDeletionMarkerStore().delete(markerOrNull);
+ if (status.equals(Status.OK) == false)
{
- machineLog.error(String.format("Cannot remove marker file '%s'", markerFileOrNull.getAbsolutePath()));
+ machineLog.error(String.format("Cannot remove marker file '%s'", getPath(destinationDirectory,
+ markerOrNull)));
}
}
}
- private boolean markAsFinished(File path)
+ private ExtendedFileStore getDeletionMarkerStore()
{
- final File markFinishedFile = MarkerFile.createCopyFinishedMarker(destinationDirectory, path.getName());
- if (destinationHost == null)
- {
- return markLocal(markFinishedFile);
- } else
+ ExtendedFileStore fileStore = destinationDirectory.tryAsExtended();
+ if (fileStore == null)
{
- return markOnSourceLocalAndCopyToRemoteDestination(markFinishedFile);
+ fileStore = sourceDirectory.tryAsExtended();
}
+ assert fileStore != null;
+ return fileStore;
}
- private boolean markLocal(File markerFile)
+ // Creates a finish-marker inside destination directory.
+ private boolean markAsFinished(StoreItem item)
{
- try
+ StoreItem markerItem = MarkerFile.createCopyFinishedMarker(item);
+ ExtendedFileStore extendedFileStore = destinationDirectory.tryAsExtended();
+ if (extendedFileStore != null)
{
- markerFile.createNewFile();
- final boolean success = markerFile.exists();
- if (success == false)
- {
- machineLog.error(String.format(FAILED_TO_CREATE_MARK_FILE_TEMPLATE, markerFile.getAbsoluteFile()));
- }
- return success;
- } catch (IOException e)
+ // We create the marker directly inside the destination directory
+ return createFileInside(extendedFileStore, markerItem);
+ } else
{
- machineLog.error(String.format(FAILED_TO_CREATE_MARK_FILE_TEMPLATE, markerFile.getAbsoluteFile()), e);
- return false;
+ // When destination is remote, we put the item directory in the source directory and copy it to destination.
+ extendedFileStore = sourceDirectory.tryAsExtended();
+ assert extendedFileStore != null;
+ return markOnSourceLocalAndCopyToRemoteDestination(extendedFileStore, markerItem);
}
}
- private boolean markOnSourceLocalAndCopyToRemoteDestination(File markerFile)
+ private boolean markOnSourceLocalAndCopyToRemoteDestination(ExtendedFileStore sourceFileStore, StoreItem markerFile)
{
- final File localMarkerFile = new File(markerFile.getParent(), markerFile.getName());
try
{
- localMarkerFile.createNewFile();
- monitor.start(localMarkerFile);
- final Status copyStatus = copier.copy(localMarkerFile, sourceHost, destinationDirectory, destinationHost);
- monitor.stop();
+ if (createFileInside(sourceFileStore, markerFile) == false)
+ {
+ return false;
+ }
+ final Status copyStatus = copyAndMonitor(markerFile);
if (StatusFlag.OK.equals(copyStatus.getFlag()))
{
return true;
} else
{
- machineLog.error(String.format(FAILED_TO_COPY_MARK_FILE_TO_REMOTE_TEMPLATE, localMarkerFile
- .getAbsoluteFile(), copyStatus.toString()));
+ machineLog.error(String.format(FAILED_TO_COPY_FILE_TO_REMOTE_TEMPLATE, markerFile, sourceFileStore,
+ copyStatus.toString()));
return false;
}
-
- } catch (IOException e)
- {
- machineLog.error(String.format(FAILED_TO_CREATE_MARK_FILE_TEMPLATE, localMarkerFile.getAbsoluteFile()), e);
- return false;
} finally
{
- localMarkerFile.delete();
+ sourceFileStore.delete(markerFile);
+ }
+ }
+
+ private static boolean createFileInside(ExtendedFileStore directory, StoreItem item)
+ {
+ boolean success = directory.createNewFile(item);
+ if (success == false)
+ {
+ machineLog.error(String.format(FAILED_TO_CREATE_FILE_TEMPLATE, item, directory));
}
+ return success;
}
+ private String getSrcPath(StoreItem item)
+ {
+ return getPath(sourceDirectory, item);
+ }
+
+ private static String getPath(FileStore directory, StoreItem item)
+ {
+ return item + " inside " + directory;
+ }
}
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/remote/rsync/RsyncCopier.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/remote/rsync/RsyncCopier.java
index d736155cbdd79e8e1232eefd85c341048e20a9ac..7a658f30588f2777fb473950db00f704c8107200 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/remote/rsync/RsyncCopier.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/remote/rsync/RsyncCopier.java
@@ -32,9 +32,8 @@ import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.exceptions.StatusFlag;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
-import ch.systemsx.cisd.common.utilities.ProcessExecutionHelper;
-import ch.systemsx.cisd.common.utilities.FileUtilities;
import ch.systemsx.cisd.common.utilities.OSUtilities;
+import ch.systemsx.cisd.common.utilities.ProcessExecutionHelper;
import ch.systemsx.cisd.datamover.filesystem.intf.IPathCopier;
import ch.systemsx.cisd.datamover.filesystem.remote.rsync.RsyncVersionChecker.RsyncVersion;
@@ -86,21 +85,21 @@ public class RsyncCopier implements IPathCopier
* Constructs an <code>RsyncCopier</code>.
*
* @param rsyncExecutable The <code>rsync</code> binary to call for copying.
- * @param sshExecutable The <code>ssh</code> binary to use for creating tunnels, or <code>null</code>, if no
- * <code>ssh</code> is available on this machine.
+ * @param sshExecutableOrNull The <code>ssh</code> binary to use for creating tunnels, or <code>null</code>, if
+ * no <code>ssh</code> is available on this machine.
* @param destinationDirectoryRequiresDeletionBeforeCreation If <code>true</code>, already existing files and
* directories on the remote side will be deleted before starting the copy process (no overwriting of
* paths).
*/
- public RsyncCopier(File rsyncExecutable, File sshExecutable,
+ public RsyncCopier(File rsyncExecutable, File sshExecutableOrNull,
boolean destinationDirectoryRequiresDeletionBeforeCreation, boolean overwrite, String... cmdLineFlags)
{
assert rsyncExecutable != null && rsyncExecutable.exists();
- assert sshExecutable == null || rsyncExecutable.exists();
+ assert sshExecutableOrNull == null || rsyncExecutable.exists();
this.rsyncExecutable = rsyncExecutable.getAbsolutePath();
this.rsyncVersion = RsyncVersionChecker.getVersion(rsyncExecutable.getAbsolutePath());
- this.sshExecutable = (sshExecutable != null) ? sshExecutable.getPath() : null;
+ this.sshExecutable = (sshExecutableOrNull != null) ? sshExecutableOrNull.getPath() : null;
this.destinationDirectoryRequiresDeletionBeforeCreation = destinationDirectoryRequiresDeletionBeforeCreation;
this.copyProcessReference = new AtomicReference<Process>(null);
this.overwrite = overwrite;
@@ -130,30 +129,30 @@ public class RsyncCopier implements IPathCopier
return copy(sourcePath, null, destinationDirectory, null);
}
- public Status copy(File sourcePath, String sourceHost, File destinationDirectory, String destinationHost)
+ public Status copyFromRemote(File sourcePath, String sourceHost, File destinationDirectory)
+ {
+ return copy(sourcePath, sourceHost, destinationDirectory, null);
+ }
+
+ public Status copyToRemote(File sourcePath, File destinationDirectory, String destinationHost)
+ {
+ return copy(sourcePath, null, destinationDirectory, destinationHost);
+ }
+
+ private Status copy(File sourcePath, String sourceHostOrNull, File destinationDirectory,
+ String destinationHostOrNull)
{
assert sourcePath != null;
- assert sourceHost != null || sourcePath.exists() : logNonExistent(sourcePath);
+ assert sourceHostOrNull != null || sourcePath.exists() : logNonExistent(sourcePath);
assert destinationDirectory != null;
- assert destinationHost != null || destinationDirectory.isDirectory() : logNonExistent(sourcePath);
- assert sourceHost == null || destinationHost == null; // only one side can be remote
+ assert destinationHostOrNull != null || destinationDirectory.isDirectory() : logNonExistent(sourcePath);
+ assert sourceHostOrNull == null || destinationHostOrNull == null; // only one side can be remote
- final File destinationPath = new File(destinationDirectory, sourcePath.getName());
- if (destinationDirectoryRequiresDeletionBeforeCreation && destinationPath.exists())
- {
- if (operationLog.isDebugEnabled())
- {
- operationLog.debug(String.format(
- "Remove path '%s' since it exists and the remote file system doesn't support overwriting.",
- destinationPath));
- }
- // TODO 2007-08-13 Tomasz Pylak: Bernd, what if destinationHost is set?
- FileUtilities.deleteRecursively(destinationPath);
- }
try
{
final ProcessBuilder copyProcessBuilder =
- new ProcessBuilder(createCommandLine(sourcePath, sourceHost, destinationDirectory, destinationHost));
+ new ProcessBuilder(createCommandLine(sourcePath, sourceHostOrNull, destinationDirectory,
+ destinationHostOrNull));
copyProcessBuilder.redirectErrorStream(true);
if (operationLog.isDebugEnabled())
{
@@ -344,7 +343,7 @@ public class RsyncCopier implements IPathCopier
}
}
- public boolean exists(File destinationDirectory, String destinationHost)
+ public boolean existsRemotely(File destinationDirectory, String destinationHost)
{
assert destinationDirectory != null && destinationDirectory.isDirectory();
assert destinationHost != null;
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreLocal.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreLocal.java
new file mode 100644
index 0000000000000000000000000000000000000000..284d9ed053888ddea9951ee6e4535df3c24fbb21
--- /dev/null
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreLocal.java
@@ -0,0 +1,184 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.datamover.filesystem.store;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.log4j.Logger;
+
+import ch.systemsx.cisd.common.exceptions.Status;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.utilities.FileUtilities;
+import ch.systemsx.cisd.datamover.common.MarkerFile;
+import ch.systemsx.cisd.datamover.common.StoreItem;
+import ch.systemsx.cisd.datamover.filesystem.intf.FileStore;
+import ch.systemsx.cisd.datamover.filesystem.intf.IFileSysOperationsFactory;
+import ch.systemsx.cisd.datamover.filesystem.intf.IPathMover;
+import ch.systemsx.cisd.datamover.filesystem.intf.IPathRemover;
+import ch.systemsx.cisd.datamover.filesystem.intf.IStoreCopier;
+import ch.systemsx.cisd.datamover.filesystem.intf.FileStore.ExtendedFileStore;
+
+/**
+ * @author Tomasz Pylak on Oct 9, 2007
+ */
+public class FileStoreLocal extends ExtendedFileStore
+{
+ private static final Logger machineLog = LogFactory.getLogger(LogCategory.MACHINE, FileStoreLocal.class);
+
+ private final IPathMover mover;
+
+ private final IPathRemover remover;
+
+ public FileStoreLocal(File file, String desription, IFileSysOperationsFactory factory)
+ {
+ super(file, null, false, desription, factory);
+ this.remover = factory.getRemover();
+ this.mover = factory.getMover();
+ }
+
+ @Override
+ public Status delete(StoreItem item)
+ {
+ return remover.remove(getChildFile(item));
+ }
+
+ @Override
+ public boolean exists(StoreItem item)
+ {
+ return getChildFile(item).exists();
+ }
+
+ @Override
+ public long lastChanged(StoreItem item)
+ {
+ return FileUtilities.lastChanged(getChildFile(item));
+ }
+
+ @Override
+ public String tryCheckDirectoryFullyAccessible()
+ {
+ return FileUtilities.checkDirectoryFullyAccessible(super.getPath(), super.getDescription());
+ }
+
+ @Override
+ public IStoreCopier getCopier(FileStore destinationDirectory)
+ {
+ boolean requiresDeletion = false;
+ final IStoreCopier simpleCopier = constructStoreCopier(destinationDirectory, requiresDeletion);
+ if (requiresDeletionBeforeCreation(destinationDirectory, simpleCopier))
+ {
+ requiresDeletion = true;
+ return constructStoreCopier(destinationDirectory, requiresDeletion);
+ } else
+ {
+ return simpleCopier;
+ }
+ }
+
+ @Override
+ public ExtendedFileStore tryAsExtended()
+ {
+ return this;
+ }
+
+ @Override
+ public boolean createNewFile(StoreItem item)
+ {
+ try
+ {
+ File itemFile = getChildFile(item);
+ itemFile.createNewFile();
+ return itemFile.exists(); // success also when file existed before
+ } catch (IOException ex)
+ {
+ ex.printStackTrace();
+ return false;
+ }
+ }
+
+ @Override
+ public File tryMoveLocal(StoreItem sourceItem, File destinationDir, String newFilePrefix)
+ {
+ return mover.tryMove(getChildFile(sourceItem), destinationDir, newFilePrefix);
+ }
+
+ @Override
+ public String toString()
+ {
+ String pathStr = super.path.getPath();
+ return "[local fs]" + pathStr;
+ }
+
+ // ------
+
+ /**
+ * @return <code>true</code> if the <var>simpleCopier</var> on the file system where the
+ * <var>destinationDirectory</var> resides requires deleting an existing file before it can be overwritten.
+ */
+ protected boolean requiresDeletionBeforeCreation(FileStore destinationDirectory, final IStoreCopier simpleCopier)
+ {
+ StoreItem item = MarkerFile.createRequiresDeletionBeforeCreationMarker();
+ createNewFile(item);
+ simpleCopier.copy(item);
+ boolean requiresDeletion;
+ try
+ {
+ // If we have e.g. a Cellera NAS server, the next call will raise an IOException.
+ requiresDeletion = Status.OK.equals(simpleCopier.copy(item)) == false;
+ logCopierOverwriteState(destinationDirectory, requiresDeletion);
+ } catch (Exception e)
+ {
+ logFIleSystemNeedsOverwrite(destinationDirectory);
+ requiresDeletion = true;
+ } finally
+ {
+ // We don't check for success because there is nothing we can do if we fail.
+ delete(item);
+ destinationDirectory.delete(item);
+ }
+ return requiresDeletion;
+ }
+
+ private static void logFIleSystemNeedsOverwrite(FileStore destinationDirectory)
+ {
+ if (machineLog.isInfoEnabled())
+ {
+ machineLog.info(String.format(
+ "The file system on '%s' requires deletion before creation of existing files.",
+ destinationDirectory));
+ }
+ }
+
+ private static void logCopierOverwriteState(FileStore destinationDirectory, boolean requiresDeletion)
+ {
+ if (machineLog.isInfoEnabled())
+ {
+ if (requiresDeletion)
+ {
+ machineLog.info(String.format(
+ "Copier on directory '%s' requires deletion before creation of existing files.",
+ destinationDirectory));
+ } else
+ {
+ machineLog.info(String.format("Copier on directory '%s' works with overwriting existing files.",
+ destinationDirectory));
+ }
+ }
+ }
+}
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreRemote.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreRemote.java
new file mode 100644
index 0000000000000000000000000000000000000000..14399bd3fdb8032f2c26dd5066555e10280f8029
--- /dev/null
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreRemote.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.datamover.filesystem.store;
+
+import java.io.File;
+
+import ch.systemsx.cisd.common.exceptions.Status;
+import ch.systemsx.cisd.datamover.common.StoreItem;
+import ch.systemsx.cisd.datamover.filesystem.intf.FileStore;
+import ch.systemsx.cisd.datamover.filesystem.intf.IFileSysOperationsFactory;
+import ch.systemsx.cisd.datamover.filesystem.intf.IStoreCopier;
+
+/**
+ * @author Tomasz Pylak on Oct 9, 2007
+ */
+public class FileStoreRemote extends FileStore
+{
+ public FileStoreRemote(File path, String host, String kind, IFileSysOperationsFactory factory)
+ {
+ super(path, host, true, kind, factory);
+ }
+
+ @Override
+ public ExtendedFileStore tryAsExtended()
+ {
+ return null;
+ }
+
+ @Override
+ public Status delete(StoreItem item)
+ {
+ // TODO 2007-10-09 implement ssh tunneling mode
+ return Status.OK;
+ }
+
+ @Override
+ public boolean exists(StoreItem item)
+ {
+ return factory.getCopier(false).existsRemotely(path, hostOrNull);
+ }
+
+ @Override
+ public IStoreCopier getCopier(FileStore destinationDirectory)
+ {
+ boolean requiresDeletion = false;
+ return constructStoreCopier(destinationDirectory, requiresDeletion);
+ }
+
+ @Override
+ public long lastChanged(StoreItem item)
+ {
+ // TODO 2007-10-09 implement ssh tunneling mode
+ return 0;
+ }
+
+ @Override
+ public String tryCheckDirectoryFullyAccessible()
+ {
+ // TODO 2007-10-09 implement ssh tunneling mode. E.g. check if directory exists
+ return null;
+ }
+
+ @Override
+ public String toString()
+ {
+ String pathStr = path.getPath();
+ return "[remote fs]" + hostOrNull + ":" + pathStr;
+ }
+}
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreRemoteMounted.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreRemoteMounted.java
new file mode 100644
index 0000000000000000000000000000000000000000..e00c5332421339c29f79e1981bf329c1d7e9a3eb
--- /dev/null
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreRemoteMounted.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.datamover.filesystem.store;
+
+import java.io.File;
+
+import ch.systemsx.cisd.common.exceptions.Status;
+import ch.systemsx.cisd.datamover.common.StoreItem;
+import ch.systemsx.cisd.datamover.filesystem.intf.FileStore;
+import ch.systemsx.cisd.datamover.filesystem.intf.IFileSysOperationsFactory;
+import ch.systemsx.cisd.datamover.filesystem.intf.IStoreCopier;
+
+/**
+ * @author Tomasz Pylak on Oct 9, 2007
+ */
+public class FileStoreRemoteMounted extends FileStore
+{
+ private final FileStoreLocal localImpl;
+
+ public FileStoreRemoteMounted(File file, String desription, IFileSysOperationsFactory factory)
+ {
+ super(file, null, true, desription, factory);
+ this.localImpl = new FileStoreLocal(file, desription, factory);
+ }
+
+ @Override
+ public ExtendedFileStore tryAsExtended()
+ {
+ return null;
+ }
+
+ @Override
+ public IStoreCopier getCopier(FileStore destinationDirectory)
+ {
+ boolean requiresDeletion = false;
+ return constructStoreCopier(destinationDirectory, requiresDeletion);
+ }
+
+ @Override
+ public String toString()
+ {
+ String pathStr = path.getPath();
+ return "[mounted remote fs]" + pathStr;
+ }
+
+ @Override
+ public Status delete(StoreItem item)
+ {
+ return localImpl.delete(item);
+ }
+
+ @Override
+ public boolean exists(StoreItem item)
+ {
+ return localImpl.exists(item);
+ }
+
+ @Override
+ public long lastChanged(StoreItem item)
+ {
+ return localImpl.lastChanged(item);
+ }
+
+ @Override
+ public String tryCheckDirectoryFullyAccessible()
+ {
+ return localImpl.tryCheckDirectoryFullyAccessible();
+ }
+}
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/utils/FileStore.java b/datamover/source/java/ch/systemsx/cisd/datamover/utils/FileStore.java
deleted file mode 100644
index afb60e57afd198430216f5066b489bdf288b65a9..0000000000000000000000000000000000000000
--- a/datamover/source/java/ch/systemsx/cisd/datamover/utils/FileStore.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Copyright 2007 ETH Zuerich, CISD
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package ch.systemsx.cisd.datamover.utils;
-
-import java.io.File;
-import java.io.IOException;
-
-import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
-
-/**
- * A class to holds the information about a file store.
- *
- * @author Bernd Rinn
- */
-public class FileStore
-{
- private final File path;
-
- private final String canonicalPath;
-
- private final String kind;
-
- private final String host;
-
- private final boolean remote;
-
- public FileStore(File path, String kind, String host, boolean remote)
- {
- this.path = path;
- this.kind = kind;
- if (path != null)
- {
- this.canonicalPath = getCanonicalPath(path, host);
- } else
- {
- this.canonicalPath = null;
- }
- this.host = host;
- this.remote = remote;
- }
-
- private static String getCanonicalPath(File path, String host)
- {
- if (host != null)
- {
- return path.getPath();
- }
- try
- {
- return path.getCanonicalPath() + File.separator;
- } catch (IOException e)
- {
- throw EnvironmentFailureException.fromTemplate(e, "Cannot determine canonical form of path '%s'", path
- .getPath());
- }
- }
-
- /**
- * Returns the remote host, if copying is done via an ssh tunnel, or <code>null</code> otherwise.
- */
- public String getHost()
- {
- return host;
- }
-
- /**
- * Returns the path of the file store. If {@link #getHost()} is <code>null</code>, and only then, it may be
- * interpreted to be on the local host.
- */
- public File getPath()
- {
- return path;
- }
-
- /**
- * Returns the canonical form of the path as a string, if {@link #getHost()} is <code>null</code>. Note that the
- * canonical name is only available if {@link #getHost()} is <code>null</code>, otherwise the normal path (as
- * returned by {@link File#getPath()}) will be returned by this method.
- */
- public String getCanonicalPath()
- {
- return canonicalPath;
- }
-
- /**
- * Returns the kind of file store (used for error and log messages).
- */
- public String getKind()
- {
- return kind;
- }
-
- /**
- * Returns <code>true</code>, if the file store resides on a remote computer and <code>false</code> otherwise.
- * <p>
- * Note that this method can return <code>true</code> despite {@link #getHost()} returning <code>null</code>.
- * In this case the file store is on a remote share mounted on local host via NFS or CIFS.
- */
- public boolean isRemote()
- {
- return remote;
- }
-
-}
\ No newline at end of file
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/utils/QuietPeriodFileFilter.java b/datamover/source/java/ch/systemsx/cisd/datamover/utils/QuietPeriodFileFilter.java
index 4ef46b413f7b2206c9fd0469e03848f2a2a8699c..65f37023b6068fc7753cd09e90b545c55656a7dd 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/utils/QuietPeriodFileFilter.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/utils/QuietPeriodFileFilter.java
@@ -19,7 +19,7 @@ package ch.systemsx.cisd.datamover.utils;
import java.io.File;
import java.io.FileFilter;
-import ch.systemsx.cisd.datamover.filesystem.intf.IReadPathOperations;
+import ch.systemsx.cisd.common.utilities.FileUtilities;
import ch.systemsx.cisd.datamover.intf.ITimingParameters;
/**
@@ -29,31 +29,23 @@ import ch.systemsx.cisd.datamover.intf.ITimingParameters;
*/
public class QuietPeriodFileFilter implements FileFilter
{
-
private final long quietPeriodMillis;
- private final IReadPathOperations readOperations;
-
/**
* Creates a <var>QuietPeriodFileFilter</var>.
*
* @param timingParameters The timing paramter object to get the quiet period from.
- * @param readOperations Used to check when a pathname was changed.
*/
- public QuietPeriodFileFilter(ITimingParameters timingParameters, IReadPathOperations readOperations)
+ public QuietPeriodFileFilter(ITimingParameters timingParameters)
{
assert timingParameters != null;
- assert readOperations != null;
-
this.quietPeriodMillis = timingParameters.getQuietPeriodMillis();
- this.readOperations = readOperations;
-
assert quietPeriodMillis > 0;
}
public boolean accept(File pathname)
{
- return (System.currentTimeMillis() - readOperations.lastChanged(pathname)) > quietPeriodMillis;
+ return (System.currentTimeMillis() - FileUtilities.lastChanged(pathname)) > quietPeriodMillis;
}
}
diff --git a/datamover/sourceTest/java/ch/systemsx/cisd/datamover/MainTest.java b/datamover/sourceTest/java/ch/systemsx/cisd/datamover/MainTest.java
index bb1d350a8fa5e65429a9cdbd38ee7a3616a29776..29d6161c9b601df58e6210f9c9c7905189a4d18f 100644
--- a/datamover/sourceTest/java/ch/systemsx/cisd/datamover/MainTest.java
+++ b/datamover/sourceTest/java/ch/systemsx/cisd/datamover/MainTest.java
@@ -18,8 +18,9 @@ package ch.systemsx.cisd.datamover;
import static ch.systemsx.cisd.datamover.testhelper.FileSystemHelper.assertEmptyDir;
import static ch.systemsx.cisd.datamover.testhelper.FileSystemHelper.createDir;
-
-import static org.testng.AssertJUnit.*;
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertFalse;
+import static org.testng.AssertJUnit.assertTrue;
import java.io.File;
import java.io.IOException;
@@ -49,7 +50,7 @@ public class MainTest
// time needed be a single test to complete. After this time we kill data mover and check if results are correct. It
// can happen, that tests running on slow machines fail because they need more time. Than this constant should be
// adjusted.
- private static final long DATA_MOVER_COMPLETION_TIME = 5000;
+ private static final long DATA_MOVER_COMPLETION_TIME = 6000;
private static final long DATA_MOVER_COMPLETION_TIME_LONG = 7000;
@@ -207,8 +208,8 @@ public class MainTest
{
return createList("--incoming-dir", dirs.incoming.getPath(), "--buffer-dir", dirs.buffer.getPath(),
"--outgoing-dir", dirs.outgoing.getPath(), "--check-interval", Integer.toString(CHECK_INTERVAL),
- "--check-interval-internal", Integer.toString(CHECK_INTERVAL_INTERNAL),
- "--quiet-period", Integer.toString(QUIET_PERIOD), "--treat-incoming-as-remote");
+ "--check-interval-internal", Integer.toString(CHECK_INTERVAL_INTERNAL), "--quiet-period", Integer
+ .toString(QUIET_PERIOD), "--treat-incoming-as-remote");
}
private static ArrayList<String> getManualInterventionParameters(ExternalDirs dirs, String filteredName)
@@ -268,8 +269,8 @@ public class MainTest
private static LocalBufferDirs createBufferDirs(Parameters parameters)
{
- return new LocalBufferDirs(parameters.getBufferStore().getPath(), "test-copy-in-progress",
- "test-copy-complete", "test-ready-to-move", "test-temp");
+ return new LocalBufferDirs(parameters.getBufferDirectoryPath(), "test-copy-in-progress", "test-copy-complete",
+ "test-ready-to-move", "test-temp");
}
private static interface IFSPreparator
@@ -409,7 +410,7 @@ public class MainTest
}
@Test(groups =
- { "slow" }, dataProvider = "delays")
+ { "slowXX" }, dataProvider = "delays")
// recovery after failure when data are copied to 'copy-completed', but before deletion has been finished
public void testRecoveryIncomingCompleteDeletionInProgress(long delay) throws Exception
{
@@ -631,7 +632,7 @@ public class MainTest
}
@Test(groups =
- { "slow" })
+ { "xxx" })
// normal work-flow tests, no extra copy is created
public void testWholePipelineNoExtraCopy() throws Exception
{
diff --git a/datamover/sourceTest/java/ch/systemsx/cisd/datamover/ParametersTest.java b/datamover/sourceTest/java/ch/systemsx/cisd/datamover/ParametersTest.java
index 04e5deda98f8f6b9d93072dda31f84174cced3a1..a865e5e82eb8eb10565667a1145fe92a8de3ece7 100644
--- a/datamover/sourceTest/java/ch/systemsx/cisd/datamover/ParametersTest.java
+++ b/datamover/sourceTest/java/ch/systemsx/cisd/datamover/ParametersTest.java
@@ -27,6 +27,11 @@ import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import ch.systemsx.cisd.common.utilities.SystemExit;
+import ch.systemsx.cisd.datamover.filesystem.FileStoreFactory;
+import ch.systemsx.cisd.datamover.filesystem.FileSysOperationsFactory;
+import ch.systemsx.cisd.datamover.filesystem.intf.FileStore;
+import ch.systemsx.cisd.datamover.filesystem.intf.IFileSysOperationsFactory;
+import ch.systemsx.cisd.datamover.intf.IFileSysParameters;
/**
* Test cases for the {@link Parameters} class.
@@ -83,7 +88,7 @@ public class ParametersTest
{
final String CLEANSING_REGEX = "[0-9]+";
final Parameters parameters = parse("--cleansing-regex", CLEANSING_REGEX);
- assertEquals(CLEANSING_REGEX, parameters.getCleansingRegex().pattern());
+ assertEquals(CLEANSING_REGEX, parameters.tryGetCleansingRegex().pattern());
}
@Test(expectedExceptions = RuntimeException.class)
@@ -98,7 +103,7 @@ public class ParametersTest
{
final String LOCAL_DATADIR = ".." + File.separator + "test_it_data";
final Parameters parameters = parse("--incoming-dir", LOCAL_DATADIR);
- assertEquals(LOCAL_DATADIR, parameters.getIncomingStore().getPath().getPath());
+ assertEquals(createIncomingStore(LOCAL_DATADIR, null, parameters), getIncomingStore(parameters));
}
@Test
@@ -106,23 +111,7 @@ public class ParametersTest
{
final String LOCAL_TEMPDIR = "test_it_tmp";
final Parameters parameters = parse("--buffer-dir", LOCAL_TEMPDIR);
- assertEquals(LOCAL_TEMPDIR, parameters.getBufferStore().getPath().getPath());
- }
-
- @Test
- public void testSetRemoteDirLong() throws Exception
- {
- final String REMOTE_DATADIR = "test_it_remote";
- final Parameters parameters = parse("--outgoing-dir", REMOTE_DATADIR);
- assertEquals(REMOTE_DATADIR, parameters.getOutgoingStore().getPath().getPath());
- }
-
- @Test
- public void testSetRemoteHostLong() throws Exception
- {
- final String REMOTE_HOST = "test_it_remote";
- final Parameters parameters = parse("--outgoing-host", REMOTE_HOST);
- assertEquals(REMOTE_HOST, parameters.getOutgoingStore().getHost());
+ assertEquals(LOCAL_TEMPDIR, parameters.getBufferDirectoryPath().getPath());
}
@Test
@@ -227,14 +216,14 @@ public class ParametersTest
final Parameters parameters = parse();
assertFalse(parameters.isRsyncOverwrite());
}
-
+
@Test
public void testSetRsyncOverwrite()
{
final Parameters parameters = parse("--rsync-overwrite");
assertTrue(parameters.isRsyncOverwrite());
}
-
+
@Test
public void testSetMandatoryOptions() throws Exception
{
@@ -243,9 +232,9 @@ public class ParametersTest
final String REMOTE_DATADIR = "rrr";
final Parameters parameters =
parse("--incoming-dir", LOCAL_DATADIR, "--buffer-dir", LOCAL_TEMPDIR, "--outgoing-dir", REMOTE_DATADIR);
- assertEquals(LOCAL_DATADIR, parameters.getIncomingStore().getPath().getPath());
- assertEquals(LOCAL_TEMPDIR, parameters.getBufferStore().getPath().getPath());
- assertEquals(REMOTE_DATADIR, parameters.getOutgoingStore().getPath().getPath());
+ assertEquals(createIncomingStore(LOCAL_DATADIR, null, parameters), getIncomingStore(parameters));
+ assertEquals(LOCAL_TEMPDIR, parameters.getBufferDirectoryPath().getPath());
+ assertEquals(createOutgoingStore(REMOTE_DATADIR, null, parameters), getOutgoingStore(parameters));
}
@Test
@@ -264,17 +253,58 @@ public class ParametersTest
parse("--incoming-dir", LOCAL_DATADIR, "--buffer-dir", LOCAL_TEMPDIR, "--outgoing-dir", REMOTE_DATADIR,
"--outgoing-host", REMOTE_HOST, "--check-interval", Integer.toString(CHECK_INTERVAL),
"--quiet-period", Integer.toString(QUIET_PERIOD), "--treat-incoming-as-remote",
- "--incoming-host", REMOTE_INCOMING_HOST, "--extra-copy-dir", EXTRA_COPY_DIR, "--rsync-overwrite");
- assertEquals(LOCAL_DATADIR, parameters.getIncomingStore().getPath().getPath());
- assertEquals(REMOTE_INCOMING_HOST, parameters.getIncomingStore().getHost());
- assertEquals(LOCAL_TEMPDIR, parameters.getBufferStore().getPath().getPath());
- assertEquals(REMOTE_DATADIR, parameters.getOutgoingStore().getPath().getPath());
- assertEquals(REMOTE_HOST, parameters.getOutgoingStore().getHost());
+ "--incoming-host", REMOTE_INCOMING_HOST, "--extra-copy-dir", EXTRA_COPY_DIR,
+ "--rsync-overwrite");
+ FileStore incomingStoreExpected = createIncomingStore(LOCAL_DATADIR, REMOTE_INCOMING_HOST, parameters);
+ FileStore incomingStore = getIncomingStore(parameters);
+ FileStore outgoingStoreExpected = createOutgoingStore(REMOTE_DATADIR, REMOTE_HOST, parameters);
+ FileStore outgoingStore = getOutgoingStore(parameters);
+
+ assertEquals(incomingStoreExpected, incomingStore);
+ assertEquals(LOCAL_TEMPDIR, parameters.getBufferDirectoryPath().getPath());
+ assertEquals(outgoingStoreExpected, outgoingStore);
assertEquals(EXTRA_COPY_DIR, parameters.tryGetExtraCopyDir().getPath());
assertEquals(1000 * CHECK_INTERVAL, parameters.getCheckIntervalMillis());
assertEquals(1000 * QUIET_PERIOD, parameters.getQuietPeriodMillis());
- assertEquals(true, parameters.getTreatIncomingAsRemote());
+ assertTrue(incomingStore.isRemote());
assertTrue(parameters.isRsyncOverwrite());
}
+ private FileStore getIncomingStore(Parameters parameters)
+ {
+ IFileSysOperationsFactory factory = new FileSysOperationsFactory(parameters);
+ return parameters.getIncomingStore(factory);
+ }
+
+ private FileStore getOutgoingStore(Parameters parameters)
+ {
+ IFileSysOperationsFactory factory = new FileSysOperationsFactory(parameters);
+ return parameters.getOutgoingStore(factory);
+ }
+
+ private static FileStore createIncomingStore(final String path, final String hostOrNull,
+ IFileSysParameters parameters)
+ {
+ return createStore(path, hostOrNull, Parameters.INCOMING_KIND_DESC, parameters);
+ }
+
+ private static FileStore createOutgoingStore(final String path, final String hostOrNull,
+ IFileSysParameters parameters)
+ {
+ return createStore(path, hostOrNull, Parameters.OUTGOING_KIND_DESC, parameters);
+ }
+
+ private static FileStore createStore(final String path, final String hostOrNull, String kind,
+ IFileSysParameters parameters)
+ {
+ IFileSysOperationsFactory factory = new FileSysOperationsFactory(parameters);
+ File file = new File(path);
+ if (hostOrNull == null)
+ {
+ return FileStoreFactory.createLocal(file, kind, factory);
+ } else
+ {
+ return FileStoreFactory.createRemoteHost(file, hostOrNull, kind, factory);
+ }
+ }
}
diff --git a/datamover/sourceTest/java/ch/systemsx/cisd/datamover/SelfTestTest.java b/datamover/sourceTest/java/ch/systemsx/cisd/datamover/SelfTestTest.java
index 3b313e4a4535abd4fbaf38d0dc6403ada87bc3b6..ac3abd78fcaa66f42338528baaf0a30441f21d8d 100644
--- a/datamover/sourceTest/java/ch/systemsx/cisd/datamover/SelfTestTest.java
+++ b/datamover/sourceTest/java/ch/systemsx/cisd/datamover/SelfTestTest.java
@@ -23,11 +23,14 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.logging.LogInitializer;
import ch.systemsx.cisd.common.utilities.FileUtilities;
+import ch.systemsx.cisd.datamover.filesystem.FileStoreFactory;
+import ch.systemsx.cisd.datamover.filesystem.intf.FileStore;
import ch.systemsx.cisd.datamover.filesystem.intf.IPathCopier;
-import ch.systemsx.cisd.datamover.utils.FileStore;
+import ch.systemsx.cisd.datamover.testhelper.FileOperationsUtil;
/**
* Test cases for the {@link SelfTest}.
@@ -43,19 +46,17 @@ public class SelfTestTest
private static final File incomingDirectory = new File(workingDirectory, "local/incoming");
- private static final FileStore incomingStore = new FileStore(incomingDirectory, "incoming", null, false);
+ private static final FileStore incomingStore = createLocalStore(incomingDirectory, "incoming");
private static final File bufferDirectory = new File(workingDirectory, "local/buffer");
- private static final FileStore bufferStore = new FileStore(bufferDirectory, "buffer", null, false);
+ private static final FileStore bufferStore = createLocalStore(bufferDirectory, "buffer");
private static final File outgoingDirectory = new File(workingDirectory, "outgoing");
- private static final FileStore outgoingStore = new FileStore(outgoingDirectory, "outgoing", null, false);
+ private static final FileStore outgoingStore = createLocalStore(outgoingDirectory, "outgoing");
- private static final FileStore dummyStore = new FileStore(null, "dummy", null, false);
-
- private static final IPathCopier mockCopier = new MockPathCopier(false);
+ private static final IPathCopier mockCopier = createMockCopier();
// ////////////////////////////////////////
// Initialization methods.
@@ -87,71 +88,52 @@ public class SelfTestTest
// Mocks.
//
- private static class MockPathCopier implements IPathCopier
+ private static IPathCopier createMockCopier()
{
- private final boolean existRemote;
-
- File destinationDirectoryQueried;
-
- String destinationHostQueried;
-
- MockPathCopier(boolean existRemote)
- {
- this.existRemote = existRemote;
- }
-
- public Status copy(File sourcePath, File destinationDirectory)
- {
- throw new AssertionError();
- }
-
- public Status copy(File sourcePath, String sourceHost, File destinationDirectory, String destinationHost)
- {
- throw new AssertionError();
- }
-
- public boolean exists(File destinationDirectory, String destinationHost)
- {
- assert destinationDirectoryQueried == null;
- assert destinationHostQueried == null;
- assert destinationDirectory != null;
- assert destinationHost != null;
-
- destinationDirectoryQueried = destinationDirectory;
- destinationHostQueried = destinationHost;
- return existRemote;
- }
-
- public boolean terminate()
- {
- return false;
- }
-
- public void check()
- {
- }
-
+ return new IPathCopier()
+ {
+ public Status copy(File sourcePath, File destinationDirectory)
+ {
+ throw new AssertionError();
+ }
+
+ public Status copyFromRemote(File sourcePath, String sourceHost, File destinationDirectory)
+ {
+ throw new AssertionError();
+ }
+
+ public Status copyToRemote(File sourcePath, File destinationDirectory, String destinationHost)
+ {
+ throw new AssertionError();
+ }
+
+ public boolean existsRemotely(File destinationDirectory, String destinationHost)
+ {
+ throw new AssertionError();
+ }
+
+ public boolean terminate()
+ {
+ return true;
+ }
+
+ public void check() throws EnvironmentFailureException, ConfigurationFailureException
+ {
+ }
+
+ };
}
// ////////////////////////////////////////
// Test cases.
//
- @Test
- public void testHappyCaseWithRemoteShare()
- {
- SelfTest.check(mockCopier, incomingStore, bufferStore, outgoingStore, dummyStore);
- }
-
@Test
public void testHappyCaseWithRemoteHost()
{
final String outgoingHost = "some_remote_host";
- final FileStore remoteHostOutgoingStore = new FileStore(outgoingDirectory, "outgoing", outgoingHost, true);
- final MockPathCopier myMockCopier = new MockPathCopier(true);
- SelfTest.check(myMockCopier, incomingStore, bufferStore, remoteHostOutgoingStore, dummyStore);
- assert outgoingHost.equals(myMockCopier.destinationHostQueried);
- assert outgoingDirectory.equals(myMockCopier.destinationDirectoryQueried);
+ final FileStore remoteHostOutgoingStore = createRemoteStore(outgoingDirectory, outgoingHost, "outgoing");
+ SelfTest.check(mockCopier, remoteHostOutgoingStore);
}
@Test(expectedExceptions = ConfigurationFailureException.class)
@@ -164,7 +146,7 @@ public class SelfTestTest
public void testContainingPaths()
{
final File illegalBufferDirectory = new File(incomingDirectory, "temp");
- final FileStore illegalBufferStore = new FileStore(illegalBufferDirectory, "buffer", null, false);
+ final FileStore illegalBufferStore = createLocalStore(illegalBufferDirectory, "buffer");
SelfTest.check(mockCopier, incomingStore, illegalBufferStore, outgoingStore);
}
@@ -172,16 +154,18 @@ public class SelfTestTest
public void testNonExistentPaths()
{
final File nonExistentIncomingDirectory = new File(workingDirectory, "data");
- final FileStore nonExistentIncomingStore = new FileStore(nonExistentIncomingDirectory, "incoming", null, false);
+ final FileStore nonExistentIncomingStore = createLocalStore(nonExistentIncomingDirectory, "incoming");
SelfTest.check(mockCopier, nonExistentIncomingStore, bufferStore, outgoingStore);
}
- @Test(expectedExceptions = ConfigurationFailureException.class)
- public void testRemoteHostAndDirectoryDoesNotExist()
+ private FileStore createRemoteStore(File path, String host, String description)
+ {
+ return FileStoreFactory.createRemoteHost(path, host, description, FileOperationsUtil.createTestFatory());
+ }
+
+ private static FileStore createLocalStore(File path, String description)
{
- final String remoteHost = "some_remote_host";
- final FileStore outgoingStoreWithRemoteHost = new FileStore(outgoingDirectory, "outgoing", remoteHost, true);
- SelfTest.check(mockCopier, incomingStore, bufferStore, outgoingStoreWithRemoteHost);
+ return FileStoreFactory.createLocal(path, description, FileOperationsUtil.createTestFatory());
}
}
diff --git a/datamover/sourceTest/java/ch/systemsx/cisd/datamover/filesystem/impl/RecursiveHardLinkMakerTest.java b/datamover/sourceTest/java/ch/systemsx/cisd/datamover/filesystem/impl/RecursiveHardLinkMakerTest.java
index a46aa677872ed4cc505c85a8654b662b30eb25ec..8f1773492b9aac9ea8c01478f0c1d2e89a6b6333 100644
--- a/datamover/sourceTest/java/ch/systemsx/cisd/datamover/filesystem/impl/RecursiveHardLinkMakerTest.java
+++ b/datamover/sourceTest/java/ch/systemsx/cisd/datamover/filesystem/impl/RecursiveHardLinkMakerTest.java
@@ -31,7 +31,6 @@ import org.testng.annotations.Test;
import ch.systemsx.cisd.common.logging.LogInitializer;
import ch.systemsx.cisd.common.utilities.CollectionIO;
import ch.systemsx.cisd.common.utilities.FileUtilities;
-import ch.systemsx.cisd.datamover.filesystem.impl.RecursiveHardLinkMaker;
import ch.systemsx.cisd.datamover.filesystem.intf.IPathImmutableCopier;
/**
diff --git a/datamover/sourceTest/java/ch/systemsx/cisd/datamover/filesystem/remote/CopyActivityMonitorTest.java b/datamover/sourceTest/java/ch/systemsx/cisd/datamover/filesystem/remote/CopyActivityMonitorTest.java
index bf49a0061cdb2a089edb51b45b5a0d7cec7fefe6..c993b7b932b6a08923d4b4f7f12bbf0a861ccf17 100644
--- a/datamover/sourceTest/java/ch/systemsx/cisd/datamover/filesystem/remote/CopyActivityMonitorTest.java
+++ b/datamover/sourceTest/java/ch/systemsx/cisd/datamover/filesystem/remote/CopyActivityMonitorTest.java
@@ -17,7 +17,6 @@
package ch.systemsx.cisd.datamover.filesystem.remote;
import java.io.File;
-import java.io.FileFilter;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeClass;
@@ -25,17 +24,20 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import ch.systemsx.cisd.common.exceptions.CheckedExceptionTunnel;
-import ch.systemsx.cisd.common.logging.ISimpleLogger;
+import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.logging.LogInitializer;
import ch.systemsx.cisd.common.logging.LogMonitoringAppender;
import ch.systemsx.cisd.common.utilities.ITerminable;
import ch.systemsx.cisd.common.utilities.StoringUncaughtExceptionHandler;
-import ch.systemsx.cisd.datamover.filesystem.FileSysOperationsFactory;
-import ch.systemsx.cisd.datamover.filesystem.intf.IReadPathOperations;
-import ch.systemsx.cisd.datamover.intf.IFileSysParameters;
+import ch.systemsx.cisd.datamover.common.StoreItem;
+import ch.systemsx.cisd.datamover.filesystem.intf.FileStore;
+import ch.systemsx.cisd.datamover.filesystem.intf.IFileSysOperationsFactory;
+import ch.systemsx.cisd.datamover.filesystem.intf.IStoreCopier;
+import ch.systemsx.cisd.datamover.filesystem.store.FileStoreLocal;
import ch.systemsx.cisd.datamover.intf.ITimingParameters;
+import ch.systemsx.cisd.datamover.testhelper.FileOperationsUtil;
/**
* Test cases for the {@link CopyActivityMonitor} class.
@@ -84,63 +86,14 @@ public class CopyActivityMonitorTest
}
}
- private static class ReadOperationsOriginalImpl implements IReadPathOperations
+ private static interface LastChangedChecker
{
- private final IReadPathOperations impl;
-
- public ReadOperationsOriginalImpl()
- {
- final IFileSysParameters dummyFileSysParameters = new IFileSysParameters()
- {
- public String getHardLinkExecutable()
- {
- return null;
- }
-
- public String getRsyncExecutable()
- {
- return null;
- }
-
- public String getSshExecutable()
- {
- return null;
- }
-
- public boolean isRsyncOverwrite()
- {
- return false;
- }
-
- };
- this.impl = new FileSysOperationsFactory(dummyFileSysParameters).getReadPathOperations();
- }
-
- public long lastChanged(File path)
- {
- return impl.lastChanged(path);
- }
-
- public boolean exists(File file)
- {
- return impl.exists(file);
- }
-
- public File[] tryListFiles(File directory, FileFilter filter, ISimpleLogger loggerOrNull)
- {
- return impl.tryListFiles(directory, filter, loggerOrNull);
- }
-
- public File[] tryListFiles(File directory, ISimpleLogger logger)
- {
- return impl.tryListFiles(directory, logger);
- }
+ public long lastChanged(StoreItem item);
}
- private final class HappyPathLastChangedChecker extends ReadOperationsOriginalImpl
+ private final class HappyPathLastChangedChecker implements LastChangedChecker
{
- @Override
- public long lastChanged(File path)
+ public long lastChanged(StoreItem item)
{
return System.currentTimeMillis() - INACTIVITY_PERIOD_MILLIS / 2;
}
@@ -182,6 +135,55 @@ public class CopyActivityMonitorTest
}
}
+ private FileStore asFileStore(File directory, final LastChangedChecker checker)
+ {
+ IFileSysOperationsFactory factory = FileOperationsUtil.createTestFatory();
+ return asFileStore(directory, checker, factory);
+ }
+
+ private FileStore asFileStore(File directory, final LastChangedChecker checker, IFileSysOperationsFactory factory)
+ {
+ final FileStoreLocal localImpl = new FileStoreLocal(directory, "input-test", factory);
+ return new FileStore(directory, null, false, "input-test", factory)
+ {
+ @Override
+ public Status delete(StoreItem item)
+ {
+ return localImpl.delete(item);
+ }
+
+ @Override
+ public boolean exists(StoreItem item)
+ {
+ return localImpl.exists(item);
+ }
+
+ @Override
+ public long lastChanged(StoreItem item)
+ {
+ return checker.lastChanged(item);
+ }
+
+ @Override
+ public String tryCheckDirectoryFullyAccessible()
+ {
+ return localImpl.tryCheckDirectoryFullyAccessible();
+ }
+
+ @Override
+ public ExtendedFileStore tryAsExtended()
+ {
+ return localImpl.tryAsExtended();
+ }
+
+ @Override
+ public IStoreCopier getCopier(FileStore destinationDirectory)
+ {
+ return localImpl.getCopier(destinationDirectory);
+ }
+ };
+ }
+
// ////////////////////////////////////////
// Initialization methods.
//
@@ -218,15 +220,13 @@ public class CopyActivityMonitorTest
{ "slow" })
public void testHappyPath() throws Throwable
{
- final IReadPathOperations checker = new HappyPathLastChangedChecker();
+ final LastChangedChecker checker = new HappyPathLastChangedChecker();
final ITerminable dummyTerminable = new DummyTerminable();
final ITimingParameters parameters = new MyTimingParameters(0);
final CopyActivityMonitor monitor =
- new CopyActivityMonitor(workingDirectory, checker, dummyTerminable, parameters);
- final File directory = new File(workingDirectory, "some-directory");
- directory.mkdir();
- directory.deleteOnExit();
- monitor.start(directory);
+ new CopyActivityMonitor(asFileStore(workingDirectory, checker), dummyTerminable, parameters);
+ StoreItem item = createDirectoryInside(workingDirectory);
+ monitor.start(item);
Thread.sleep(INACTIVITY_PERIOD_MILLIS * 15);
monitor.stop();
}
@@ -235,24 +235,23 @@ public class CopyActivityMonitorTest
{ "slow" })
public void testCopyStalled() throws Throwable
{
- final IReadPathOperations checker = new PathLastChangedCheckerStalled();
+ final LastChangedChecker checker = new PathLastChangedCheckerStalled();
final MockTerminable copyProcess = new MockTerminable();
final ITimingParameters parameters = new MyTimingParameters(0);
- final CopyActivityMonitor monitor = new CopyActivityMonitor(workingDirectory, checker, copyProcess, parameters);
- final File file = new File(workingDirectory, "some-directory");
- file.mkdir();
- monitor.start(file);
+ final CopyActivityMonitor monitor =
+ new CopyActivityMonitor(asFileStore(workingDirectory, checker), copyProcess, parameters);
+ StoreItem item = createDirectoryInside(workingDirectory);
+ monitor.start(item);
Thread.sleep(INACTIVITY_PERIOD_MILLIS * 15);
monitor.stop();
assert copyProcess.isTerminated();
}
- private final class SimulateShortInterruptionChangedChecker extends ReadOperationsOriginalImpl
+ private final class SimulateShortInterruptionChangedChecker implements LastChangedChecker
{
private int numberOfTimesCalled = 0;
- @Override
- public long lastChanged(File path)
+ public long lastChanged(StoreItem item)
{
++numberOfTimesCalled;
if (numberOfTimesCalled == 2)
@@ -269,33 +268,32 @@ public class CopyActivityMonitorTest
}
/**
- * This test case catches a case that I first hadn't thought of: since we use <code>rsync</code> (or
- * <code>xcopy</code>) in a mode where at the end of copying a file they set the "last modified" time back to the
- * one of the source file, there is a short time interval after finishing copying one file anst starting copying the
- * next file where the copy monitor could be tempted to trigger false alarm: the just finished file will have
- * already the "last modified" time of the source file (which is when the data produce finished writing the source
- * file). In fact everything is fine but still the copy process will be cancelled.
+ * This test case catches a case that I first hadn't thought of: since we use <code>rsync</code> in a mode where
+ * at the end of copying a file they set the "last modified" time back to the one of the source file, there is a
+ * short time interval after finishing copying one file anst starting copying the next file where the copy monitor
+ * could be tempted to trigger false alarm: the just finished file will have already the "last modified" time of the
+ * source file (which is when the data produce finished writing the source file). In fact everything is fine but
+ * still the copy process will be cancelled.
*/
@Test(groups =
{ "slow" })
public void testCopySeemsStalledButActuallyIsFine() throws Throwable
{
- final IReadPathOperations checker = new SimulateShortInterruptionChangedChecker();
+ final LastChangedChecker checker = new SimulateShortInterruptionChangedChecker();
final MockTerminable copyProcess = new MockTerminable();
final ITimingParameters parameters = new MyTimingParameters(0);
- final CopyActivityMonitor monitor = new CopyActivityMonitor(workingDirectory, checker, copyProcess, parameters);
- final File file = new File(workingDirectory, "some-directory");
- file.mkdir();
- monitor.start(file);
+ final CopyActivityMonitor monitor =
+ new CopyActivityMonitor(asFileStore(workingDirectory, checker), copyProcess, parameters);
+ StoreItem item = createDirectoryInside(workingDirectory);
+ monitor.start(item);
Thread.sleep(INACTIVITY_PERIOD_MILLIS * 15);
monitor.stop();
assert copyProcess.isTerminated() == false;
}
- private final class PathLastChangedCheckerStalled extends ReadOperationsOriginalImpl
+ private final class PathLastChangedCheckerStalled implements LastChangedChecker
{
- @Override
- public long lastChanged(File path)
+ public long lastChanged(StoreItem item)
{
return System.currentTimeMillis() - INACTIVITY_PERIOD_MILLIS * 2;
}
@@ -308,14 +306,13 @@ public class CopyActivityMonitorTest
LogMonitoringAppender appender =
LogMonitoringAppender.addAppender(LogCategory.OPERATION, "Activity monitor got terminated");
LogFactory.getLogger(LogCategory.OPERATION, CopyActivityMonitor.class).addAppender(appender);
- final IReadPathOperations checker = new PathLastChangedCheckerStuck();
+ final LastChangedChecker checker = new PathLastChangedCheckerStuck();
final MockTerminable copyProcess = new MockTerminable();
final ITimingParameters parameters = new MyTimingParameters(0);
- final CopyActivityMonitor monitor = new CopyActivityMonitor(workingDirectory, checker, copyProcess, parameters);
- final File directory = new File(workingDirectory, "some-directory");
- directory.mkdir();
- directory.deleteOnExit();
- monitor.start(directory);
+ final CopyActivityMonitor monitor =
+ new CopyActivityMonitor(asFileStore(workingDirectory, checker), copyProcess, parameters);
+ StoreItem item = createDirectoryInside(workingDirectory);
+ monitor.start(item);
Thread.sleep(INACTIVITY_PERIOD_MILLIS * 15);
monitor.stop();
LogMonitoringAppender.removeAppender(appender);
@@ -323,12 +320,20 @@ public class CopyActivityMonitorTest
appender.verifyLogHasHappened();
}
- private final class PathLastChangedCheckerStuck extends ReadOperationsOriginalImpl
+ private StoreItem createDirectoryInside(File parentDir)
+ {
+ StoreItem item = new StoreItem("some-directory");
+ final File directory = new File(parentDir, item.getName());
+ directory.mkdir();
+ directory.deleteOnExit();
+ return item;
+ }
+
+ private final class PathLastChangedCheckerStuck implements LastChangedChecker
{
private boolean interrupted = false;
- @Override
- public long lastChanged(File path)
+ public long lastChanged(StoreItem item)
{
try
{
diff --git a/datamover/sourceTest/java/ch/systemsx/cisd/datamover/filesystem/remote/rsync/RsyncCopierTest.java b/datamover/sourceTest/java/ch/systemsx/cisd/datamover/filesystem/remote/rsync/RsyncCopierTest.java
index 0f341df87852c677961dd662a392e1ee5695d7b8..22c20afe83522544b08f243880c40221381a6008 100644
--- a/datamover/sourceTest/java/ch/systemsx/cisd/datamover/filesystem/remote/rsync/RsyncCopierTest.java
+++ b/datamover/sourceTest/java/ch/systemsx/cisd/datamover/filesystem/remote/rsync/RsyncCopierTest.java
@@ -36,7 +36,6 @@ import ch.systemsx.cisd.common.logging.LogInitializer;
import ch.systemsx.cisd.common.utilities.CollectionIO;
import ch.systemsx.cisd.common.utilities.FileUtilities;
import ch.systemsx.cisd.common.utilities.StoringUncaughtExceptionHandler;
-import ch.systemsx.cisd.datamover.filesystem.remote.rsync.RsyncCopier;
/**
* Test cases for the {@link RsyncCopier} class.
@@ -103,8 +102,8 @@ public class RsyncCopierTest
{ "requires_unix" })
public void testRsyncOK() throws IOException, InterruptedException
{
- final File dummyRsyncBinary = createRsync(0);
- final RsyncCopier copier = new RsyncCopier(dummyRsyncBinary, null, false, false);
+ final File buggyRsyncBinary = createRsync(0);
+ final RsyncCopier copier = new RsyncCopier(buggyRsyncBinary, null, false, false);
Status status = copier.copy(sourceFile, destinationDirectory);
assert Status.OK == status;
}
@@ -114,8 +113,8 @@ public class RsyncCopierTest
public void testRsyncRetriableFailure() throws IOException, InterruptedException
{
final int exitValue = 11;
- final File dummyRsyncBinary = createRsync(exitValue);
- final RsyncCopier copier = new RsyncCopier(dummyRsyncBinary, null, false, false);
+ final File buggyRsyncBinary = createRsync(exitValue);
+ final RsyncCopier copier = new RsyncCopier(buggyRsyncBinary, null, false, false);
Status status = copier.copy(sourceFile, destinationDirectory);
assertEquals(StatusFlag.RETRIABLE_ERROR, status.getFlag());
assertEquals(RsyncExitValueTranslator.getMessage(exitValue), status.getMessage());
@@ -126,8 +125,8 @@ public class RsyncCopierTest
public void testRsyncFatalFailure() throws IOException, InterruptedException
{
final int exitValue = 1;
- final File dummyRsyncBinary = createRsync(exitValue);
- final RsyncCopier copier = new RsyncCopier(dummyRsyncBinary, null, false, false);
+ final File buggyRsyncBinary = createRsync(exitValue);
+ final RsyncCopier copier = new RsyncCopier(buggyRsyncBinary, null, false, false);
Status status = copier.copy(sourceFile, destinationDirectory);
assertEquals(StatusFlag.FATAL_ERROR, status.getFlag());
assertEquals(RsyncExitValueTranslator.getMessage(exitValue), status.getMessage());
diff --git a/datamover/sourceTest/java/ch/systemsx/cisd/datamover/testhelper/FileOperationsUtil.java b/datamover/sourceTest/java/ch/systemsx/cisd/datamover/testhelper/FileOperationsUtil.java
new file mode 100644
index 0000000000000000000000000000000000000000..d5d30bc3552f98d034c40e3b97b3615a918c6617
--- /dev/null
+++ b/datamover/sourceTest/java/ch/systemsx/cisd/datamover/testhelper/FileOperationsUtil.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.datamover.testhelper;
+
+import ch.systemsx.cisd.datamover.filesystem.FileSysOperationsFactory;
+import ch.systemsx.cisd.datamover.filesystem.intf.IFileSysOperationsFactory;
+import ch.systemsx.cisd.datamover.intf.IFileSysParameters;
+
+/**
+ * @author Tomasz Pylak on Oct 11, 2007
+ */
+public class FileOperationsUtil
+{
+ public final static IFileSysOperationsFactory createTestFatory()
+ {
+ return new FileSysOperationsFactory(createDummyFileSysParameters());
+ }
+
+ private static IFileSysParameters createDummyFileSysParameters()
+ {
+ return new IFileSysParameters()
+ {
+ public String getHardLinkExecutable()
+ {
+ return null;
+ }
+
+ public String getRsyncExecutable()
+ {
+ return null;
+ }
+
+ public String getSshExecutable()
+ {
+ return null;
+ }
+
+ public boolean isRsyncOverwrite()
+ {
+ return false;
+ }
+ };
+ }
+}
diff --git a/datamover/sourceTest/java/ch/systemsx/cisd/datamover/testhelper/FileStructEngine.java b/datamover/sourceTest/java/ch/systemsx/cisd/datamover/testhelper/FileStructEngine.java
index 6230616db2e02f673198371991e463f79fc6b9a5..04baf32946eed8d99376827113ec87cd044a4f4e 100644
--- a/datamover/sourceTest/java/ch/systemsx/cisd/datamover/testhelper/FileStructEngine.java
+++ b/datamover/sourceTest/java/ch/systemsx/cisd/datamover/testhelper/FileStructEngine.java
@@ -16,17 +16,21 @@
package ch.systemsx.cisd.datamover.testhelper;
+import static ch.systemsx.cisd.datamover.testhelper.FileSystemHelper.assertDirExists;
+import static ch.systemsx.cisd.datamover.testhelper.FileSystemHelper.createDir;
+import static ch.systemsx.cisd.datamover.testhelper.FileSystemHelper.createEmptyFile;
+import static ch.systemsx.cisd.datamover.testhelper.FileSystemHelper.createFile;
+import static org.testng.AssertJUnit.assertTrue;
+
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import static org.testng.AssertJUnit.*;
-import static ch.systemsx.cisd.datamover.testhelper.FileSystemHelper.*;
-
import ch.systemsx.cisd.common.utilities.CollectionIO;
import ch.systemsx.cisd.datamover.common.MarkerFile;
+import ch.systemsx.cisd.datamover.common.StoreItem;
/**
* Immutable helper for creating a sample directory structure and manipulating it.
@@ -87,7 +91,8 @@ public class FileStructEngine
private static File createDeletionInProgressMarkerFile(File parentDir, String originalName)
{
- return MarkerFile.createDeletionInProgressMarker(parentDir, originalName);
+ StoreItem marker = MarkerFile.createDeletionInProgressMarker(new StoreItem(originalName));
+ return new File(parentDir, marker.getName());
}
private static List<String> createSampleFileContent()