From 3636224c227fba437aad985caec21100000d3325 Mon Sep 17 00:00:00 2001
From: brinn <brinn>
Date: Fri, 31 Aug 2007 15:30:03 +0000
Subject: [PATCH] move: IntraFSPathMover to datamover, since it is now more
datamover specific fix: recovery problem in copy_complete when incoming is
local add: support for prefix-for-incoming parameter (DMV-6)
SVN: 1566
---
datamover/dist/etc/service.properties | 4 +-
.../cisd/datamover/IntraFSPathMover.java | 84 +++++++++++++++++++
.../cisd/datamover/MonitorStarter.java | 33 ++++----
.../systemsx/cisd/datamover/Parameters.java | 18 ++++
.../datamover/helper/FileSystemHelper.java | 58 +++++++++++--
5 files changed, 173 insertions(+), 24 deletions(-)
create mode 100644 datamover/source/java/ch/systemsx/cisd/datamover/IntraFSPathMover.java
diff --git a/datamover/dist/etc/service.properties b/datamover/dist/etc/service.properties
index 4ad21ade5f4..d8c061369b2 100644
--- a/datamover/dist/etc/service.properties
+++ b/datamover/dist/etc/service.properties
@@ -6,6 +6,7 @@ incoming-dir = data/incoming
buffer-dir = data/buffer
outgoing-dir = data/outgoing
manual-intervention-dir = data/manual_intervention
+prefix-for-incoming = %t_
#
# Optional (remove comments when changing the values)
@@ -23,4 +24,5 @@ manual-intervention-dir = data/manual_intervention
# cleansing-regex = <regex of paths that should be removed before moving a path to outgoing>
# manual-intervention-regex = <regex of paths that need manual intervention>
# treat-incoming-as-remote = <true or false, when switched on, than incoming directory is treated as remote>
-# extra-copy-dir = <path to a directory. If specified, a copy of incoming data will be made there>
\ No newline at end of file
+# extra-copy-dir = <path to a directory. If specified, a copy of incoming data will be made there>
+# prefix-for-incoming = prefix that is put in front of every incoming data directory, %t will be replaced with time stamp
\ No newline at end of file
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/IntraFSPathMover.java b/datamover/source/java/ch/systemsx/cisd/datamover/IntraFSPathMover.java
new file mode 100644
index 00000000000..3e74872d603
--- /dev/null
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/IntraFSPathMover.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.datamover;
+
+import java.io.File;
+
+import org.apache.log4j.Logger;
+
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.utilities.DirectoryScanningTimerTask;
+import ch.systemsx.cisd.common.utilities.FileUtilities;
+import ch.systemsx.cisd.datamover.helper.FileSystemHelper;
+
+/**
+ * A {@link DirectoryScanningTimerTask.IPathHandler} that moves paths out of the way within one file system by calling
+ * {@link File#renameTo(File)}..
+ *
+ * @author Bernd Rinn
+ */
+public class IntraFSPathMover implements DirectoryScanningTimerTask.IPathHandler
+{
+
+ private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION, IntraFSPathMover.class);
+
+ private static final Logger notificationLog = LogFactory.getLogger(LogCategory.NOTIFY, IntraFSPathMover.class);
+
+ private final File destinationDirectory;
+
+ private final String prefixTemplate;
+
+ /**
+ * Creates a <var>PathMover</var>.
+ *
+ * @param destinationDirectory The directory to move paths to.
+ */
+ public IntraFSPathMover(File destinationDirectory, String prefixTemplate)
+ {
+ assert destinationDirectory != null;
+ assert FileUtilities.checkDirectoryFullyAccessible(destinationDirectory, "destination") == null : FileUtilities
+ .checkDirectoryFullyAccessible(destinationDirectory, "destination");
+ assert prefixTemplate != null;
+
+ this.destinationDirectory = destinationDirectory;
+ this.prefixTemplate = prefixTemplate;
+ }
+
+ public boolean handle(File path)
+ {
+ assert path != null;
+ assert destinationDirectory != null;
+ assert prefixTemplate != null;
+
+ final String destinationPath =
+ FileSystemHelper.createDestinationPath(path, null, destinationDirectory, prefixTemplate);
+ if (operationLog.isInfoEnabled())
+ {
+ operationLog
+ .info(String.format("Moving path '%s' to '%s'", path.getPath(), destinationPath));
+ }
+ boolean movedOK = path.renameTo(new File(destinationPath));
+ if (movedOK == false)
+ {
+ notificationLog.error(String.format("Moving path '%s' to directory '%s' failed.", path,
+ destinationDirectory));
+ }
+ return movedOK;
+ }
+
+}
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/MonitorStarter.java b/datamover/source/java/ch/systemsx/cisd/datamover/MonitorStarter.java
index e40683e81f9..7a74d203f11 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/MonitorStarter.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/MonitorStarter.java
@@ -137,18 +137,13 @@ public class MonitorStarter
private void recoverIncomingAfterShutdown(FileStore incomingStore, IReadPathOperations incomingReadOperations,
boolean isIncomingRemote, LazyPathHandler localProcessor)
{
- if (isIncomingRemote == false)
- {
- return; // no recovery is needed
- }
-
recoverIncomingInProgress(incomingStore, incomingReadOperations, bufferDirs.getCopyInProgressDir(), bufferDirs
- .getCopyCompleteDir());
+ .getCopyCompleteDir(), parameters.getPrefixForIncoming());
recoverIncomingCopyComplete(bufferDirs.getCopyCompleteDir(), localProcessor);
}
private static void recoverIncomingInProgress(FileStore incomingStore, IReadPathOperations incomingReadOperations,
- File copyInProgressDir, File copyCompleteDir)
+ File copyInProgressDir, File copyCompleteDir, String prefixTemplate)
{
final File[] files = FileSystemHelper.listFiles(copyInProgressDir);
if (files == null || files.length == 0)
@@ -158,12 +153,12 @@ public class MonitorStarter
for (File file : files)
{
- recoverIncomingAfterShutdown(file, incomingStore, incomingReadOperations, copyCompleteDir);
+ recoverIncomingAfterShutdown(file, incomingStore, incomingReadOperations, copyCompleteDir, prefixTemplate);
}
}
private static void recoverIncomingAfterShutdown(File unfinishedFile, FileStore incomingStore,
- IReadPathOperations incomingReadOperations, File copyCompleteDir)
+ IReadPathOperations incomingReadOperations, File copyCompleteDir, String prefixTemplate)
{
if (CopyFinishedMarker.isMarker(unfinishedFile))
{
@@ -185,7 +180,7 @@ public class MonitorStarter
if (markerFile.exists())
{
// copy and marker exist - copy finished, but copied resource not moved
- tryMoveFromInProgressToFinished(localCopy, markerFile, copyCompleteDir);
+ tryMoveFromInProgressToFinished(localCopy, markerFile, copyCompleteDir, prefixTemplate);
} else
// no marker
{
@@ -197,7 +192,7 @@ public class MonitorStarter
} else
{
// move finished, but marker not created
- tryMoveFromInProgressToFinished(localCopy, null, copyCompleteDir);
+ tryMoveFromInProgressToFinished(localCopy, null, copyCompleteDir, prefixTemplate);
}
}
}
@@ -238,7 +233,7 @@ public class MonitorStarter
private boolean moveFromLocalIncoming(File source, LazyPathHandler localProcessor)
{
- final File finalFile = tryMoveLocal(source, bufferDirs.getCopyCompleteDir());
+ final File finalFile = tryMoveLocal(source, bufferDirs.getCopyCompleteDir(), parameters.getPrefixForIncoming());
if (finalFile == null)
{
return false;
@@ -255,13 +250,16 @@ public class MonitorStarter
{
return false;
}
+ FileSystemHelper.createDestinationPath(source, null, copyInProgressDir, parameters.getPrefixForIncoming());
final File copiedFile = new File(copyInProgressDir, source.getName());
assert copiedFile.exists();
final File markerFile = CopyFinishedMarker.extractMarker(copiedFile);
assert markerFile.exists();
// 2. Move to final directory, delete marker
- final File finalFile = tryMoveFromInProgressToFinished(copiedFile, markerFile, bufferDirs.getCopyCompleteDir());
+ final File finalFile =
+ tryMoveFromInProgressToFinished(copiedFile, markerFile, bufferDirs.getCopyCompleteDir(), parameters
+ .getPrefixForIncoming());
if (finalFile == null)
{
return false;
@@ -272,9 +270,10 @@ public class MonitorStarter
return true;
}
- private static File tryMoveFromInProgressToFinished(File copiedFile, File markerFileOrNull, File copyCompleteDir)
+ private static File tryMoveFromInProgressToFinished(File copiedFile, File markerFileOrNull, File copyCompleteDir,
+ String prefixTemplate)
{
- final File finalFile = tryMoveLocal(copiedFile, copyCompleteDir);
+ final File finalFile = tryMoveLocal(copiedFile, copyCompleteDir, prefixTemplate);
if (finalFile != null)
{
if (markerFileOrNull != null)
@@ -293,9 +292,9 @@ public class MonitorStarter
return createRemotePathMover(sourceHostOrNull, localDestDir, null).handle(source);
}
- private static File tryMoveLocal(File sourceFile, File destinationDir)
+ private static File tryMoveLocal(File sourceFile, File destinationDir, String prefixTemplate)
{
- return FileSystemHelper.tryMoveLocal(sourceFile, destinationDir);
+ return FileSystemHelper.tryMoveLocal(sourceFile, destinationDir, prefixTemplate);
}
// --------------------------
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/Parameters.java b/datamover/source/java/ch/systemsx/cisd/datamover/Parameters.java
index 9b1b5a9bc08..c476e6d144b 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/Parameters.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/Parameters.java
@@ -232,6 +232,14 @@ public class Parameters implements ITimingParameters
+ "incoming paths needs manual intervention. ")
private Pattern manualInterventionRegex = null;
+ /**
+ * A prefix for all incoming items. Note that '%t' will be replaced with the current timestamp in format
+ * 'yyyyMMddHHmmss'.
+ */
+ @Option(longName = "prefix-for-incoming", usage = "A string that all incoming items will be prepended with, "
+ + "'%t' will be replaced with the current time stamp.")
+ private String prefixForIncoming;
+
/**
* The command line parser.
*/
@@ -366,6 +374,7 @@ public class Parameters implements ITimingParameters
treatIncomingAsRemote =
Boolean.parseBoolean(serviceProperties.getProperty("treat-incoming-as-remote", Boolean
.toString(DEFAULT_TREAT_INCOMING_AS_REMOTE)));
+ prefixForIncoming = serviceProperties.getProperty("prefix-for-incoming", "");
if (serviceProperties.getProperty("incoming-dir") != null)
{
incomingDirectory = new File(serviceProperties.getProperty("incoming-dir"));
@@ -574,6 +583,15 @@ public class Parameters implements ITimingParameters
return manualInterventionRegex;
}
+ /**
+ * @return The prefix string to put in front of all incoming items. Note that '%t' will be replaced with the current
+ * time stamp.
+ */
+ public String getPrefixForIncoming()
+ {
+ return prefixForIncoming;
+ }
+
/**
* Logs the current parameters to the {@link LogCategory#OPERATION} log.
*/
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/helper/FileSystemHelper.java b/datamover/source/java/ch/systemsx/cisd/datamover/helper/FileSystemHelper.java
index 4f33ca6565e..cfcde8a366f 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/helper/FileSystemHelper.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/helper/FileSystemHelper.java
@@ -19,6 +19,8 @@ package ch.systemsx.cisd.datamover.helper;
import java.io.File;
import java.io.FileFilter;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.time.DateFormatUtils;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
@@ -28,7 +30,7 @@ import ch.systemsx.cisd.common.logging.Log4jSimpleLogger;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.utilities.FileUtilities;
-import ch.systemsx.cisd.common.utilities.IntraFSPathMover;
+import ch.systemsx.cisd.datamover.IntraFSPathMover;
import ch.systemsx.cisd.datamover.LocalProcessorHandler;
import ch.systemsx.cisd.datamover.intf.IReadPathOperations;
@@ -98,12 +100,56 @@ public class FileSystemHelper
};
}
- /** moves source file to destination directory */
+ /**
+ * Moves source file to destination directory.
+ */
public static File tryMoveLocal(File sourceFile, File destinationDir)
{
- boolean ok = new IntraFSPathMover(destinationDir).handle(sourceFile);
- if (!ok)
+ return tryMoveLocal(sourceFile, destinationDir, "");
+ }
+
+ /**
+ * Moves source file to destination directory, putting <var>prefixTemplate</var> in front of its name after
+ * replacing '%t' with the current time stamp.
+ */
+ public static File tryMoveLocal(File sourceFile, File destinationDirectory, String prefixTemplate)
+ {
+ boolean ok = new IntraFSPathMover(destinationDirectory, prefixTemplate).handle(sourceFile);
+ if (ok == false)
+ {
return null;
- return new File(destinationDir, sourceFile.getName());
+ }
+ return new File(createDestinationPath(sourceFile, null, destinationDirectory, prefixTemplate));
}
-}
+
+ /**
+ * Creates a destination path for copying <var>sourcePath</var> to <var>destinationDirectory</var> with prefix
+ * defined by <var>prefixTemplate</var>. Note that '%t' in <var>prefixTemplate</var> will be replaced by the
+ * current time stamp in format YYYYmmddhhMMss.
+ */
+ public static String createDestinationPath(File sourcePath, String destinationHostOrNull,
+ File destinationDirectory, String prefixTemplate)
+ {
+ assert sourcePath != null;
+ assert destinationDirectory != null;
+ assert prefixTemplate != null;
+
+ if (destinationHostOrNull != null)
+ {
+ return destinationHostOrNull + ":" + destinationDirectory.getPath() + File.separator
+ + createPrefix(prefixTemplate) + sourcePath.getName();
+
+ } else
+ {
+ return destinationDirectory.getAbsolutePath() + File.separator + createPrefix(prefixTemplate)
+ + sourcePath.getName();
+ }
+ }
+
+ private static String createPrefix(String prefixTemplate)
+ {
+ return StringUtils.replace(prefixTemplate, "%t", DateFormatUtils.format(System.currentTimeMillis(),
+ "yyyyMMddHHmmss"));
+ }
+
+}
\ No newline at end of file
--
GitLab