diff --git a/datamover/.classpath b/datamover/.classpath index 2895c606f4f620f1f4082a610d28c8cae905e8db..20b4eb8efd8f59eb3b7d3d4762ee7cd87d6bdde0 100644 --- a/datamover/.classpath +++ b/datamover/.classpath @@ -10,5 +10,6 @@ <classpathentry combineaccessrules="false" kind="src" path="/common"/> <classpathentry kind="lib" path="/libraries/args4j/args4j.jar" sourcepath="/libraries/args4j/args4j-src.jar"/> <classpathentry kind="lib" path="/libraries/commons-io/commons-io.jar" sourcepath="/libraries/commons-io/src.zip"/> + <classpathentry kind="lib" path="/libraries/commons-lang/commons-lang.jar" sourcepath="/libraries/commons-lang/src.zip"/> <classpathentry kind="output" path="targets/classes"/> </classpath> diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/FileStore.java b/datamover/source/java/ch/systemsx/cisd/datamover/FileStore.java new file mode 100644 index 0000000000000000000000000000000000000000..b6b6932b5506f8232209871f1d63e7adc0614393 --- /dev/null +++ b/datamover/source/java/ch/systemsx/cisd/datamover/FileStore.java @@ -0,0 +1,118 @@ +/* + * Copyright 2007 ETH Zuerich, CISD + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.systemsx.cisd.datamover; + +import java.io.File; +import java.io.IOException; + +import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException; + +/** + * A class to holds the information about a file store. + * + * @author Bernd Rinn + */ +public class FileStore +{ + private final File path; + + private final String canonicalPath; + + private final String kind; + + private final String host; + + private final boolean remote; + + FileStore(File path, String kind, String host, boolean remote) + { + this.path = path; + this.kind = kind; + if (path != null) + { + this.canonicalPath = getCanonicalPath(path, host); + } else + { + this.canonicalPath = null; + } + this.host = host; + this.remote = remote; + } + + private static String getCanonicalPath(File path, String host) + { + if (host != null) + { + return path.getPath(); + } + try + { + return path.getCanonicalPath() + File.separator; + } catch (IOException e) + { + throw EnvironmentFailureException.fromTemplate(e, "Cannot determine canonical form of path '%s'", path + .getPath()); + } + } + + /** + * Returns the remote host, if copying is done via an ssh tunnel, or <code>null</code> otherwise. + */ + public String getHost() + { + return host; + } + + /** + * Returns the path of the file store. If {@link #getHost()} is <code>null</code>, and only then, it may be + * interpreted to be on the local host. + */ + public File getPath() + { + return path; + } + + /** + * Returns the canonical form of the path as a string, if {@link #getHost()} is <code>null</code>. Note that the + * canonical name is only available if {@link #getHost()} is <code>null</code>, otherwise the normal path (as + * returned by {@link File#getPath()}) will be returned by this method. + */ + public String getCanonicalPath() + { + return canonicalPath; + } + + /** + * Returns the kind of file store (used for error and log messages). + */ + public String getKind() + { + return kind; + } + + /** + * Returns <code>true</code>, if the file store resides on a remote computer and <code>false</code> otherwise. + * <p> + * Note that this method can return <code>true</code> despite {@link #getHost()} returning <code>null</code>. + * In this case the file store is on a remote share mounted on local host via NFS or CIFS. + */ + public boolean isRemote() + { + return remote; + } + +} \ No newline at end of file diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/Main.java b/datamover/source/java/ch/systemsx/cisd/datamover/Main.java index 78b114e7b1cb28c409de90047c550780599e30e2..618f9840b96076abbbf7726b19b67f4ec04e7ab5 100644 --- a/datamover/source/java/ch/systemsx/cisd/datamover/Main.java +++ b/datamover/source/java/ch/systemsx/cisd/datamover/Main.java @@ -142,18 +142,13 @@ public class Main */ private static IPathCopier getPathCopier(final Parameters parameters) { - final File incomingDirectory = parameters.getIncomingDirectory(); - final File bufferDirectory = parameters.getBufferDirectory(); - final File outgoingDirectory = parameters.getOutgoingDirectory(); - final File manualInterventionDirectory = parameters.getManualInterventionDirectory(); - final String outgoingHost = parameters.getOutgoingHost(); IPathCopier copyProcess = null; // Convince Eclipse compiler that the variable has been initialized. try { copyProcess = suggestPathCopier(parameters, false); // This is part of the self-test. - SelfTest.check(incomingDirectory, bufferDirectory, outgoingDirectory, manualInterventionDirectory, - outgoingHost, copyProcess); + SelfTest.check(copyProcess, parameters.getIncomingStore(), parameters.getBufferStore(), parameters + .getOutgoingStore(), parameters.getManualInterventionStore()); } catch (HighLevelException e) { System.err.printf("Self test failed: [%s: %s]\n", e.getClass().getSimpleName(), e.getMessage()); @@ -164,7 +159,8 @@ public class Main e.printStackTrace(); System.exit(1); } - if (SelfTest.requiresDeletionBeforeCreation(copyProcess, bufferDirectory, outgoingDirectory)) + if (SelfTest.requiresDeletionBeforeCreation(copyProcess, parameters.getBufferStore().getPath(), parameters + .getOutgoingStore().getPath())) { copyProcess = suggestPathCopier(parameters, true); } @@ -173,8 +169,8 @@ public class Main private static void startupIncomingMovingProcess(final Parameters parameters, final IFileSystemOperations operations) { - final File incomingDirectory = parameters.getIncomingDirectory(); - final File bufferDirectory = parameters.getBufferDirectory(); + final File incomingDirectory = parameters.getIncomingStore().getPath(); + final File bufferDirectory = parameters.getBufferStore().getPath(); final File manualInterventionDirectory = parameters.getManualInterventionDirectory(); final RegexFileFilter cleansingFilter = new RegexFileFilter(); if (parameters.getCleansingRegex() != null) @@ -200,9 +196,9 @@ public class Main private static void startupOutgoingMovingProcess(final Parameters parameters, final IFileSystemOperations operations) { - final File bufferDirectory = parameters.getBufferDirectory(); - final File outgoingDirectory = parameters.getOutgoingDirectory(); - final String outgoingHost = parameters.getOutgoingHost(); + final File bufferDirectory = parameters.getBufferStore().getPath(); + final File outgoingDirectory = parameters.getOutgoingStore().getPath(); + final String outgoingHost = parameters.getOutgoingStore().getHost(); final CopyActivityMonitor monitor = new CopyActivityMonitor(outgoingDirectory, operations, operations.getCopier(), parameters); final IPathHandler remoteMover = diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/Parameters.java b/datamover/source/java/ch/systemsx/cisd/datamover/Parameters.java index 9db765177a62d2da676a03a3b7a545d3bb977b40..933290f471fc5a80468dbe9e8cd02aba9b10cb07 100644 --- a/datamover/source/java/ch/systemsx/cisd/datamover/Parameters.java +++ b/datamover/source/java/ch/systemsx/cisd/datamover/Parameters.java @@ -175,7 +175,27 @@ public class Parameters implements ITimingParameters private Pattern cleansingRegex = null; /** - * The regular expression to use for deciding whether a path in the incoming directory needs manual intervetion. + * The store where the files come in. + */ + private final FileStore incomingStore; + + /** + * The store to buffer the files before copying to outgoing. + */ + private final FileStore bufferStore; + + /** + * The store to copy the files to. + */ + private final FileStore outgoingStore; + + /** + * The store to copy files to that need manual intervention. + */ + private final FileStore manualInterventionStore; + + /** + * The regular expression to use for deciding whether a path in the incoming directory needs manual intervention. */ @Option(longName = "manual-intervention-regex", usage = "The regular expression to use for deciding whether an " + "incoming paths needs manual intervention. ") @@ -263,6 +283,10 @@ public class Parameters implements ITimingParameters throw new ConfigurationFailureException( "No 'manual-intervention-dir' defined, but 'manual-intervention-regex'."); } + incomingStore = new FileStore(incomingDirectory, "incoming", null, false); + bufferStore = new FileStore(bufferDirectory, "buffer", null, false); + manualInterventionStore = new FileStore(manualInterventionDirectory, "manual intervention", null, false); + outgoingStore = new FileStore(outgoingDirectory, "outgoing", outgoingHost, true); } catch (Exception ex) { outputException(ex); @@ -421,47 +445,49 @@ public class Parameters implements ITimingParameters } /** - * @return The (local) directory to monitor for new files and directories to move to outgoing. + * @return The store to monitor for new files and directories to move to the buffer. */ - public File getIncomingDirectory() + public FileStore getIncomingStore() { - return incomingDirectory; + return incomingStore; } /** - * @return The directory to move files and directories to that have been quiet in the local data directory for long + * @return The store to move files and directories to that have been quiet in the incoming directory for long * enough and thus are considered to be ready to be moved to remote. Note that this directory needs to be on - * the same file system than {@link #getIncomingDirectory}. + * the same file system as {@link #getIncomingStore}. */ - public File getBufferDirectory() + public FileStore getBufferStore() { - return bufferDirectory; + return bufferStore; } /** - * @return The directory to move files and directories to that have been quiet in the local data directory for long - * enough and that need manual intervention. Note that this directory needs to be on the same file system - * than {@link #getIncomingDirectory}. + * @return The store to copy the data to. */ - public File getManualInterventionDirectory() + public FileStore getOutgoingStore() { - return manualInterventionDirectory; + return outgoingStore; } /** - * @return The directory on the remote side to move the local files to once they are quiet. + * @return The directory to move files and directories to that have been quiet in the local data directory for long + * enough and that need manual intervention. Note that this directory needs to be on the same file system + * as {@link #getIncomingStore}. */ - public File getOutgoingDirectory() + public File getManualInterventionDirectory() { - return outgoingDirectory; + return manualInterventionDirectory; } /** - * @return The remote host to copy the data to (only with rsync, will use an ssh tunnel). + * @return The store to move files and directories to that have been quiet in the local data directory for long + * enough and that need manual intervention. Note that this directory needs to be on the same file system + * as {@link #getIncomingStore}. */ - public String getOutgoingHost() + public FileStore getManualInterventionStore() { - return outgoingHost; + return manualInterventionStore; } /** @@ -490,17 +516,17 @@ public class Parameters implements ITimingParameters { if (operationLog.isInfoEnabled()) { - operationLog.info(String.format("Incoming directory: '%s'.", getIncomingDirectory().getAbsolutePath())); - operationLog.info(String.format("Buffer directory: '%s'.", getBufferDirectory().getAbsolutePath())); - operationLog.info(String.format("Outgoing directory: '%s'.", getOutgoingDirectory().getAbsolutePath())); - if (null != getOutgoingHost()) + operationLog.info(String.format("Incoming directory: '%s'.", incomingDirectory.getAbsolutePath())); + operationLog.info(String.format("Buffer directory: '%s'.", bufferDirectory.getAbsolutePath())); + operationLog.info(String.format("Outgoing directory: '%s'.", outgoingDirectory.getAbsolutePath())); + if (null != outgoingHost) { - operationLog.info(String.format("Outgoing host: '%s'.", getOutgoingHost())); + operationLog.info(String.format("Outgoing host: '%s'.", outgoingHost)); } if (null != getManualInterventionDirectory()) { operationLog.info(String.format("Manual interventions directory: '%s'.", - getManualInterventionDirectory().getAbsolutePath())); + manualInterventionDirectory.getAbsolutePath())); } operationLog.info(String.format("Check intervall: %d s.", getCheckIntervalMillis() / 1000)); operationLog.info(String.format("Quiet period: %d s.", getQuietPeriodMillis() / 1000)); diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/SelfTest.java b/datamover/source/java/ch/systemsx/cisd/datamover/SelfTest.java index 64c9d2ac9fd0f11e3a67df8a9b03db9cbbec28f9..09be1b3197ebb1d0ea0bcc7466ae957cc02bba26 100644 --- a/datamover/source/java/ch/systemsx/cisd/datamover/SelfTest.java +++ b/datamover/source/java/ch/systemsx/cisd/datamover/SelfTest.java @@ -19,6 +19,7 @@ package ch.systemsx.cisd.datamover; import java.io.File; import java.io.IOException; +import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException; @@ -29,7 +30,6 @@ import ch.systemsx.cisd.common.logging.LogCategory; import ch.systemsx.cisd.common.logging.LogFactory; import ch.systemsx.cisd.common.logging.LogInitializer; import ch.systemsx.cisd.common.utilities.FileUtilities; -import ch.systemsx.cisd.common.utilities.ISelfTestable; /** * A class that can perform a self test of the data mover. @@ -43,130 +43,81 @@ public class SelfTest private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION, SelfTest.class); - private static final String LOCAL_DATA_AND_TEMP_DIR_CONTAIN_EACHOTHER_TEMPLATE = - "The local data directory '%s' and the local temporary directory '%s' contain each other."; - - private static final String LOCAL_AND_REMOTE_DATA_DIR_CONTAIN_EACHOTHER_TEMPLATE = - "The local data directory '%s' and the remote data directory '%s' contain each other."; - - private static final String LOCAL_TEMP_AND_REMOTE_DATA_DIR_CONTAIN_EACHOTHER_TEMPLATE = - "The local temporary directory '%s' and the remote data directory '%s' contain each other."; - static { LogInitializer.init(); } - private static void checkDirectories(File localDataDirectory, File localTemporaryDirectory, - File remoteDataDirectory, File manualInterventionDirectory, String remoteHost, IPathCopier copier) - throws ConfigurationFailureException + private static void checkPathRecords(FileStore[] pathRecords, IPathCopier copier) { - assert localDataDirectory != null; - assert localTemporaryDirectory != null; - assert remoteDataDirectory != null; - assert copier != null; + assert pathRecords != null; - if (null == remoteHost) + checkPathRecordsContainEachOther(pathRecords); + for (FileStore pathRecord : pathRecords) { - checkDirectoriesWithRemoteShare(localDataDirectory, localTemporaryDirectory, remoteDataDirectory); - } else - { - checkDirectoriesWithRemoteHost(localDataDirectory, localTemporaryDirectory, remoteDataDirectory, - remoteHost, copier); - } - if (manualInterventionDirectory != null) - { - String errorMessage = - FileUtilities.checkDirectoryFullyAccessible(manualInterventionDirectory, "manual intervention"); - if (errorMessage != null) + if (pathRecord.getPath() == null) + { + continue; + } + if (pathRecord.getHost() == null) + { + checkDirectoryOnLocalHost(pathRecord); + } else { - throw new ConfigurationFailureException(errorMessage); + checkDirectoryOnRemoteHost(pathRecord, copier); } } } - private static void checkDirectoriesWithRemoteHost(File localDataDirectory, File localTemporaryDirectory, - File remoteDataDirectory, String remoteHost, IPathCopier copier) throws ConfigurationFailureException + private static void checkPathRecordsContainEachOther(FileStore[] store) throws ConfigurationFailureException { - checkLocalDirectories(localDataDirectory, localTemporaryDirectory); - checkDirectoryOnRemoteHost(remoteDataDirectory, remoteHost, copier); + for (int i = 1; i < store.length; ++i) + { + for (int j = 0; j < i; ++j) + { + if (StringUtils.equals(store[i].getHost(), store[j].getHost()) + && containOneAnother(store[i].getCanonicalPath(), store[j].getCanonicalPath())) + { + throw ConfigurationFailureException.fromTemplate("Directory '%s' and '%s' contain each other", + store[i].getCanonicalPath(), store[j].getCanonicalPath()); + } + } + } } - private static void checkLocalDirectories(File localDataDirectory, File localTemporaryDirectory) + private static void checkDirectoryOnRemoteHost(FileStore pathRecord, IPathCopier copier) throws ConfigurationFailureException { - final String localDataCanonicalPath = getCanonicalPath(localDataDirectory); - final String localTempCanonicalPath = getCanonicalPath(localTemporaryDirectory); + if (false == copier.exists(pathRecord.getPath(), pathRecord.getHost())) - if (localDataCanonicalPath.startsWith(localTempCanonicalPath) - || localTempCanonicalPath.startsWith(localDataCanonicalPath)) { - throw new ConfigurationFailureException(String.format(LOCAL_DATA_AND_TEMP_DIR_CONTAIN_EACHOTHER_TEMPLATE, - localDataCanonicalPath, localTempCanonicalPath)); + throw ConfigurationFailureException.fromTemplate("Cannot access %s directory '%s' on host '%s'", pathRecord + .getKind(), pathRecord.getCanonicalPath(), pathRecord.getHost()); } + } - String errorMessage = FileUtilities.checkDirectoryFullyAccessible(localDataDirectory, "local data"); - if (errorMessage != null) - { - throw new ConfigurationFailureException(errorMessage); - } - errorMessage = FileUtilities.checkDirectoryFullyAccessible(localTemporaryDirectory, "local temporary"); + private static void checkDirectoryOnLocalHost(FileStore pathRecord) + { + String errorMessage = FileUtilities.checkDirectoryFullyAccessible(pathRecord.getPath(), pathRecord.getKind()); if (errorMessage != null) { throw new ConfigurationFailureException(errorMessage); } - } - private static void checkDirectoryOnRemoteHost(File remoteDataDirectory, String remoteHost, IPathCopier copier) - throws ConfigurationFailureException - { - if (false == copier.exists(remoteDataDirectory, remoteHost)) - { - throw ConfigurationFailureException.fromTemplate("Cannot access directory '%s' on host '%s'", - remoteDataDirectory.getPath(), remoteHost); - } } - private static void checkDirectoriesWithRemoteShare(File localDataDirectory, File localTemporaryDirectory, - File remoteDataDirectory) throws ConfigurationFailureException + private static boolean containOneAnother(String directory1, String directory2) { - final String localDataCanonicalPath = getCanonicalPath(localDataDirectory); - final String localTempCanonicalPath = getCanonicalPath(localTemporaryDirectory); - final String remoteDataCanonicalPath = getCanonicalPath(remoteDataDirectory); - - if (localDataCanonicalPath.startsWith(localTempCanonicalPath) - || localTempCanonicalPath.startsWith(localDataCanonicalPath)) - { - throw ConfigurationFailureException.fromTemplate(LOCAL_DATA_AND_TEMP_DIR_CONTAIN_EACHOTHER_TEMPLATE, - localDataCanonicalPath, localTempCanonicalPath); - } - if (localDataCanonicalPath.startsWith(remoteDataCanonicalPath) - || remoteDataCanonicalPath.startsWith(localDataCanonicalPath)) - { - throw ConfigurationFailureException.fromTemplate(LOCAL_AND_REMOTE_DATA_DIR_CONTAIN_EACHOTHER_TEMPLATE, - localDataCanonicalPath, remoteDataCanonicalPath); - } - if (localTempCanonicalPath.startsWith(remoteDataCanonicalPath) - || remoteDataCanonicalPath.startsWith(localTempCanonicalPath)) - { - throw ConfigurationFailureException.fromTemplate(LOCAL_TEMP_AND_REMOTE_DATA_DIR_CONTAIN_EACHOTHER_TEMPLATE, - localTempCanonicalPath, remoteDataCanonicalPath); - } - - String errorMessage = FileUtilities.checkDirectoryFullyAccessible(localDataDirectory, "local data"); - if (errorMessage != null) + if (directory1 == null || directory2 == null) { - throw new ConfigurationFailureException(errorMessage); + return false; } - errorMessage = FileUtilities.checkDirectoryFullyAccessible(localTemporaryDirectory, "local temporary"); - if (errorMessage != null) + if (directory1.length() < directory2.length()) { - throw new ConfigurationFailureException(errorMessage); - } - errorMessage = FileUtilities.checkDirectoryFullyAccessible(remoteDataDirectory, "remote"); - if (errorMessage != null) + return directory2.startsWith(directory1); + } else { - throw new ConfigurationFailureException(errorMessage); + return directory1.startsWith(directory2); } } @@ -175,26 +126,26 @@ public class SelfTest * considered "past", otherwise the exception will have more information on what went wrong. This method performs * failure logging of {@link ConfigurationFailureException}s and {@link EnvironmentFailureException}s. */ - public static void check(File localDataDirectory, File localTemporaryDirectory, File remoteDataDirectory, - File manualInterventionDirectory, String remoteHost, IPathCopier copier, ISelfTestable... selfTestables) - throws ConfigurationFailureException, EnvironmentFailureException + public static void check(IPathCopier copier, FileStore... stores) { try { - if (null != remoteHost && false == copier.supportsExplicitHost()) + if (false == copier.supportsExplicitHost()) { - throw ConfigurationFailureException.fromTemplate( - "Copier %s does not support explicit remote hosts, but remote host given:%s", copier.getClass() - .getSimpleName(), remoteHost); + for (FileStore store : stores) + { + if (null != store.getHost()) + { + throw ConfigurationFailureException + .fromTemplate( + "Copier %s does not support explicit remote hosts, but %s store is on a remote host (%s)", + copier.getClass().getSimpleName(), store.getKind(), store.getHost()); + } + } } - checkDirectories(localDataDirectory, localTemporaryDirectory, remoteDataDirectory, - manualInterventionDirectory, remoteHost, copier); copier.check(); + checkPathRecords(stores, copier); - for (ISelfTestable selfTestable : selfTestables) - { - selfTestable.check(); - } if (operationLog.isInfoEnabled()) { operationLog.info("Self test successfully completed."); @@ -212,18 +163,6 @@ public class SelfTest } } - private static String getCanonicalPath(File path) - { - try - { - return path.getCanonicalPath() + File.separator; - } catch (IOException e) - { - throw EnvironmentFailureException.fromTemplate(e, "Cannot determine canonical form of path '%s'", path - .getPath()); - } - } - /** * @return <code>true</code> if the <var>copyProcess</var> on the file system where the <var>destinationDirectory</var> * resides requires deleting an existing file before it can be overwritten. @@ -276,57 +215,4 @@ public class SelfTest } } - public static void main(String[] args) - { - if (args.length == 3) - { - final File localDataDirectory = new File(args[0]); - final File localTemporaryDirectory = new File(args[1]); - final File remoteDataDirectory = new File(args[2]); - try - { - check(localDataDirectory, localTemporaryDirectory, remoteDataDirectory, null, null, new IPathCopier() - { - public Status copy(File sourcePath, File destinationDirectory) - { - return null; - } - - public Status copy(File sourcePath, File destinationDirectory, String destinationHost) - { - return null; - } - - public boolean exists(File destinationDirectory, String destinationHost) - { - return false; - } - - public boolean supportsExplicitHost() - { - return false; - } - - public boolean terminate() - { - return false; - } - - public void check() - { - } - }); - System.err.println("Self test passed."); - } catch (Exception e) - { - System.err.println("Self test failed:"); - e.printStackTrace(); - System.exit(1); - } - } else - { - System.err.println("Syntax: SelfTest <localDataDir> <localTempDir> <remoteDataDir>"); - System.exit(2); - } - } }