diff --git a/bds/source/java/ch/systemsx/cisd/bds/storage/filesystem/Directory.java b/bds/source/java/ch/systemsx/cisd/bds/storage/filesystem/Directory.java index 2c13cc619f6cd891db43008e91f8cfac29ee0d2f..ce1edb24fa16b34426d4cb6110e9f29cedd869ac 100644 --- a/bds/source/java/ch/systemsx/cisd/bds/storage/filesystem/Directory.java +++ b/bds/source/java/ch/systemsx/cisd/bds/storage/filesystem/Directory.java @@ -159,7 +159,7 @@ final class Directory extends AbstractNode implements IDirectory assert name != null : "Name can not be null."; final java.io.File file = getNodeFile(node); final boolean ok = - LinkMakerProvider.getLinkMaker().copyFileImmutably(file, nodeFile, name); + LinkMakerProvider.getLinkMaker().copyImmutably(file, nodeFile, name); if (ok) { final Link link = (Link) NodeFactory.createLinkNode(name, file); diff --git a/bds/source/java/ch/systemsx/cisd/bds/storage/filesystem/LinkMakerProvider.java b/bds/source/java/ch/systemsx/cisd/bds/storage/filesystem/LinkMakerProvider.java index 40f2f3263f94a91bff7c3b6d46db65069bf6179e..296e9cee8d49e1e45e872ca6eb9a63e09cdf3732 100644 --- a/bds/source/java/ch/systemsx/cisd/bds/storage/filesystem/LinkMakerProvider.java +++ b/bds/source/java/ch/systemsx/cisd/bds/storage/filesystem/LinkMakerProvider.java @@ -18,11 +18,11 @@ package ch.systemsx.cisd.bds.storage.filesystem; import ch.systemsx.cisd.common.Constants; import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException; -import ch.systemsx.cisd.common.utilities.IFileImmutableCopier; -import ch.systemsx.cisd.common.utilities.RecursiveHardLinkMaker; +import ch.systemsx.cisd.common.utilities.FastRecursiveHardLinkMaker; +import ch.systemsx.cisd.common.utilities.IImmutableCopier; /** - * A provider of {@link IFileImmutableCopier} implementations. + * A provider of {@link IImmutableCopier} implementations. * * @author Christian Ribeaud */ @@ -33,17 +33,17 @@ public final class LinkMakerProvider private static final int MAX_COPY_RETRIES = 7; - private static IFileImmutableCopier hardLinkMaker; + private static IImmutableCopier hardLinkMaker; private LinkMakerProvider() { // This class can not be instantiated. } - private final static IFileImmutableCopier tryCreateHardLinkMaker() + private final static IImmutableCopier tryCreateHardLinkMaker() { - final IFileImmutableCopier copier = - RecursiveHardLinkMaker.tryCreateRetrying(Constants.MILLIS_TO_WAIT_BEFORE_TIMEOUT, + final IImmutableCopier copier = + FastRecursiveHardLinkMaker.tryCreate(Constants.MILLIS_TO_WAIT_BEFORE_TIMEOUT, MAX_COPY_RETRIES, Constants.MILLIS_TO_SLEEP_BEFORE_RETRYING); if (copier != null) { @@ -58,7 +58,7 @@ public final class LinkMakerProvider * Returns an <code>IPathImmutableCopier</code> implementation which makes <i>hard links</i> * using the underlying <i>operating system</i>. */ - public final static IFileImmutableCopier getLinkMaker() + public final static IImmutableCopier getLinkMaker() { if (hardLinkMaker == null) { diff --git a/common/source/java/ch/systemsx/cisd/common/utilities/FastRecursiveHardLinkMaker.java b/common/source/java/ch/systemsx/cisd/common/utilities/FastRecursiveHardLinkMaker.java new file mode 100644 index 0000000000000000000000000000000000000000..6031aa2324643f17ec338b92764d68bca2f5759a --- /dev/null +++ b/common/source/java/ch/systemsx/cisd/common/utilities/FastRecursiveHardLinkMaker.java @@ -0,0 +1,154 @@ +/* + * Copyright 2008 ETH Zuerich, CISD + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.systemsx.cisd.common.utilities; + +import java.io.File; + +import org.apache.log4j.Logger; + +import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException; +import ch.systemsx.cisd.common.logging.LogCategory; +import ch.systemsx.cisd.common.logging.LogFactory; + +/** + * A fast {@link IImmutableCopier} that uses a fallback option whenever one of the fast copiers for + * either files or directories is not available. + * + * @author Bernd Rinn + */ +public class FastRecursiveHardLinkMaker implements IImmutableCopier +{ + + private static final Logger operationLog = + LogFactory.getLogger(LogCategory.OPERATION, FastRecursiveHardLinkMaker.class); + + private static final String RSYNC_EXEC = "rsync"; + + private final static long MILLIS = 1000L; + + private final static long DEFAULT_INACTIVITY_TRESHOLD_MILLIS = 300 * MILLIS; + + private final static int DEFAULT_MAX_ERRORS_TO_IGNORE = 3; + + private final static int DEFAULT_MAX_ATTEMPTS = 10; + + private final static long DEFAULT_TIME_TO_SLEEP_AFTER_COPY_FAILS = 5 * MILLIS; + + private final IImmutableCopier fallbackCopierOrNull; + + private final IFileImmutableCopier fastFileCopierOrNull; + + private final IDirectoryImmutableCopier fastDirectoryCopierOrNull; + + public final static IImmutableCopier tryCreate(final File rsyncExecutable) + { + return tryCreate(rsyncExecutable, DEFAULT_INACTIVITY_TRESHOLD_MILLIS, DEFAULT_MAX_ATTEMPTS, + DEFAULT_TIME_TO_SLEEP_AFTER_COPY_FAILS); + } + + public final static IImmutableCopier tryCreate(final long millisToWaitForCompletion, + final int maxRetryOnFailure, final long millisToSleepOnFailure) + { + return tryCreate(OSUtilities.findExecutable(RSYNC_EXEC), millisToWaitForCompletion, + maxRetryOnFailure, millisToSleepOnFailure); + } + + public final static IImmutableCopier tryCreate(final File rsyncExecutable, + final long millisToWaitForCompletion, final int maxRetryOnFailure, + final long millisToSleepOnFailure) + { + try + { + return new FastRecursiveHardLinkMaker(rsyncExecutable, millisToSleepOnFailure, + maxRetryOnFailure, millisToSleepOnFailure); + } catch (ConfigurationFailureException ex) + { + return null; + } + } + + private FastRecursiveHardLinkMaker(final File rsyncExcutable, + final long inactivityThresholdMillis, final int maxRetryOnFailure, + final long millisToSleepOnFailure) throws ConfigurationFailureException + { + this.fastFileCopierOrNull = + FastHardLinkMaker.tryCreate(inactivityThresholdMillis, maxRetryOnFailure, + millisToSleepOnFailure); + this.fastDirectoryCopierOrNull = + new RsyncBasedRecursiveHardLinkMaker(rsyncExcutable, inactivityThresholdMillis, + DEFAULT_MAX_ERRORS_TO_IGNORE, maxRetryOnFailure, millisToSleepOnFailure); + if (fastFileCopierOrNull == null) + { + this.fallbackCopierOrNull = + RecursiveHardLinkMaker.tryCreate(HardLinkMaker.tryCreateRetrying( + inactivityThresholdMillis, maxRetryOnFailure, millisToSleepOnFailure)); + } else + { + this.fallbackCopierOrNull = RecursiveHardLinkMaker.tryCreate(fastFileCopierOrNull); + } + if ((fastFileCopierOrNull == null && fallbackCopierOrNull == null) + || (fastDirectoryCopierOrNull == null && fallbackCopierOrNull == null)) + { + throw new ConfigurationFailureException("Not operational"); + } + if (operationLog.isInfoEnabled()) + { + if (fastFileCopierOrNull != null) + { + operationLog.info("Using native library to create hard link copies of files."); + } else + { + operationLog.info("Using 'ln' to create hard link copies of files."); + } + if (fastDirectoryCopierOrNull != null) + { + operationLog.info("Using 'rsync' to traverse directories when making recursive " + + "hard links copies."); + } else + { + operationLog.info("Using Java to traverse directories when making recursive hard " + + "link copies"); + } + } + } + + public boolean copyImmutably(File source, File destinationDirectory, String nameOrNull) + { + if (source.isDirectory()) + { + if (fastDirectoryCopierOrNull != null) + { + return fastDirectoryCopierOrNull.copyDirectoryImmutably(source, + destinationDirectory, nameOrNull); + } else + { + return fallbackCopierOrNull.copyImmutably(source, destinationDirectory, nameOrNull); + } + } else + { + if (fastFileCopierOrNull != null) + { + return fastFileCopierOrNull.copyFileImmutably(source, destinationDirectory, + nameOrNull); + } else + { + return fallbackCopierOrNull.copyImmutably(source, destinationDirectory, nameOrNull); + } + } + } + +} diff --git a/common/source/java/ch/systemsx/cisd/common/utilities/HardLinkMaker.java b/common/source/java/ch/systemsx/cisd/common/utilities/HardLinkMaker.java new file mode 100644 index 0000000000000000000000000000000000000000..755384d207655fb6a1a8cb18d86bb24040f8f77f --- /dev/null +++ b/common/source/java/ch/systemsx/cisd/common/utilities/HardLinkMaker.java @@ -0,0 +1,219 @@ +/* + * Copyright 2008 ETH Zuerich, CISD + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.systemsx.cisd.common.utilities; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; + +import org.apache.commons.io.FileUtils; +import org.apache.log4j.Logger; + +import ch.systemsx.cisd.common.exceptions.StopException; +import ch.systemsx.cisd.common.logging.LogCategory; +import ch.systemsx.cisd.common.logging.LogFactory; +import ch.systemsx.cisd.common.process.CallableExecutor; +import ch.systemsx.cisd.common.process.ProcessExecutionHelper; + +/** + * @author Bernd Rinn + */ +public class HardLinkMaker implements IFileImmutableCopier +{ + private static final Logger operationLog = + LogFactory.getLogger(LogCategory.OPERATION, HardLinkMaker.class); + + private static final Logger machineLog = + LogFactory.getLogger(LogCategory.MACHINE, HardLinkMaker.class); + + private static final String HARD_LINK_EXEC = "ln"; + + private final String linkExecPath; + + private final RetryingOperationTimeout singleFileLinkTimeout; + + private static class RetryingOperationTimeout + { + private final long millisToWaitForCompletion; + + private final int maxRetryOnFailure; + + private final long millisToSleepOnFailure; + + private RetryingOperationTimeout(long millisToWaitForCompletion, int maxRetryOnFailure, + long millisToSleepOnFailure) + { + this.millisToWaitForCompletion = millisToWaitForCompletion; + this.maxRetryOnFailure = maxRetryOnFailure; + this.millisToSleepOnFailure = millisToSleepOnFailure; + } + + public long getMillisToWaitForCompletion() + { + return millisToWaitForCompletion; + } + + public int getMaxRetryOnFailure() + { + return maxRetryOnFailure; + } + + public long getMillisToSleepOnFailure() + { + return millisToSleepOnFailure; + } + } + + private HardLinkMaker(final String linkExecPath, + RetryingOperationTimeout singleFileLinkTimeoutOrNull) + { + this.linkExecPath = linkExecPath; + this.singleFileLinkTimeout = + singleFileLinkTimeoutOrNull != null ? singleFileLinkTimeoutOrNull + : createNoTimeout(); + } + + // + // Factory methods + // + + /** + * Creates copier which won't retry an operation if it fails. + * + * @param linkExecPath The path to the <code>ln</code> executable. + */ + public static final IFileImmutableCopier create(final String linkExecPath) + { + return new HardLinkMaker(linkExecPath, null); + } + + /** + * Creates copier trying to find the path to the <code>ln</code> executable. + * + * @return <code>null</code> if the <code>ln</code> executable was not found. + */ + public static final IFileImmutableCopier tryCreate() + { + return tryCreate(null); + } + + /** + * Creates copier which is able to retry the operation of creating the hard link of a file if + * it does not complete after a specified timeout. + * + * @param millisToWaitForCompletion The time to wait for the process creating one hard link to a + * file to complete in milli seconds. If the process is not finished after that time, + * it will be terminated. + * @param maxRetryOnFailure The number of times we should try to create each hard link if copy + * operation fails. + * @param millisToSleepOnFailure The number of milliseconds we should wait before re-executing + * the copy of a single file. Specify 0 to wait till the first operation completes. + */ + public static final IFileImmutableCopier tryCreateRetrying( + final long millisToWaitForCompletion, final int maxRetryOnFailure, + final long millisToSleepOnFailure) + { + RetryingOperationTimeout timeout = + new RetryingOperationTimeout(millisToWaitForCompletion, maxRetryOnFailure, + millisToSleepOnFailure); + return tryCreate(timeout); + } + + private static final IFileImmutableCopier tryCreate( + RetryingOperationTimeout singleFileLinkTimeoutOrNull) + { + final File lnExec = OSUtilities.findExecutable(HARD_LINK_EXEC); + if (lnExec == null) + { + return null; + } + return new HardLinkMaker(lnExec.getAbsolutePath(), singleFileLinkTimeoutOrNull); + } + + private RetryingOperationTimeout createNoTimeout() + { + return new RetryingOperationTimeout(0, 1, 0); + } + + public boolean copyFileImmutably(final File source, final File destinationDirectory, + final String nameOrNull) + { + assert source.isFile() : String + .format("Given file '%s' must be a file and is not.", source); + final File destFile = + new File(destinationDirectory, nameOrNull == null ? source.getName() : nameOrNull); + final List<String> cmd = createLnCmdLine(source, destFile); + final Callable<Boolean> processTask = new Callable<Boolean>() + { + public final Boolean call() + { + boolean result = + ProcessExecutionHelper.runAndLog(cmd, operationLog, machineLog, + singleFileLinkTimeout.getMillisToWaitForCompletion()); + // NOTE: we have noticed that sometimes the result is false although the file + // have been copied + if (result == false && destFile.exists() + && checkIfIdenticalContent(source, destFile)) + { + machineLog + .warn("Link creator reported failure, but the exact copy of the file '" + + source.getPath() + + "' seems to exist in '" + + destFile.getPath() + "'. Error will be ignored."); + result = true; + } + return result; + } + }; + boolean ok = + runRepeatableProcess(processTask, singleFileLinkTimeout.getMaxRetryOnFailure(), + singleFileLinkTimeout.getMillisToSleepOnFailure()); + return ok; + } + + private static boolean checkIfIdenticalContent(final File file1, final File file2) + { + StopException.check(); + try + { + return FileUtils.contentEquals(file1, file2); + } catch (IOException e) + { + machineLog.warn("Error when comparing the content of a file and its link: " + + e.getMessage()); + } + return false; + } + + private final List<String> createLnCmdLine(final File srcFile, final File destFile) + { + final List<String> tokens = new ArrayList<String>(); + tokens.add(linkExecPath); + tokens.add(srcFile.getAbsolutePath()); + tokens.add(destFile.getAbsolutePath()); + return tokens; + } + + private static boolean runRepeatableProcess(final Callable<Boolean> task, + final int maxRetryOnFailure, final long millisToSleepOnFailure) + { + return new CallableExecutor(maxRetryOnFailure, millisToSleepOnFailure) + .executeCallable(task); + } +} diff --git a/common/source/java/ch/systemsx/cisd/common/utilities/IDirectoryImmutableCopier.java b/common/source/java/ch/systemsx/cisd/common/utilities/IDirectoryImmutableCopier.java index 96ce8ccee7b6155c6badb017efd48a6c7f5d1eb7..33ac977b7804b69cefb1f6004e844a8d2e00171c 100644 --- a/common/source/java/ch/systemsx/cisd/common/utilities/IDirectoryImmutableCopier.java +++ b/common/source/java/ch/systemsx/cisd/common/utilities/IDirectoryImmutableCopier.java @@ -19,35 +19,36 @@ package ch.systemsx.cisd.common.utilities; import java.io.File; /** - * A role which can perform an immutable copy of a direcoty. <i>Immutable</i> here means, that none - * of the copied file must be changed or else the corresponding original file may be changed, tool + * A role which can perform an immutable copy of a directory. <i>Immutable</i> here means, that none + * of the copied files can be changed or else the original file may be changed, too. It is, however, + * safe to delete the files and directories. This restrictions allows to use hard links for + * performing the copy which can save a lot of disk space. * * @author Bernd Rinn */ public interface IDirectoryImmutableCopier { /** - * Creates a copy of the directory <code>file</code> (which may be a file or a directory) in - * <code>destinationDirectory</code>, which must not be modified later. + * Creates an immutable copy of the {@link File} <code>source</code> (which has to be a + * directory) in <code>destinationDirectory</code>. * <p> - * Note that this method don't do any checks about whether paths are directories and whether - * they exist or not. Use methods like - * {@link FileUtilities#checkDirectoryFullyAccessible(File, String)} for checking prior to - * calling this method where appropriate. + * Note that this method does not perform any checks about whether <var>source</var> exists and + * is accessible. Use methods like {@link FileUtilities#checkPathFullyAccessible(File, String)} + * for checking prior to calling this method where appropriate. * </p> * <p> * <i>Can use hard links if available.</i> * </p> * - * @param sourceDirectory The source directory. Really has to be a directory. Can not be - * <code>null</code> and needs to exists. - * @param destinationDirectory The directory to copy <var>sourceDirectory</var> to. Can not be - * <code>null</code> and must be an existing directory. - * @param targetNameOrNull The target name of the source directory. If <code>null</code>, the - * name of the source directory itself is used. - * @return <code>true</code> if the operation was successful, <code>false</code> otherwise. + * @param source The source directory. Can not be <code>null</code> and must not be a file. + * @param destinationDirectory The directory where given <var>source</var> should be copied. Can + * not be <code>null</code> and must be an existing directory. + * @param nameOrNull The link name in the destination directory. If it is <code>null</code>, the + * name of <var>source</var> will be used instead. + * @return <code>true</code>, if the source directory was copied successfully, + * <code>false</code> otherwise. */ - boolean copyDirectoryImmutably(final File sourceDirectory, final File destinationDirectory, - String targetNameOrNull); + boolean copyDirectoryImmutably(final File source, final File destinationDirectory, + String nameOrNull); } diff --git a/common/source/java/ch/systemsx/cisd/common/utilities/IFileImmutableCopier.java b/common/source/java/ch/systemsx/cisd/common/utilities/IFileImmutableCopier.java index 6a20d6ca36c040ed9ff97a5fc55a735cf24146e3..f0a72968fd98fc9bb28c3f66fa43c05266dbaa73 100644 --- a/common/source/java/ch/systemsx/cisd/common/utilities/IFileImmutableCopier.java +++ b/common/source/java/ch/systemsx/cisd/common/utilities/IFileImmutableCopier.java @@ -20,32 +20,34 @@ import java.io.File; /** * A role which can perform an immutable copy of a file. <i>Immutable</i> here means, that the - * copied file must not be changed or else the original file may be changed, too. + * copied file must not be changed or else the original file may be changed, too. It is, however, + * safe to delete the file. This restrictions allows to use hard links for performing the copy which + * can save a lot of disk space. * * @author Bernd Rinn */ public interface IFileImmutableCopier { /** - * Creates a copy of the file <code>file</code> (which may be a file or a directory) in - * <code>destinationDirectory</code>, which must not be modified later. + * Creates an immutable copy of the {@link File} <code>source</code> in + * <code>destinationDirectory</code>. * <p> - * Note that this method don't do any checks about whether paths are files and whether - * they exist or not. Use methods like - * {@link FileUtilities#checkPathFullyAccessible(File, String)} for checking prior to - * calling this method where appropriate. + * Note that this method does not perform any checks about whether <var>source</var> exists and + * is accessible. Use methods like {@link FileUtilities#checkPathFullyAccessible(File, String)} + * for checking prior to calling this method where appropriate. * </p> * <p> * <i>Can use hard links if available.</i> * </p> * - * @param file The source file. This really has to be a file. Can not be <code>null</code>. - * @param destinationDirectory The directory where given <var>path</var> should be copied. Can + * @param source The source file. Can not be <code>null</code> or a directory. + * @param destinationDirectory The directory where given <var>source</var> should be copied. Can * not be <code>null</code> and must be an existing directory. - * @param nameOrNull The link name in the destination directory. If it is <code>null</code>, - * the name of <var>file</var> will be used instead. - * @return <code>true</code>, if the file was copied successfully, <code>false</code> otherwise. + * @param nameOrNull The link name in the destination file. If it is <code>null</code>, the name + * of <var>source</var> will be used instead. + * @return <code>true</code>, if the source file was copied successfully, <code>false</code> + * otherwise. */ - boolean copyFileImmutably(final File file, final File destinationDirectory, + boolean copyFileImmutably(final File source, final File destinationDirectory, final String nameOrNull); } diff --git a/common/source/java/ch/systemsx/cisd/common/utilities/IImmutableCopier.java b/common/source/java/ch/systemsx/cisd/common/utilities/IImmutableCopier.java new file mode 100644 index 0000000000000000000000000000000000000000..53c687b14f0dbffcb9e5cd5921b1cf7e10c4cf24 --- /dev/null +++ b/common/source/java/ch/systemsx/cisd/common/utilities/IImmutableCopier.java @@ -0,0 +1,55 @@ +/* + * Copyright 2007 ETH Zuerich, CISD + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.systemsx.cisd.common.utilities; + +import java.io.File; + +/** + * A role which can perform an immutable copy of a file or directory. <i>Immutable</i> here means, + * that none of the copied files can be changed or else the original file may be changed, too. It + * is, however, safe to delete the files and directories. This restrictions allows to use hard links + * for performing the copy which can save a lot of disk space. + * + * @author Tomasz Pylak + * @author Bernd Rinn + */ +public interface IImmutableCopier +{ + /** + * Creates an immutable copy of the {@link File} <code>source</code> in + * <code>destinationDirectory</code>. + * <p> + * Note that this method does not perform any checks about whether <var>source</var> exists and + * is accessible. Use methods like {@link FileUtilities#checkPathFullyAccessible(File, String)} + * for checking prior to calling this method where appropriate. + * </p> + * <p> + * <i>Can use hard links if available.</i> + * </p> + * + * @param source The source file or directory. Can not be <code>null</code>. + * @param destinationDirectory The directory where given <var>source</var> should be copied. Can + * not be <code>null</code> and must be an existing directory. + * @param nameOrNull The link name in the destination file or directory. If it is + * <code>null</code>, the name of <var>source</var> will be used instead. + * @return <code>true</code>, if the source file or directory was copied successfully, + * <code>false</code> otherwise. + */ + boolean copyImmutably(final File source, final File destinationDirectory, + final String nameOrNull); + +} diff --git a/common/source/java/ch/systemsx/cisd/common/utilities/IPathImmutableCopier.java b/common/source/java/ch/systemsx/cisd/common/utilities/IPathImmutableCopier.java deleted file mode 100644 index 89f881dfa506b78dcee2cd007545900828fd9ddf..0000000000000000000000000000000000000000 --- a/common/source/java/ch/systemsx/cisd/common/utilities/IPathImmutableCopier.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2007 ETH Zuerich, CISD - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package ch.systemsx.cisd.common.utilities; - -/** - * Utility to create copies of files/resources, which should not be modified later. - * - * @author Tomasz Pylak - */ -public interface IPathImmutableCopier extends IFileImmutableCopier, IDirectoryImmutableCopier -{ -} diff --git a/common/source/java/ch/systemsx/cisd/common/utilities/RecursiveHardLinkMaker.java b/common/source/java/ch/systemsx/cisd/common/utilities/RecursiveHardLinkMaker.java index 1dc78dd99ffb075b64da512a11fba1883a95dccf..f403e6cea4eec0b849ec3883e736010b298b424d 100644 --- a/common/source/java/ch/systemsx/cisd/common/utilities/RecursiveHardLinkMaker.java +++ b/common/source/java/ch/systemsx/cisd/common/utilities/RecursiveHardLinkMaker.java @@ -17,19 +17,12 @@ package ch.systemsx.cisd.common.utilities; import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.Callable; -import org.apache.commons.io.FileUtils; import org.apache.log4j.Logger; import ch.systemsx.cisd.common.exceptions.StopException; import ch.systemsx.cisd.common.logging.LogCategory; import ch.systemsx.cisd.common.logging.LogFactory; -import ch.systemsx.cisd.common.process.CallableExecutor; -import ch.systemsx.cisd.common.process.ProcessExecutionHelper; /** * Utility to create a hard link of a file or copy recursively a directories structure, creating a @@ -38,214 +31,107 @@ import ch.systemsx.cisd.common.process.ProcessExecutionHelper; * * @author Tomasz Pylak */ -public final class RecursiveHardLinkMaker implements IPathImmutableCopier +public final class RecursiveHardLinkMaker implements IImmutableCopier { - private static final String HARD_LINK_EXEC = "ln"; - private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION, RecursiveHardLinkMaker.class); private static final Logger machineLog = LogFactory.getLogger(LogCategory.MACHINE, RecursiveHardLinkMaker.class); - private final String linkExecPath; - - private final RetryingOperationTimeout singleFileLinkTimeout; - - private RecursiveHardLinkMaker(final String linkExecPath, - RetryingOperationTimeout singleFileLinkTimeoutOrNull) - { - this.linkExecPath = linkExecPath; - this.singleFileLinkTimeout = - singleFileLinkTimeoutOrNull != null ? singleFileLinkTimeoutOrNull - : createNoTimeout(); - } - - private RetryingOperationTimeout createNoTimeout() - { - return new RetryingOperationTimeout(0, 1, 0); - } + private final IFileImmutableCopier fileCopier; - private static class RetryingOperationTimeout + private RecursiveHardLinkMaker(final IFileImmutableCopier fileCopier) { - private final long millisToWaitForCompletion; - - private final int maxRetryOnFailure; - - private final long millisToSleepOnFailure; - - private RetryingOperationTimeout(long millisToWaitForCompletion, int maxRetryOnFailure, - long millisToSleepOnFailure) - { - this.millisToWaitForCompletion = millisToWaitForCompletion; - this.maxRetryOnFailure = maxRetryOnFailure; - this.millisToSleepOnFailure = millisToSleepOnFailure; - } - - public long getMillisToWaitForCompletion() - { - return millisToWaitForCompletion; - } - - public int getMaxRetryOnFailure() - { - return maxRetryOnFailure; - } - - public long getMillisToSleepOnFailure() - { - return millisToSleepOnFailure; - } + this.fileCopier = fileCopier; } // // Factory methods // - - /** - * Creates copier which won't retry an operation if it fails. - * - * @param linkExecPath The path to the <code>ln</code> executable. - */ - public static final IPathImmutableCopier create(final String linkExecPath) - { - return new RecursiveHardLinkMaker(linkExecPath, null); - } - - /** - * Creates copier trying to find the path to the <code>ln</code> executable. - * - * @return <code>null</code> if the <code>ln</code> executable was not found. - */ - public static final IPathImmutableCopier tryCreate() + public static IImmutableCopier tryCreate(final IFileImmutableCopier fileCopierOrNull) { - return tryCreate(null); - } - - /** - * Creates copier which is able to retry the operation of creating each hard link of a file if - * it does not complete after a specified timeout. - * - * @param millisToWaitForCompletion The time to wait for the process creating one hard link to a - * file to complete in milli seconds. If the process is not finished after that time, - * it will be terminated. - * @param maxRetryOnFailure The number of times we should try to create each hard link if copy - * operation fails. - * @param millisToSleepOnFailure The number of milliseconds we should wait before re-executing - * the copy of a single file. Specify 0 to wait till the first operation completes. - */ - public static final IPathImmutableCopier tryCreateRetrying( - final long millisToWaitForCompletion, final int maxRetryOnFailure, - final long millisToSleepOnFailure) - { - RetryingOperationTimeout timeout = - new RetryingOperationTimeout(millisToWaitForCompletion, maxRetryOnFailure, - millisToSleepOnFailure); - return tryCreate(timeout); - } - - private static final IPathImmutableCopier tryCreate( - RetryingOperationTimeout singleFileLinkTimeoutOrNull) - { - final File lnExec = OSUtilities.findExecutable(HARD_LINK_EXEC); - if (lnExec == null) + if (fileCopierOrNull == null) { return null; + } else + { + return new RecursiveHardLinkMaker(fileCopierOrNull); } - return new RecursiveHardLinkMaker(lnExec.getAbsolutePath(), singleFileLinkTimeoutOrNull); } + + // + // IImmutableCopier + // /** - * Copies <var>path</var> (file or directory) to <var>destinationDirectory</var> by - * duplicating directory structure and creating hard link for each file. + * Copies <var>source</var> (file or directory) to <var>destinationDirectory</var> by + * duplicating the directory structure and creating a hard link for each file. * <p> - * <i>Note that <var>nameOrNull</var> cannot already exist in given <var>destinationDirectory</var>.</i> + * <i>Note that <var>nameOrNull</var> cannot already exist in given + * <var>destinationDirectory</var>.</i> * </p> */ - private final File tryImmutableCopy(final File path, final File destinationDirectory, - final String nameOrNull) + public boolean copyImmutably(File source, File destinationDirectory, String nameOrNull) { - assert path != null : "Given path can not be null."; - assert destinationDirectory != null && destinationDirectory.isDirectory() : "Given destination directory can not be null and must be a directory."; - final String destName = (nameOrNull == null) ? path.getName() : nameOrNull; + assert source != null && source.exists(); + assert destinationDirectory != null && destinationDirectory.isDirectory(); + final String destName = (nameOrNull == null) ? source.getName() : nameOrNull; final File destFile = new File(destinationDirectory, destName); if (destFile.exists()) { operationLog.error(String.format( "File '%s' already exists in given destination directory '%s'", destName, destinationDirectory)); - return null; + return false; } if (operationLog.isTraceEnabled()) { - operationLog.trace(String.format("Creating a hard link copy of '%s' in '%s'.", path + operationLog.trace(String.format("Creating a hard link copy of '%s' in '%s'.", source .getPath(), destinationDirectory.getPath())); } - return tryMakeCopy(path, destinationDirectory, nameOrNull); - } - - // - // IDirectoryImmutableCopier - // - - public boolean copyDirectoryImmutably(File sourceDirectory, File destinationDirectory, - String targetNameOrNull) - { - assert sourceDirectory != null && sourceDirectory.isDirectory(); - assert destinationDirectory != null && destinationDirectory.isDirectory(); - return (tryImmutableCopy(sourceDirectory, destinationDirectory, targetNameOrNull) != null); - } - - // - // IFileImmutableCopier - // - - public boolean copyFileImmutably(File file, File destinationDirectory, String nameOrNull) - { - assert file != null && file.isFile(); - assert destinationDirectory != null && destinationDirectory.isDirectory(); - return (tryImmutableCopy(file, destinationDirectory, nameOrNull) != null); + return primCopyImmutably(source, destinationDirectory, nameOrNull); } - private final File tryMakeCopy(final File resource, final File destinationDirectory, + private final boolean primCopyImmutably(final File source, final File destinationDirectory, final String nameOrNull) { - if (resource.isFile()) + if (source.isFile()) { - return tryCreateHardLinkIn(resource, destinationDirectory, nameOrNull); + return fileCopier.copyFileImmutably(source, destinationDirectory, nameOrNull); } else { - final String name = nameOrNull == null ? resource.getName() : nameOrNull; + final String name = nameOrNull == null ? source.getName() : nameOrNull; final File dir = tryCreateDir(name, destinationDirectory); if (dir == null) { - return null; + return false; } - final File[] files = resource.listFiles(); + final File[] files = source.listFiles(); if (files != null) { for (final File file : files) { StopException.check(); - if (tryMakeCopy(file, dir, null) == null) + if (primCopyImmutably(file, dir, null) == false) { - return null; + return false; } } } else // Shouldn't happen, but just to be sure. { - if (resource.exists() == false) + if (source.exists() == false) { operationLog.error(String.format("Path '%s' vanished during processing.", - resource)); + source)); } else { operationLog.error(String.format( - "Found path '%s' that is neither a file nor a directory.", resource)); + "Found path '%s' that is neither a file nor a directory.", source)); } } - return dir; + return true; } } @@ -273,72 +159,4 @@ public final class RecursiveHardLinkMaker implements IPathImmutableCopier return ok ? dir : null; } - private final File tryCreateHardLinkIn(final File file, final File destDir, - final String nameOrNull) - { - assert file.isFile() : String.format("Given file '%s' must be a file and is not.", file); - final File destFile = new File(destDir, nameOrNull == null ? file.getName() : nameOrNull); - final List<String> cmd = createLnCmdLine(file, destFile); - final Callable<Boolean> processTask = new Callable<Boolean>() - { - // - // Callable - // - - public final Boolean call() - { - boolean result = - ProcessExecutionHelper.runAndLog(cmd, operationLog, machineLog, - singleFileLinkTimeout.getMillisToWaitForCompletion()); - // NOTE: we have noticed that sometimes the result is false although the file - // have been copied - if (result == false && destFile.exists() - && checkIfIdenticalContent(file, destFile)) - { - machineLog - .warn("Link creator reported failure, but the exact copy of the file '" - + file.getPath() - + "' seems to exist in '" - + destFile.getPath() + "'. Error will be ignored."); - result = true; - } - return result; - } - }; - boolean ok = - runRepeatableProcess(processTask, singleFileLinkTimeout.getMaxRetryOnFailure(), - singleFileLinkTimeout.getMillisToSleepOnFailure()); - return ok ? destFile : null; - } - - private static boolean checkIfIdenticalContent(final File file1, final File file2) - { - StopException.check(); - try - { - return FileUtils.contentEquals(file1, file2); - } catch (IOException e) - { - machineLog - .warn("It was not possible to compare the content of the file to check if creating links worked: " - + e.getMessage()); - } - return false; - } - - private final List<String> createLnCmdLine(final File srcFile, final File destFile) - { - final List<String> tokens = new ArrayList<String>(); - tokens.add(linkExecPath); - tokens.add(srcFile.getAbsolutePath()); - tokens.add(destFile.getAbsolutePath()); - return tokens; - } - - private static boolean runRepeatableProcess(final Callable<Boolean> task, - final int maxRetryOnFailure, final long millisToSleepOnFailure) - { - return new CallableExecutor(maxRetryOnFailure, millisToSleepOnFailure) - .executeCallable(task); - } } diff --git a/common/sourceTest/java/ch/systemsx/cisd/common/utilities/RecursiveHardLinkMakerTest.java b/common/sourceTest/java/ch/systemsx/cisd/common/utilities/RecursiveHardLinkMakerTest.java index d41290fe7418e8e9d0785722722d3f387701cadb..c8b5d900cce996cb90dbdda3f5346ac4598248bb 100644 --- a/common/sourceTest/java/ch/systemsx/cisd/common/utilities/RecursiveHardLinkMakerTest.java +++ b/common/sourceTest/java/ch/systemsx/cisd/common/utilities/RecursiveHardLinkMakerTest.java @@ -31,7 +31,7 @@ import org.testng.annotations.Test; import ch.systemsx.cisd.common.collections.CollectionIO; import ch.systemsx.cisd.common.logging.LogInitializer; import ch.systemsx.cisd.common.utilities.FileUtilities; -import ch.systemsx.cisd.common.utilities.IPathImmutableCopier; +import ch.systemsx.cisd.common.utilities.IImmutableCopier; import ch.systemsx.cisd.common.utilities.RecursiveHardLinkMaker; /** @@ -105,9 +105,9 @@ public class RecursiveHardLinkMakerTest assert file.isFile(); } - private static IPathImmutableCopier createHardLinkCopier() + private static IImmutableCopier createHardLinkCopier() { - IPathImmutableCopier copier = RecursiveHardLinkMaker.tryCreate(); + IImmutableCopier copier = RecursiveHardLinkMaker.tryCreate(HardLinkMaker.tryCreate()); assert copier != null; return copier; } @@ -143,7 +143,7 @@ public class RecursiveHardLinkMakerTest { File inputDir = createDirectory(workingDirectory, "resource-to-copy"); createStructure(inputDir); - assertTrue(createHardLinkCopier().copyDirectoryImmutably(inputDir, outputDir, null)); + assertTrue(createHardLinkCopier().copyImmutably(inputDir, outputDir, null)); File newInput = new File(outputDir, inputDir.getName()); assertStructureExists(newInput); @@ -170,7 +170,7 @@ public class RecursiveHardLinkMakerTest File src = createFile(workingDirectory, "fileXXX"); assertFileExists(src); - assertTrue(createHardLinkCopier().copyFileImmutably(src, outputDir, null)); + assertTrue(createHardLinkCopier().copyImmutably(src, outputDir, null)); File dest = new File(outputDir, src.getName()); assertFileExists(dest); diff --git a/datamover/build/build.xml b/datamover/build/build.xml index 9c4f5ca7a52d85e8ef37fd6dbbae39c6315258de..ed540f7c5c8590592079a338ccc98f19a401a604 100644 --- a/datamover/build/build.xml +++ b/datamover/build/build.xml @@ -9,6 +9,9 @@ <property name="jar.file" value="${dist.datamover.lib}/datamover.jar" /> <property name="dist.file.prefix" value="${dist}/datamover" /> <property name="lib" value="../libraries" /> + <property name="nativesrc" value="${lib}/filelink/native" /> + <property name="nativeroot" value="${targets}/ant" /> + <property name="native" value="${nativeroot}/native" /> <target name="clean"> <delete dir="${dist}" /> @@ -26,11 +29,19 @@ <mkdir dir="${dist.datamover.lib}" /> <build-info revision="revision.number" version="version.number" clean="clean.flag"/> <echo file="${build.info.file}">${version.number}:${revision.number}:${clean.flag}</echo> + <copy todir="${native}"> + <fileset dir="${nativesrc}"> + <include name="**/jlink.so"/> + </fileset> + </copy> <recursive-jar destfile="${jar.file}"> <fileset dir="${classes}"> <include name="**/*.class" /> <include name="${build.info.filename}" /> </fileset> + <fileset dir="${nativeroot}"> + <include name="**/jlink.so"/> + </fileset> <manifest> <attribute name="Main-Class" value="ch.systemsx.cisd.datamover.Main" /> <attribute name="Class-Path" diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/LocalProcessor.java b/datamover/source/java/ch/systemsx/cisd/datamover/LocalProcessor.java index 5a812febdff55dfa7eec554b337fd07f6b05e9ae..e7fe31f8391eaee68468f1bcbe58f3c6ef0699c7 100644 --- a/datamover/source/java/ch/systemsx/cisd/datamover/LocalProcessor.java +++ b/datamover/source/java/ch/systemsx/cisd/datamover/LocalProcessor.java @@ -31,7 +31,7 @@ import ch.systemsx.cisd.common.logging.Log4jSimpleLogger; import ch.systemsx.cisd.common.logging.LogCategory; import ch.systemsx.cisd.common.logging.LogFactory; import ch.systemsx.cisd.common.utilities.FileUtilities; -import ch.systemsx.cisd.common.utilities.IDirectoryImmutableCopier; +import ch.systemsx.cisd.common.utilities.IImmutableCopier; import ch.systemsx.cisd.common.utilities.IPathHandler; import ch.systemsx.cisd.datamover.filesystem.intf.IPathMover; import ch.systemsx.cisd.datamover.filesystem.intf.IRecoverableTimerTaskFactory; @@ -56,7 +56,7 @@ public final class LocalProcessor implements IPathHandler, IRecoverableTimerTask private static final Logger notificationLog = LogFactory.getLogger(LogCategory.NOTIFY, LocalProcessor.class); - private final IDirectoryImmutableCopier copier; + private final IImmutableCopier copier; private final IPathMover mover; @@ -81,7 +81,7 @@ public final class LocalProcessor implements IPathHandler, IRecoverableTimerTask private final File manualInterventionDir; LocalProcessor(final Parameters parameters, final LocalBufferDirs bufferDirs, - final IDirectoryImmutableCopier copier, final IPathMover mover) + final IImmutableCopier copier, final IPathMover mover) { this.inputDir = bufferDirs.getCopyCompleteDir(); this.outputDir = bufferDirs.getReadyToMoveDir(); @@ -277,7 +277,7 @@ public final class LocalProcessor implements IPathHandler, IRecoverableTimerTask operationLog.info(String.format("Creating extra copy of directory '%s' to '%s'.", path.getAbsolutePath(), tempDir.getAbsoluteFile())); } - final boolean ok = copier.copyDirectoryImmutably(path, tempDir, null); + final boolean ok = copier.copyImmutably(path, tempDir, null); if (ok == false) { notificationLog.error(String.format("Creating extra copy of '%s' failed.", path diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/FileSysOperationsFactory.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/FileSysOperationsFactory.java index 146c7aa0330bb2b3d7b07913b26d3d2438cdc484..ab59e8464d9e7fb2b83296712c6805fc12f4300f 100644 --- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/FileSysOperationsFactory.java +++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/FileSysOperationsFactory.java @@ -23,9 +23,9 @@ import org.apache.commons.lang.StringUtils; import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException; import ch.systemsx.cisd.common.filesystem.IPathCopier; import ch.systemsx.cisd.common.filesystem.rsync.RsyncCopier; -import ch.systemsx.cisd.common.utilities.IDirectoryImmutableCopier; +import ch.systemsx.cisd.common.utilities.FastRecursiveHardLinkMaker; +import ch.systemsx.cisd.common.utilities.IImmutableCopier; import ch.systemsx.cisd.common.utilities.OSUtilities; -import ch.systemsx.cisd.common.utilities.RsyncBasedRecursiveHardLinkMaker; import ch.systemsx.cisd.datamover.filesystem.intf.IFileSysOperationsFactory; import ch.systemsx.cisd.datamover.filesystem.intf.IPathMover; import ch.systemsx.cisd.datamover.filesystem.intf.IPathRemover; @@ -78,10 +78,10 @@ public class FileSysOperationsFactory implements IFileSysOperationsFactory parameters.getIntervalToWaitAfterFailure()); } - public final IDirectoryImmutableCopier getImmutableCopier() + public final IImmutableCopier getImmutableCopier() { final File rsyncExecutable = findRsyncExecutable(); - return new RsyncBasedRecursiveHardLinkMaker(rsyncExecutable); + return FastRecursiveHardLinkMaker.tryCreate(rsyncExecutable); } public final IPathCopier getCopier(final boolean requiresDeletionBeforeCreation) diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IFileSysOperationsFactory.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IFileSysOperationsFactory.java index 1e16b8b4417397d2a8db60aca30401638a56d830..3d1531a4e0f2a11ca98f54099b9c2ce0aeaa30ee 100644 --- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IFileSysOperationsFactory.java +++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/IFileSysOperationsFactory.java @@ -18,7 +18,7 @@ package ch.systemsx.cisd.datamover.filesystem.intf; import java.io.File; import ch.systemsx.cisd.common.filesystem.IPathCopier; -import ch.systemsx.cisd.common.utilities.IDirectoryImmutableCopier; +import ch.systemsx.cisd.common.utilities.IImmutableCopier; /** * A role that provides access to the roles which perform file system operations. @@ -29,7 +29,7 @@ public interface IFileSysOperationsFactory { public IPathCopier getCopier(boolean requiresDeletionBeforeCreation); - public IDirectoryImmutableCopier getImmutableCopier(); + public IImmutableCopier getImmutableCopier(); public IPathRemover getRemover();