diff --git a/common/source/java/ch/systemsx/cisd/common/filesystem/rsync/RsyncCopier.java b/common/source/java/ch/systemsx/cisd/common/filesystem/rsync/RsyncCopier.java
index 8f86b6f58cac7cb34d9b340883b60e310b9af11e..997d0c01e7323fed05edab854a4a502d5a307bf6 100644
--- a/common/source/java/ch/systemsx/cisd/common/filesystem/rsync/RsyncCopier.java
+++ b/common/source/java/ch/systemsx/cisd/common/filesystem/rsync/RsyncCopier.java
@@ -51,11 +51,11 @@ import ch.systemsx.cisd.common.utilities.ITerminable;
*/
public final class RsyncCopier implements IPathCopier, IDirectoryImmutableCopier
{
- private static final Logger machineLog =
- LogFactory.getLogger(LogCategory.MACHINE, RsyncCopier.class);
+ private static final Logger machineLog = LogFactory.getLogger(LogCategory.MACHINE,
+ RsyncCopier.class);
- private static final Logger operationLog =
- LogFactory.getLogger(LogCategory.OPERATION, RsyncCopier.class);
+ private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
+ RsyncCopier.class);
/**
* The {@link Status} returned if the process was terminated by {@link Process#destroy()}.
@@ -63,11 +63,11 @@ public final class RsyncCopier implements IPathCopier, IDirectoryImmutableCopier
@Private
static final Status TERMINATED_STATUS = Status.createRetriableError("Process was terminated.");
- private static final Status INTERRUPTED_STATUS =
- Status.createRetriableError("Process was interrupted.");
+ private static final Status INTERRUPTED_STATUS = Status
+ .createRetriableError("Process was interrupted.");
- private static final Status TIMEOUT_STATUS =
- Status.createRetriableError("Process has stopped because of timeout.");
+ private static final Status TIMEOUT_STATUS = Status
+ .createRetriableError("Process has stopped because of timeout.");
private final String rsyncExecutable;
@@ -103,7 +103,10 @@ public final class RsyncCopier implements IPathCopier, IDirectoryImmutableCopier
private final String sshExecutable;
- private final List<String> additionalCmdLineFlags;
+ private final List<String> additionalCmdLineFlagsOrNull;
+
+ /** If set, overrides all command line parameters for mutable copying. */
+ private final List<String> cmdLineFlagsOrNull;
private final boolean overwrite;
@@ -127,6 +130,30 @@ public final class RsyncCopier implements IPathCopier, IDirectoryImmutableCopier
this(rsyncExecutable, null, false, false);
}
+ /**
+ * Constructs an <code>RsyncCopier</code> with fully custom command line flags.
+ *
+ * @param rsyncExecutable The <code>rsync</code> binary to call for copying.
+ * @param sshExecutableOrNull The <code>ssh</code> binary to use for creating tunnels, or
+ * <code>null</code>, if no <code>ssh</code> is available on this machine.
+ * @param cmdLineFlags The command line flags to use for the rsync command.
+ */
+ public RsyncCopier(final File rsyncExecutable, final File sshExecutableOrNull,
+ String... cmdLineFlags)
+ {
+ assert rsyncExecutable != null && rsyncExecutable.exists();
+ assert sshExecutableOrNull == null || rsyncExecutable.exists();
+
+ this.rsyncExecutable = rsyncExecutable.getAbsolutePath();
+ this.rsyncVersion = RsyncVersionChecker.getVersion(rsyncExecutable.getAbsolutePath());
+ this.sshExecutable = (sshExecutableOrNull != null) ? sshExecutableOrNull.getPath() : null;
+ this.rsyncTerminator = new AtomicReference<ITerminable>(null);
+ this.overwrite = false;
+ this.destinationDirectoryRequiresDeletionBeforeCreation = false;
+ this.additionalCmdLineFlagsOrNull = null;
+ this.cmdLineFlagsOrNull = Arrays.asList(cmdLineFlags);
+ }
+
/**
* Constructs an <code>RsyncCopier</code>.
*
@@ -153,11 +180,12 @@ public final class RsyncCopier implements IPathCopier, IDirectoryImmutableCopier
this.rsyncTerminator = new AtomicReference<ITerminable>(null);
if (cmdLineFlags.length > 0)
{
- this.additionalCmdLineFlags = Arrays.asList(cmdLineFlags);
+ this.additionalCmdLineFlagsOrNull = Arrays.asList(cmdLineFlags);
} else
{
- this.additionalCmdLineFlags = null;
+ this.additionalCmdLineFlagsOrNull = null;
}
+ this.cmdLineFlagsOrNull = null;
}
private boolean rsyncSupportsAppend(RsyncVersion versionOrNull)
@@ -237,8 +265,10 @@ public final class RsyncCopier implements IPathCopier, IDirectoryImmutableCopier
assert destinationDirectory.isDirectory() : destinationDirectory.getAbsolutePath();
final List<String> commandLine =
- createCommandLineForImmutableCopy(sourceDirectory, createTargetDirectory(
- sourceDirectory, destinationDirectory, targetNameOrNull));
+ createCommandLineForImmutableCopy(
+ sourceDirectory,
+ createTargetDirectory(sourceDirectory, destinationDirectory,
+ targetNameOrNull));
final ProcessResult processResult =
runCommand(commandLine, ProcessExecutionHelper.DEFAULT_OUTPUT_READING_STRATEGY);
return processResult.isOK();
@@ -329,8 +359,8 @@ public final class RsyncCopier implements IPathCopier, IDirectoryImmutableCopier
}
if (rsyncVersion.isRsyncPreReleaseVersion())
{
- machineLog.warn(String
- .format(
+ machineLog
+ .warn(String.format(
"The rsync executable '%s' is a pre-release version. It is not recommended "
+ "to use such a version in a production environment.",
rsyncExecutable));
@@ -425,12 +455,13 @@ public final class RsyncCopier implements IPathCopier, IDirectoryImmutableCopier
{
if (result.getOutput().size() == 0)
{
- machineLog.warn(String.format("No output on command '%s'.", StringUtils.join(
- commandLineList, ' ')));
+ machineLog.warn(String.format("No output on command '%s'.",
+ StringUtils.join(commandLineList, ' ')));
} else
{
- machineLog.warn(String.format("Unexpected output on command '%s':\n%s", StringUtils
- .join(commandLineList, ' '), StringUtils.join(result.getOutput(), '\n')));
+ machineLog.warn(String.format("Unexpected output on command '%s':\n%s",
+ StringUtils.join(commandLineList, ' '),
+ StringUtils.join(result.getOutput(), '\n')));
}
}
if (result.isOK() && result.getOutput().size() == 1)
@@ -485,19 +516,25 @@ public final class RsyncCopier implements IPathCopier, IDirectoryImmutableCopier
|| (destinationHostOrNull == null);
assert (sourceHostOrNull != null && sshExecutable != null) || (sourceHostOrNull == null);
- final List<String> standardParameters = Arrays.asList("--archive", "--delete", "--inplace");
final List<String> commandLineList = new ArrayList<String>();
final RsyncRecord remoteRsyncOrNull =
tryGetRemoteRsync(sourceHostOrNull, destinationHostOrNull);
-
commandLineList.add(rsyncExecutable);
- commandLineList.addAll(standardParameters);
- if (isOverwriteMode(remoteRsyncOrNull))
+ if (cmdLineFlagsOrNull != null)
{
- commandLineList.add("--whole-file");
+ commandLineList.addAll(cmdLineFlagsOrNull);
} else
{
- commandLineList.add("--append");
+ final List<String> standardParameters = Arrays.asList("--archive", "--delete", "--inplace");
+
+ commandLineList.addAll(standardParameters);
+ if (isOverwriteMode(remoteRsyncOrNull))
+ {
+ commandLineList.add("--whole-file");
+ } else
+ {
+ commandLineList.add("--append");
+ }
}
if (sshExecutable != null && (destinationHostOrNull != null || sourceHostOrNull != null)
&& rsyncModuleNameOrNull == null)
@@ -516,9 +553,9 @@ public final class RsyncCopier implements IPathCopier, IDirectoryImmutableCopier
commandLineList.add("--password-file");
commandLineList.add(rsyncPasswordFileOrNull);
}
- if (additionalCmdLineFlags != null)
+ if (additionalCmdLineFlagsOrNull != null)
{
- commandLineList.addAll(additionalCmdLineFlags);
+ commandLineList.addAll(additionalCmdLineFlagsOrNull);
}
commandLineList.add(buildPath(sourceHostOrNull, sourcePath, rsyncModuleNameOrNull,
copyDirectoryContent));
diff --git a/common/sourceTest/java/ch/systemsx/cisd/common/filesystem/rsync/RsyncCopierTest.java b/common/sourceTest/java/ch/systemsx/cisd/common/filesystem/rsync/RsyncCopierTest.java
index 2ef3343cc012f421fc797e6e6f45652078b14a51..ec8932aca9ece271bb55c9ad2e1cf2e0b3e94a95 100644
--- a/common/sourceTest/java/ch/systemsx/cisd/common/filesystem/rsync/RsyncCopierTest.java
+++ b/common/sourceTest/java/ch/systemsx/cisd/common/filesystem/rsync/RsyncCopierTest.java
@@ -53,13 +53,13 @@ import ch.systemsx.cisd.common.test.StoringUncaughtExceptionHandler;
@Friend(toClasses = RsyncCopier.class)
public final class RsyncCopierTest
{
- private static final Logger operationLog =
- LogFactory.getLogger(LogCategory.OPERATION, RsyncCopierTest.class);
+ private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
+ RsyncCopierTest.class);
private static final long SLEEP_MILLIS = 1000L;
- private static final File unitTestRootDirectory =
- new File("targets" + File.separator + "unit-test-wd");
+ private static final File unitTestRootDirectory = new File("targets" + File.separator
+ + "unit-test-wd");
private static final File workingDirectory = new File(unitTestRootDirectory, "RsyncCopierTest");
@@ -128,8 +128,8 @@ public final class RsyncCopierTest
final File rsyncBinary = new File(workingDirectory, "rsync");
rsyncBinary.delete();
final List<String> lines = new ArrayList<String>();
- lines.addAll(Arrays.asList("#! /bin/sh", "if [ \"$1\" = \"--version\" ]; then ", String
- .format(" echo \"rsync version %s\"", rsyncVersion), "exit 0", "fi"));
+ lines.addAll(Arrays.asList("#! /bin/sh", "if [ \"$1\" = \"--version\" ]; then ",
+ String.format(" echo \"rsync version %s\"", rsyncVersion), "exit 0", "fi"));
lines.addAll(Arrays.asList(additionalLines));
CollectionIO.writeIterable(rsyncBinary, lines);
Runtime.getRuntime().exec(String.format("/bin/chmod +x %s", rsyncBinary.getPath()))
@@ -156,6 +156,40 @@ public final class RsyncCopierTest
assertEquals(destinationDirectory.getAbsolutePath() + "/", cmdLine.get(cmdLine.size() - 1));
}
+ @Test
+ public void testCommandLineForMutableCopyLocalNoOwnerNoGroup() throws IOException,
+ InterruptedException
+ {
+ final File rsyncBinary = createRsync(0);
+ final RsyncCopier copier =
+ new RsyncCopier(rsyncBinary, rsyncBinary, false, false, "--no-owner", "--no-group");
+ final List<String> cmdLine =
+ copier.createCommandLineForMutableCopy(sourceDirectory, null, destinationDirectory,
+ null, null, null, false);
+ assertEquals(rsyncBinary.getAbsolutePath(), cmdLine.get(0));
+ assertEquals("--no-owner", cmdLine.get(cmdLine.size() - 4));
+ assertEquals("--no-group", cmdLine.get(cmdLine.size() - 3));
+ assertEquals(sourceDirectory.getAbsolutePath(), cmdLine.get(cmdLine.size() - 2));
+ assertEquals(destinationDirectory.getAbsolutePath() + "/", cmdLine.get(cmdLine.size() - 1));
+ }
+
+ @Test
+ public void testCommandLineForMutableCopyCustomParameters() throws IOException,
+ InterruptedException
+ {
+ final File rsyncBinary = createRsync(0);
+ final RsyncCopier copier =
+ new RsyncCopier(rsyncBinary, rsyncBinary, "--archive", "--no-perms");
+ final List<String> cmdLine =
+ copier.createCommandLineForMutableCopy(sourceDirectory, null, destinationDirectory,
+ null, null, null, false);
+ assertEquals(rsyncBinary.getAbsolutePath(), cmdLine.get(0));
+ assertEquals("--archive", cmdLine.get(1));
+ assertEquals("--no-perms", cmdLine.get(2));
+ assertEquals(sourceDirectory.getAbsolutePath(), cmdLine.get(3));
+ assertEquals(destinationDirectory.getAbsolutePath() + "/", cmdLine.get(4));
+ }
+
@Test
public void testCommandLineForMutableCopyLocalContentRatherThanDirectory() throws IOException,
InterruptedException
@@ -332,14 +366,14 @@ public final class RsyncCopierTest
{
final File parametersLogFile = new File(workingDirectory, "parameters.log");
final File loggingRsyncBinary =
- createRsync("2.6.7", String.format("echo \"$@\" > %s", parametersLogFile
- .getAbsolutePath()));
+ createRsync("2.6.7",
+ String.format("echo \"$@\" > %s", parametersLogFile.getAbsolutePath()));
final RsyncCopier copier = new RsyncCopier(loggingRsyncBinary, null, false, false);
final Status status = copier.copy(sourceDirectory, destinationDirectory);
assertEquals(Status.OK, status);
final String expectedRsyncCmdLine =
- String.format("--archive --delete --inplace --append %s %s/\n", sourceDirectory
- .getAbsolutePath(), destinationDirectory.getAbsolutePath());
+ String.format("--archive --delete --inplace --append %s %s/\n",
+ sourceDirectory.getAbsolutePath(), destinationDirectory.getAbsolutePath());
final String observedRsyncCmdLine = FileUtilities.loadToString(parametersLogFile);
assertEquals(expectedRsyncCmdLine, observedRsyncCmdLine);
}
@@ -350,14 +384,14 @@ public final class RsyncCopierTest
{
final File parametersLogFile = new File(workingDirectory, "parameters.log");
final File loggingRsyncBinary =
- createRsync("2.6.7", String.format("echo \"$@\" > %s", parametersLogFile
- .getAbsolutePath()));
+ createRsync("2.6.7",
+ String.format("echo \"$@\" > %s", parametersLogFile.getAbsolutePath()));
final RsyncCopier copier = new RsyncCopier(loggingRsyncBinary, null, false, false);
final Status status = copier.copyContent(sourceDirectory, destinationDirectory);
assertEquals(Status.OK, status);
final String expectedRsyncCmdLine =
- String.format("--archive --delete --inplace --append %s/ %s/\n", sourceDirectory
- .getAbsolutePath(), destinationDirectory.getAbsolutePath());
+ String.format("--archive --delete --inplace --append %s/ %s/\n",
+ sourceDirectory.getAbsolutePath(), destinationDirectory.getAbsolutePath());
final String observedRsyncCmdLine = FileUtilities.loadToString(parametersLogFile);
assertEquals(expectedRsyncCmdLine, observedRsyncCmdLine);
}
@@ -368,8 +402,8 @@ public final class RsyncCopierTest
{
final File parametersLogFile = new File(workingDirectory, "parameters.log");
final File loggingRsyncBinary =
- createRsync("2.6.7", String.format("echo \"$@\" > %s", parametersLogFile
- .getAbsolutePath()));
+ createRsync("2.6.7",
+ String.format("echo \"$@\" > %s", parametersLogFile.getAbsolutePath()));
final RsyncCopier copier = new RsyncCopier(loggingRsyncBinary, null, false, false);
assertTrue(copier.copyDirectoryImmutably(sourceDirectory, destinationDirectory, null));
final String absWd = workingDirectory.getAbsolutePath();
@@ -385,8 +419,8 @@ public final class RsyncCopierTest
{
final File parametersLogFile = new File(workingDirectory, "parameters.log");
final File loggingRsyncBinary =
- createRsync("2.6.7", String.format("echo \"$@\" > %s", parametersLogFile
- .getAbsolutePath()));
+ createRsync("2.6.7",
+ String.format("echo \"$@\" > %s", parametersLogFile.getAbsolutePath()));
final RsyncCopier copier = new RsyncCopier(loggingRsyncBinary, null, false, false);
final String name = "xxx";
assertTrue(copier.copyDirectoryImmutably(sourceDirectory, destinationDirectory, name));
@@ -443,8 +477,8 @@ public final class RsyncCopierTest
{
final File parametersLogFile = new File(workingDirectory, "parameters.log");
final File loggingRsyncBinary =
- createRsync("2.6.7", String.format("echo \"$@\" > %s", parametersLogFile
- .getAbsolutePath()));
+ createRsync("2.6.7",
+ String.format("echo \"$@\" > %s", parametersLogFile.getAbsolutePath()));
final RsyncCopier copier = new RsyncCopier(loggingRsyncBinary, null, false, false);
copier.copy(sourceFile, destinationDirectory);
final String rsyncParameters = FileUtilities.loadToString(parametersLogFile);
@@ -458,8 +492,8 @@ public final class RsyncCopierTest
{
final File parametersLogFile = new File(workingDirectory, "parameters.log");
final File loggingRsyncBinary =
- createRsync("2.6.6", String.format("echo \"$@\" > %s", parametersLogFile
- .getAbsolutePath()));
+ createRsync("2.6.6",
+ String.format("echo \"$@\" > %s", parametersLogFile.getAbsolutePath()));
final RsyncCopier copier = new RsyncCopier(loggingRsyncBinary, null, false, false);
copier.copy(sourceFile, destinationDirectory);
final String rsyncParameters = FileUtilities.loadToString(parametersLogFile);
@@ -473,8 +507,8 @@ public final class RsyncCopierTest
{
final File parametersLogFile = new File(workingDirectory, "parameters.log");
final File loggingRsyncBinary =
- createRsync("2.6.7", String.format("echo \"$@\" > %s", parametersLogFile
- .getAbsolutePath()));
+ createRsync("2.6.7",
+ String.format("echo \"$@\" > %s", parametersLogFile.getAbsolutePath()));
final RsyncCopier copier = new RsyncCopier(loggingRsyncBinary, null, false, true);
copier.copy(sourceFile, destinationDirectory);
final String rsyncParameters = FileUtilities.loadToString(parametersLogFile);