add: support for using custom parameters with rsync

SVN: 18644

common/source/java/ch/systemsx/cisd/common/filesystem/rsync/RsyncCopier.java

@@ -51,11 +51,11 @@ import ch.systemsx.cisd.common.utilities.ITerminable;
  */
 public final class RsyncCopier implements IPathCopier, IDirectoryImmutableCopier
 {
-    private static final Logger machineLog =
-            LogFactory.getLogger(LogCategory.MACHINE, RsyncCopier.class);
+    private static final Logger machineLog = LogFactory.getLogger(LogCategory.MACHINE,
+            RsyncCopier.class);
 
-    private static final Logger operationLog =
-            LogFactory.getLogger(LogCategory.OPERATION, RsyncCopier.class);
+    private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
+            RsyncCopier.class);
 
     /**
      * The {@link Status} returned if the process was terminated by {@link Process#destroy()}.
@@ -63,11 +63,11 @@ public final class RsyncCopier implements IPathCopier, IDirectoryImmutableCopier
     @Private
     static final Status TERMINATED_STATUS = Status.createRetriableError("Process was terminated.");
 
-    private static final Status INTERRUPTED_STATUS =
-            Status.createRetriableError("Process was interrupted.");
+    private static final Status INTERRUPTED_STATUS = Status
+            .createRetriableError("Process was interrupted.");
 
-    private static final Status TIMEOUT_STATUS =
-            Status.createRetriableError("Process has stopped because of timeout.");
+    private static final Status TIMEOUT_STATUS = Status
+            .createRetriableError("Process has stopped because of timeout.");
 
     private final String rsyncExecutable;
 
@@ -103,7 +103,10 @@ public final class RsyncCopier implements IPathCopier, IDirectoryImmutableCopier
 
     private final String sshExecutable;
 
-    private final List<String> additionalCmdLineFlags;
+    private final List<String> additionalCmdLineFlagsOrNull;
+    
+    /** If set, overrides all command line parameters for mutable copying. */
+    private final List<String> cmdLineFlagsOrNull;
 
     private final boolean overwrite;
 
@@ -127,6 +130,30 @@ public final class RsyncCopier implements IPathCopier, IDirectoryImmutableCopier
         this(rsyncExecutable, null, false, false);
     }
 
+    /**
+     * Constructs an <code>RsyncCopier</code> with fully custom command line flags.
+     * 
+     * @param rsyncExecutable The <code>rsync</code> binary to call for copying.
+     * @param sshExecutableOrNull The <code>ssh</code> binary to use for creating tunnels, or
+     *            <code>null</code>, if no <code>ssh</code> is available on this machine.
+     * @param cmdLineFlags The command line flags to use for the rsync command.
+     */
+    public RsyncCopier(final File rsyncExecutable, final File sshExecutableOrNull,
+            String... cmdLineFlags)
+    {
+        assert rsyncExecutable != null && rsyncExecutable.exists();
+        assert sshExecutableOrNull == null || rsyncExecutable.exists();
+
+        this.rsyncExecutable = rsyncExecutable.getAbsolutePath();
+        this.rsyncVersion = RsyncVersionChecker.getVersion(rsyncExecutable.getAbsolutePath());
+        this.sshExecutable = (sshExecutableOrNull != null) ? sshExecutableOrNull.getPath() : null;
+        this.rsyncTerminator = new AtomicReference<ITerminable>(null);
+        this.overwrite = false;
+        this.destinationDirectoryRequiresDeletionBeforeCreation = false;
+        this.additionalCmdLineFlagsOrNull = null;
+        this.cmdLineFlagsOrNull = Arrays.asList(cmdLineFlags);
+    }
+
     /**
      * Constructs an <code>RsyncCopier</code>.
      * 
@@ -153,11 +180,12 @@ public final class RsyncCopier implements IPathCopier, IDirectoryImmutableCopier
         this.rsyncTerminator = new AtomicReference<ITerminable>(null);
         if (cmdLineFlags.length > 0)
         {
-            this.additionalCmdLineFlags = Arrays.asList(cmdLineFlags);
+            this.additionalCmdLineFlagsOrNull = Arrays.asList(cmdLineFlags);
         } else
         {
-            this.additionalCmdLineFlags = null;
+            this.additionalCmdLineFlagsOrNull = null;
         }
+        this.cmdLineFlagsOrNull = null;
     }
 
     private boolean rsyncSupportsAppend(RsyncVersion versionOrNull)
@@ -237,8 +265,10 @@ public final class RsyncCopier implements IPathCopier, IDirectoryImmutableCopier
         assert destinationDirectory.isDirectory() : destinationDirectory.getAbsolutePath();
 
         final List<String> commandLine =
-                createCommandLineForImmutableCopy(sourceDirectory, createTargetDirectory(
-                        sourceDirectory, destinationDirectory, targetNameOrNull));
+                createCommandLineForImmutableCopy(
+                        sourceDirectory,
+                        createTargetDirectory(sourceDirectory, destinationDirectory,
+                                targetNameOrNull));
         final ProcessResult processResult =
                 runCommand(commandLine, ProcessExecutionHelper.DEFAULT_OUTPUT_READING_STRATEGY);
         return processResult.isOK();
@@ -329,8 +359,8 @@ public final class RsyncCopier implements IPathCopier, IDirectoryImmutableCopier
         }
         if (rsyncVersion.isRsyncPreReleaseVersion())
         {
-            machineLog.warn(String
-                    .format(
+            machineLog
+                    .warn(String.format(
                             "The rsync executable '%s' is a pre-release version. It is not recommended "
                                     + "to use such a version in a production environment.",
                             rsyncExecutable));
@@ -425,12 +455,13 @@ public final class RsyncCopier implements IPathCopier, IDirectoryImmutableCopier
         {
             if (result.getOutput().size() == 0)
             {
-                machineLog.warn(String.format("No output on command '%s'.", StringUtils.join(
-                        commandLineList, ' ')));
+                machineLog.warn(String.format("No output on command '%s'.",
+                        StringUtils.join(commandLineList, ' ')));
             } else
             {
-                machineLog.warn(String.format("Unexpected output on command '%s':\n%s", StringUtils
-                        .join(commandLineList, ' '), StringUtils.join(result.getOutput(), '\n')));
+                machineLog.warn(String.format("Unexpected output on command '%s':\n%s",
+                        StringUtils.join(commandLineList, ' '),
+                        StringUtils.join(result.getOutput(), '\n')));
             }
         }
         if (result.isOK() && result.getOutput().size() == 1)
@@ -485,19 +516,25 @@ public final class RsyncCopier implements IPathCopier, IDirectoryImmutableCopier
                 || (destinationHostOrNull == null);
         assert (sourceHostOrNull != null && sshExecutable != null) || (sourceHostOrNull == null);
 
-        final List<String> standardParameters = Arrays.asList("--archive", "--delete", "--inplace");
         final List<String> commandLineList = new ArrayList<String>();
         final RsyncRecord remoteRsyncOrNull =
                 tryGetRemoteRsync(sourceHostOrNull, destinationHostOrNull);
-
         commandLineList.add(rsyncExecutable);
-        commandLineList.addAll(standardParameters);
-        if (isOverwriteMode(remoteRsyncOrNull))
+        if (cmdLineFlagsOrNull != null)
         {
-            commandLineList.add("--whole-file");
+            commandLineList.addAll(cmdLineFlagsOrNull);
         } else
         {
-            commandLineList.add("--append");
+            final List<String> standardParameters = Arrays.asList("--archive", "--delete", "--inplace");
+    
+            commandLineList.addAll(standardParameters);
+            if (isOverwriteMode(remoteRsyncOrNull))
+            {
+                commandLineList.add("--whole-file");
+            } else
+            {
+                commandLineList.add("--append");
+            }
         }
         if (sshExecutable != null && (destinationHostOrNull != null || sourceHostOrNull != null)
                 && rsyncModuleNameOrNull == null)
@@ -516,9 +553,9 @@ public final class RsyncCopier implements IPathCopier, IDirectoryImmutableCopier
             commandLineList.add("--password-file");
             commandLineList.add(rsyncPasswordFileOrNull);
         }
-        if (additionalCmdLineFlags != null)
+        if (additionalCmdLineFlagsOrNull != null)
         {
-            commandLineList.addAll(additionalCmdLineFlags);
+            commandLineList.addAll(additionalCmdLineFlagsOrNull);
         }
         commandLineList.add(buildPath(sourceHostOrNull, sourcePath, rsyncModuleNameOrNull,
                 copyDirectoryContent));

common/sourceTest/java/ch/systemsx/cisd/common/filesystem/rsync/RsyncCopierTest.java

@@ -53,13 +53,13 @@ import ch.systemsx.cisd.common.test.StoringUncaughtExceptionHandler;
 @Friend(toClasses = RsyncCopier.class)
 public final class RsyncCopierTest
 {
-    private static final Logger operationLog =
-            LogFactory.getLogger(LogCategory.OPERATION, RsyncCopierTest.class);
+    private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
+            RsyncCopierTest.class);
 
     private static final long SLEEP_MILLIS = 1000L;
 
-    private static final File unitTestRootDirectory =
-            new File("targets" + File.separator + "unit-test-wd");
+    private static final File unitTestRootDirectory = new File("targets" + File.separator
+            + "unit-test-wd");
 
     private static final File workingDirectory = new File(unitTestRootDirectory, "RsyncCopierTest");
 
@@ -128,8 +128,8 @@ public final class RsyncCopierTest
         final File rsyncBinary = new File(workingDirectory, "rsync");
         rsyncBinary.delete();
         final List<String> lines = new ArrayList<String>();
-        lines.addAll(Arrays.asList("#! /bin/sh", "if [ \"$1\" = \"--version\" ]; then ", String
-                .format("  echo \"rsync  version %s\"", rsyncVersion), "exit 0", "fi"));
+        lines.addAll(Arrays.asList("#! /bin/sh", "if [ \"$1\" = \"--version\" ]; then ",
+                String.format("  echo \"rsync  version %s\"", rsyncVersion), "exit 0", "fi"));
         lines.addAll(Arrays.asList(additionalLines));
         CollectionIO.writeIterable(rsyncBinary, lines);
         Runtime.getRuntime().exec(String.format("/bin/chmod +x %s", rsyncBinary.getPath()))
@@ -156,6 +156,40 @@ public final class RsyncCopierTest
         assertEquals(destinationDirectory.getAbsolutePath() + "/", cmdLine.get(cmdLine.size() - 1));
     }
 
+    @Test
+    public void testCommandLineForMutableCopyLocalNoOwnerNoGroup() throws IOException,
+            InterruptedException
+    {
+        final File rsyncBinary = createRsync(0);
+        final RsyncCopier copier =
+                new RsyncCopier(rsyncBinary, rsyncBinary, false, false, "--no-owner", "--no-group");
+        final List<String> cmdLine =
+                copier.createCommandLineForMutableCopy(sourceDirectory, null, destinationDirectory,
+                        null, null, null, false);
+        assertEquals(rsyncBinary.getAbsolutePath(), cmdLine.get(0));
+        assertEquals("--no-owner", cmdLine.get(cmdLine.size() - 4));
+        assertEquals("--no-group", cmdLine.get(cmdLine.size() - 3));
+        assertEquals(sourceDirectory.getAbsolutePath(), cmdLine.get(cmdLine.size() - 2));
+        assertEquals(destinationDirectory.getAbsolutePath() + "/", cmdLine.get(cmdLine.size() - 1));
+    }
+
+    @Test
+    public void testCommandLineForMutableCopyCustomParameters() throws IOException,
+            InterruptedException
+    {
+        final File rsyncBinary = createRsync(0);
+        final RsyncCopier copier =
+                new RsyncCopier(rsyncBinary, rsyncBinary, "--archive", "--no-perms");
+        final List<String> cmdLine =
+                copier.createCommandLineForMutableCopy(sourceDirectory, null, destinationDirectory,
+                        null, null, null, false);
+        assertEquals(rsyncBinary.getAbsolutePath(), cmdLine.get(0));
+        assertEquals("--archive", cmdLine.get(1));
+        assertEquals("--no-perms", cmdLine.get(2));
+        assertEquals(sourceDirectory.getAbsolutePath(), cmdLine.get(3));
+        assertEquals(destinationDirectory.getAbsolutePath() + "/", cmdLine.get(4));
+    }
+
     @Test
     public void testCommandLineForMutableCopyLocalContentRatherThanDirectory() throws IOException,
             InterruptedException
@@ -332,14 +366,14 @@ public final class RsyncCopierTest
     {
         final File parametersLogFile = new File(workingDirectory, "parameters.log");
         final File loggingRsyncBinary =
-                createRsync("2.6.7", String.format("echo \"$@\" > %s", parametersLogFile
-                        .getAbsolutePath()));
+                createRsync("2.6.7",
+                        String.format("echo \"$@\" > %s", parametersLogFile.getAbsolutePath()));
         final RsyncCopier copier = new RsyncCopier(loggingRsyncBinary, null, false, false);
         final Status status = copier.copy(sourceDirectory, destinationDirectory);
         assertEquals(Status.OK, status);
         final String expectedRsyncCmdLine =
-                String.format("--archive --delete --inplace --append %s %s/\n", sourceDirectory
-                        .getAbsolutePath(), destinationDirectory.getAbsolutePath());
+                String.format("--archive --delete --inplace --append %s %s/\n",
+                        sourceDirectory.getAbsolutePath(), destinationDirectory.getAbsolutePath());
         final String observedRsyncCmdLine = FileUtilities.loadToString(parametersLogFile);
         assertEquals(expectedRsyncCmdLine, observedRsyncCmdLine);
     }
@@ -350,14 +384,14 @@ public final class RsyncCopierTest
     {
         final File parametersLogFile = new File(workingDirectory, "parameters.log");
         final File loggingRsyncBinary =
-                createRsync("2.6.7", String.format("echo \"$@\" > %s", parametersLogFile
-                        .getAbsolutePath()));
+                createRsync("2.6.7",
+                        String.format("echo \"$@\" > %s", parametersLogFile.getAbsolutePath()));
         final RsyncCopier copier = new RsyncCopier(loggingRsyncBinary, null, false, false);
         final Status status = copier.copyContent(sourceDirectory, destinationDirectory);
         assertEquals(Status.OK, status);
         final String expectedRsyncCmdLine =
-                String.format("--archive --delete --inplace --append %s/ %s/\n", sourceDirectory
-                        .getAbsolutePath(), destinationDirectory.getAbsolutePath());
+                String.format("--archive --delete --inplace --append %s/ %s/\n",
+                        sourceDirectory.getAbsolutePath(), destinationDirectory.getAbsolutePath());
         final String observedRsyncCmdLine = FileUtilities.loadToString(parametersLogFile);
         assertEquals(expectedRsyncCmdLine, observedRsyncCmdLine);
     }
@@ -368,8 +402,8 @@ public final class RsyncCopierTest
     {
         final File parametersLogFile = new File(workingDirectory, "parameters.log");
         final File loggingRsyncBinary =
-                createRsync("2.6.7", String.format("echo \"$@\" > %s", parametersLogFile
-                        .getAbsolutePath()));
+                createRsync("2.6.7",
+                        String.format("echo \"$@\" > %s", parametersLogFile.getAbsolutePath()));
         final RsyncCopier copier = new RsyncCopier(loggingRsyncBinary, null, false, false);
         assertTrue(copier.copyDirectoryImmutably(sourceDirectory, destinationDirectory, null));
         final String absWd = workingDirectory.getAbsolutePath();
@@ -385,8 +419,8 @@ public final class RsyncCopierTest
     {
         final File parametersLogFile = new File(workingDirectory, "parameters.log");
         final File loggingRsyncBinary =
-                createRsync("2.6.7", String.format("echo \"$@\" > %s", parametersLogFile
-                        .getAbsolutePath()));
+                createRsync("2.6.7",
+                        String.format("echo \"$@\" > %s", parametersLogFile.getAbsolutePath()));
         final RsyncCopier copier = new RsyncCopier(loggingRsyncBinary, null, false, false);
         final String name = "xxx";
         assertTrue(copier.copyDirectoryImmutably(sourceDirectory, destinationDirectory, name));
@@ -443,8 +477,8 @@ public final class RsyncCopierTest
     {
         final File parametersLogFile = new File(workingDirectory, "parameters.log");
         final File loggingRsyncBinary =
-                createRsync("2.6.7", String.format("echo \"$@\" > %s", parametersLogFile
-                        .getAbsolutePath()));
+                createRsync("2.6.7",
+                        String.format("echo \"$@\" > %s", parametersLogFile.getAbsolutePath()));
         final RsyncCopier copier = new RsyncCopier(loggingRsyncBinary, null, false, false);
         copier.copy(sourceFile, destinationDirectory);
         final String rsyncParameters = FileUtilities.loadToString(parametersLogFile);
@@ -458,8 +492,8 @@ public final class RsyncCopierTest
     {
         final File parametersLogFile = new File(workingDirectory, "parameters.log");
         final File loggingRsyncBinary =
-                createRsync("2.6.6", String.format("echo \"$@\" > %s", parametersLogFile
-                        .getAbsolutePath()));
+                createRsync("2.6.6",
+                        String.format("echo \"$@\" > %s", parametersLogFile.getAbsolutePath()));
         final RsyncCopier copier = new RsyncCopier(loggingRsyncBinary, null, false, false);
         copier.copy(sourceFile, destinationDirectory);
         final String rsyncParameters = FileUtilities.loadToString(parametersLogFile);
@@ -473,8 +507,8 @@ public final class RsyncCopierTest
     {
         final File parametersLogFile = new File(workingDirectory, "parameters.log");
         final File loggingRsyncBinary =
-                createRsync("2.6.7", String.format("echo \"$@\" > %s", parametersLogFile
-                        .getAbsolutePath()));
+                createRsync("2.6.7",
+                        String.format("echo \"$@\" > %s", parametersLogFile.getAbsolutePath()));
         final RsyncCopier copier = new RsyncCopier(loggingRsyncBinary, null, false, true);
         copier.copy(sourceFile, destinationDirectory);
         final String rsyncParameters = FileUtilities.loadToString(parametersLogFile);