From 4f4d88a76dc5001ef5e074ad791c57a96830a751 Mon Sep 17 00:00:00 2001
From: brinn <brinn>
Date: Tue, 22 Jul 2008 13:41:33 +0000
Subject: [PATCH] [DMV-33] add support for password files when connecting to an
rsync server
SVN: 7396
---
.../cisd/common/filesystem/IPathCopier.java | 17 +++--
.../common/filesystem/rsync/RsyncCopier.java | 34 ++++++---
.../filesystem/rsync/RsyncCopierTest.java | 73 +++++++++++++++----
.../cisd/datamover/DatamoverConstants.java | 31 ++++++++
.../systemsx/cisd/datamover/Parameters.java | 4 +-
.../cisd/datamover/PropertyNames.java | 2 -
.../filesystem/intf/AbstractFileStore.java | 21 ++++--
.../systemsx/cisd/datamover/SelfTestTest.java | 9 ++-
8 files changed, 148 insertions(+), 43 deletions(-)
create mode 100644 datamover/source/java/ch/systemsx/cisd/datamover/DatamoverConstants.java
diff --git a/common/source/java/ch/systemsx/cisd/common/filesystem/IPathCopier.java b/common/source/java/ch/systemsx/cisd/common/filesystem/IPathCopier.java
index f08adda72e4..b4c68f7a9fe 100644
--- a/common/source/java/ch/systemsx/cisd/common/filesystem/IPathCopier.java
+++ b/common/source/java/ch/systemsx/cisd/common/filesystem/IPathCopier.java
@@ -59,10 +59,13 @@ public interface IPathCopier extends ITerminable, ISelfTestable
* null if it is local.
* @param rsyncModuleNameOrNull The name of the rsync module to use in the rsync protocol, or
* <code>null</code>, if the bulk transfer should be using an ssh tunnel.
+ * @param rsyncPasswordFileOrNull The name of the password file to use if an rsync server is
+ * used. May be <code>null</code> or the name of a non-existent file, in which case
+ * no password is used when accessing the rsync server.
* @return The status of the operation, {@link Status#OK} if everything went OK.
*/
Status copyToRemote(File sourcePath, File destinationDirectory, String destinationHostOrNull,
- String rsyncModuleNameOrNull);
+ String rsyncModuleNameOrNull, String rsyncPasswordFileOrNull);
/**
* Copies <var>sourcePath</var> on <var>sourceHost</var> to <var>destinationDir</var>.
@@ -75,16 +78,18 @@ public interface IPathCopier extends ITerminable, ISelfTestable
* will be overwritten.
* @param rsyncModuleNameOrNull The name of the rsync module to use in the rsync protocol, or
* <code>null</code>, if the bulk transfer should be using an ssh tunnel.
+ * @param rsyncPasswordFileOrNull The name of the password file to use if an rsync server is
+ * used. May be <code>null</code> or the name of a non-existent file, in which case
+ * no password is used when accessing the rsync server.
* @return The status of the operation, {@link Status#OK} if everything went OK.
*/
Status copyFromRemote(File sourcePath, String sourceHost, File destinationDirectory,
- String rsyncModuleNameOrNull);
-
-
+ String rsyncModuleNameOrNull, String rsyncPasswordFileOrNull);
+
/**
* Try to connect to this server now and see whether we can list the module.
*
- * @return <code>true</code> if the connection was successfull and <code>false</code> otherwise.
+ * @return <code>true</code> if the connection was successful and <code>false</code> otherwise.
*/
- boolean checkRsyncConnection(String host, String rsyncModule);
+ boolean checkRsyncConnection(String host, String rsyncModule, String rsyncPassworFileOrNull);
}
diff --git a/common/source/java/ch/systemsx/cisd/common/filesystem/rsync/RsyncCopier.java b/common/source/java/ch/systemsx/cisd/common/filesystem/rsync/RsyncCopier.java
index a84a103b999..546bced00b4 100644
--- a/common/source/java/ch/systemsx/cisd/common/filesystem/rsync/RsyncCopier.java
+++ b/common/source/java/ch/systemsx/cisd/common/filesystem/rsync/RsyncCopier.java
@@ -146,19 +146,23 @@ public final class RsyncCopier implements IPathCopier, IDirectoryImmutableCopier
public final Status copy(final File sourcePath, final File destinationDirectory)
{
- return copy(sourcePath, null, destinationDirectory, null, null);
+ return copy(sourcePath, null, destinationDirectory, null, null, null);
}
public final Status copyFromRemote(final File sourcePath, final String sourceHost,
- final File destinationDirectory, String rsyncModuleNameOrNull)
+ final File destinationDirectory, String rsyncModuleNameOrNull,
+ String rsyncPasswordFileOrNull)
{
- return copy(sourcePath, sourceHost, destinationDirectory, null, rsyncModuleNameOrNull);
+ return copy(sourcePath, sourceHost, destinationDirectory, null, rsyncModuleNameOrNull,
+ rsyncPasswordFileOrNull);
}
public final Status copyToRemote(final File sourcePath, final File destinationDirectory,
- final String destinationHost, String rsyncModuleNameOrNull)
+ final String destinationHost, String rsyncModuleNameOrNull,
+ String rsyncPasswordFileOrNull)
{
- return copy(sourcePath, null, destinationDirectory, destinationHost, rsyncModuleNameOrNull);
+ return copy(sourcePath, null, destinationDirectory, destinationHost, rsyncModuleNameOrNull,
+ rsyncPasswordFileOrNull);
}
//
@@ -278,10 +282,16 @@ public final class RsyncCopier implements IPathCopier, IDirectoryImmutableCopier
return false;
}
- public boolean checkRsyncConnection(String host, String rsyncModule)
+ public boolean checkRsyncConnection(String host, String rsyncModule,
+ String rsyncPasswordFileOrNull)
{
final List<String> commandLineList = new ArrayList<String>();
commandLineList.add(rsyncExecutable);
+ if (rsyncPasswordFileOrNull != null && new File(rsyncPasswordFileOrNull).exists())
+ {
+ commandLineList.add("--password-file");
+ commandLineList.add(rsyncPasswordFileOrNull);
+ }
commandLineList.add(buildPath(host, new File("/"), rsyncModule, false));
final ProcessResult processResult = runCommand(commandLineList);
return processResult.isOK();
@@ -289,7 +299,7 @@ public final class RsyncCopier implements IPathCopier, IDirectoryImmutableCopier
private final Status copy(final File sourcePath, final String sourceHostOrNull,
final File destinationDirectory, final String destinationHostOrNull,
- final String rsyncModuleNameOrNull)
+ final String rsyncModuleNameOrNull, final String rsyncPasswordFileOrNull)
{
assert sourcePath != null;
assert sourceHostOrNull != null || sourcePath.exists() : logNonExistent(sourcePath);
@@ -299,7 +309,7 @@ public final class RsyncCopier implements IPathCopier, IDirectoryImmutableCopier
assert sourceHostOrNull == null || destinationHostOrNull == null;
final List<String> commandLine =
createCommandLineForMutableCopy(sourcePath, sourceHostOrNull, destinationDirectory,
- destinationHostOrNull, rsyncModuleNameOrNull);
+ destinationHostOrNull, rsyncModuleNameOrNull, rsyncPasswordFileOrNull);
return createStatus(runCommand(commandLine));
}
@@ -317,7 +327,7 @@ public final class RsyncCopier implements IPathCopier, IDirectoryImmutableCopier
@Private
final List<String> createCommandLineForMutableCopy(final File sourcePath,
final String sourceHost, final File destinationDirectory, final String destinationHost,
- final String rsyncModuleNameOrNull)
+ final String rsyncModuleNameOrNull, final String rsyncPasswordFileOrNull)
{
assert sourcePath != null && (sourceHost != null || sourcePath.exists());
assert destinationDirectory != null
@@ -342,6 +352,12 @@ public final class RsyncCopier implements IPathCopier, IDirectoryImmutableCopier
commandLineList.add("--rsh");
commandLineList.add(getSshExecutableArgument(sshExecutable));
}
+ if (rsyncModuleNameOrNull != null && rsyncPasswordFileOrNull != null
+ && new File(rsyncPasswordFileOrNull).exists())
+ {
+ commandLineList.add("--password-file");
+ commandLineList.add(rsyncPasswordFileOrNull);
+ }
if (additionalCmdLineFlags != null)
{
commandLineList.addAll(additionalCmdLineFlags);
diff --git a/common/sourceTest/java/ch/systemsx/cisd/common/filesystem/rsync/RsyncCopierTest.java b/common/sourceTest/java/ch/systemsx/cisd/common/filesystem/rsync/RsyncCopierTest.java
index ef9c23be29a..5ce6c9f5a07 100644
--- a/common/sourceTest/java/ch/systemsx/cisd/common/filesystem/rsync/RsyncCopierTest.java
+++ b/common/sourceTest/java/ch/systemsx/cisd/common/filesystem/rsync/RsyncCopierTest.java
@@ -26,6 +26,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import org.apache.commons.io.FileUtils;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@@ -112,18 +113,16 @@ public final class RsyncCopierTest
}
@Test
- public void testCommandLineForMutableCopyLocal() throws IOException,
- InterruptedException
+ public void testCommandLineForMutableCopyLocal() throws IOException, InterruptedException
{
final File rsyncBinary = createRsync(0);
final RsyncCopier copier = new RsyncCopier(rsyncBinary, rsyncBinary, false, false);
final List<String> cmdLine =
copier.createCommandLineForMutableCopy(sourceDirectory, null, destinationDirectory,
- null, null);
+ null, null, null);
assertEquals(rsyncBinary.getAbsolutePath(), cmdLine.get(0));
assertEquals(sourceDirectory.getAbsolutePath(), cmdLine.get(cmdLine.size() - 2));
- assertEquals(destinationDirectory.getAbsolutePath() + "/", cmdLine.get(cmdLine
- .size() - 1));
+ assertEquals(destinationDirectory.getAbsolutePath() + "/", cmdLine.get(cmdLine.size() - 1));
}
@Test
@@ -135,7 +134,8 @@ public final class RsyncCopierTest
final String host = "hst";
final RsyncCopier copier = new RsyncCopier(rsyncBinary, rsyncBinary, false, false);
final List<String> cmdLine =
- copier.createCommandLineForMutableCopy(sourceDirectory, null, dstPath, host, null);
+ copier.createCommandLineForMutableCopy(sourceDirectory, null, dstPath, host, null,
+ null);
assertEquals(rsyncBinary.getAbsolutePath(), cmdLine.get(0));
assertEquals(sourceDirectory.getAbsolutePath(), cmdLine.get(cmdLine.size() - 2));
assertEquals(host + ":dst/", cmdLine.get(cmdLine.size() - 1));
@@ -152,8 +152,52 @@ public final class RsyncCopierTest
final RsyncCopier copier = new RsyncCopier(rsyncBinary, rsyncBinary, false, false);
final List<String> cmdLine =
copier.createCommandLineForMutableCopy(sourceDirectory, null, dstPath, host,
- rsyncModule);
+ rsyncModule, null);
+ assertEquals(rsyncBinary.getAbsolutePath(), cmdLine.get(0));
+ assertEquals(sourceDirectory.getAbsolutePath(), cmdLine.get(cmdLine.size() - 2));
+ assertEquals(host + "::" + rsyncModule + "/", cmdLine.get(cmdLine.size() - 1));
+ }
+
+ @Test
+ public void testCommandLineForMutableCopyToRemoteRsyncModuleWithPwFile() throws IOException,
+ InterruptedException
+ {
+ final File rsyncBinary = createRsync(0);
+ final File dstPath = new File("dst");
+ final String host = "hst";
+ final String rsyncModule = "rsmod";
+ final File pwFile = new File(workingDirectory, "rsync.pwd");
+ FileUtils.touch(pwFile);
+ pwFile.deleteOnExit();
+ final RsyncCopier copier = new RsyncCopier(rsyncBinary, rsyncBinary, false, false);
+ final List<String> cmdLine =
+ copier.createCommandLineForMutableCopy(sourceDirectory, null, dstPath, host,
+ rsyncModule, pwFile.getPath());
+ assertEquals(rsyncBinary.getAbsolutePath(), cmdLine.get(0));
+ assertEquals("--password-file", cmdLine.get(cmdLine.size() - 4));
+ assertEquals(workingDirectory + "/rsync.pwd", cmdLine.get(cmdLine.size() - 3));
+ assertEquals(sourceDirectory.getAbsolutePath(), cmdLine.get(cmdLine.size() - 2));
+ assertEquals(host + "::" + rsyncModule + "/", cmdLine.get(cmdLine.size() - 1));
+ }
+
+ @Test
+ public void testCommandLineForMutableCopyToRemoteRsyncModuleWithNonExistentPwFile()
+ throws IOException, InterruptedException
+ {
+ final File rsyncBinary = createRsync(0);
+ final File dstPath = new File("dst");
+ final String host = "hst";
+ final String rsyncModule = "rsmod";
+ final File pwFile = new File(workingDirectory, "rsync.pwd");
+ pwFile.delete();
+ assertFalse(pwFile.exists());
+ final RsyncCopier copier = new RsyncCopier(rsyncBinary, rsyncBinary, false, false);
+ final List<String> cmdLine =
+ copier.createCommandLineForMutableCopy(sourceDirectory, null, dstPath, host,
+ rsyncModule, pwFile.getPath());
assertEquals(rsyncBinary.getAbsolutePath(), cmdLine.get(0));
+ assertFalse("--password-file".equals(cmdLine.get(cmdLine.size() - 4)));
+ assertFalse((workingDirectory + "/rsync.pwd").equals(cmdLine.get(cmdLine.size() - 3)));
assertEquals(sourceDirectory.getAbsolutePath(), cmdLine.get(cmdLine.size() - 2));
assertEquals(host + "::" + rsyncModule + "/", cmdLine.get(cmdLine.size() - 1));
}
@@ -167,11 +211,10 @@ public final class RsyncCopierTest
final RsyncCopier copier = new RsyncCopier(rsyncBinary, rsyncBinary, false, false);
final List<String> cmdLine =
copier.createCommandLineForMutableCopy(sourceDirectory, host, destinationDirectory,
- null, null);
+ null, null, null);
assertEquals(rsyncBinary.getAbsolutePath(), cmdLine.get(0));
assertEquals(host + ":" + sourceDirectory.getPath(), cmdLine.get(cmdLine.size() - 2));
- assertEquals(destinationDirectory.getAbsolutePath() + "/", cmdLine.get(cmdLine
- .size() - 1));
+ assertEquals(destinationDirectory.getAbsolutePath() + "/", cmdLine.get(cmdLine.size() - 1));
}
@Test
@@ -181,14 +224,18 @@ public final class RsyncCopierTest
final File rsyncBinary = createRsync(0);
final String host = "hst";
final String rsyncModule = "rsmod";
+ final File pwFile = new File(workingDirectory, "rsync.pwd");
+ FileUtils.touch(pwFile);
+ pwFile.deleteOnExit();
final RsyncCopier copier = new RsyncCopier(rsyncBinary, rsyncBinary, false, false);
final List<String> cmdLine =
copier.createCommandLineForMutableCopy(sourceDirectory, host, destinationDirectory,
- null, rsyncModule);
+ null, rsyncModule, pwFile.getPath());
assertEquals(rsyncBinary.getAbsolutePath(), cmdLine.get(0));
+ assertEquals("--password-file", cmdLine.get(cmdLine.size() - 4));
+ assertEquals(workingDirectory + "/rsync.pwd", cmdLine.get(cmdLine.size() - 3));
assertEquals(host + "::" + rsyncModule, cmdLine.get(cmdLine.size() - 2));
- assertEquals(destinationDirectory.getAbsolutePath() + "/", cmdLine.get(cmdLine
- .size() - 1));
+ assertEquals(destinationDirectory.getAbsolutePath() + "/", cmdLine.get(cmdLine.size() - 1));
}
@Test(groups =
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/DatamoverConstants.java b/datamover/source/java/ch/systemsx/cisd/datamover/DatamoverConstants.java
new file mode 100644
index 00000000000..9b3d917b508
--- /dev/null
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/DatamoverConstants.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.datamover;
+
+/**
+ * Global constants used in datamover.
+ *
+ * @author Bernd Rinn
+ */
+public class DatamoverConstants
+{
+ static final String SERVICE_PROPERTIES_FILE = "etc/service.properties";
+
+ public static final String RSYNC_PASSWORD_FILE_INCOMING = "etc/rsync_incoming.passwd";
+
+ public static final String RSYNC_PASSWORD_FILE_OUTGOING = "etc/rsync_outgoing.passwd";
+}
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/Parameters.java b/datamover/source/java/ch/systemsx/cisd/datamover/Parameters.java
index 4cf1354d98a..ff60424def5 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/Parameters.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/Parameters.java
@@ -552,7 +552,7 @@ public final class Parameters implements ITimingParameters, IFileSysParameters
final Properties properties = new Properties();
try
{
- final InputStream is = new FileInputStream(PropertyNames.SERVICE_PROPERTIES_FILE);
+ final InputStream is = new FileInputStream(DatamoverConstants.SERVICE_PROPERTIES_FILE);
try
{
properties.load(is);
@@ -565,7 +565,7 @@ public final class Parameters implements ITimingParameters, IFileSysParameters
{
final String msg =
"Could not load the service properties from resource '"
- + PropertyNames.SERVICE_PROPERTIES_FILE + "'.";
+ + DatamoverConstants.SERVICE_PROPERTIES_FILE + "'.";
operationLog.warn(msg, ex);
throw new ConfigurationFailureException(msg, ex);
}
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/PropertyNames.java b/datamover/source/java/ch/systemsx/cisd/datamover/PropertyNames.java
index caf45e071fa..cfffb54a59e 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/PropertyNames.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/PropertyNames.java
@@ -77,8 +77,6 @@ public final class PropertyNames
static final String USE_RSYNC_FOR_EXTRA_COPIES = "use-rsync-for-extra-copies";
- static final String SERVICE_PROPERTIES_FILE = "etc/service.properties";
-
static final String SSH_EXECUTABLE = "ssh-executable";
static final String TREAT_INCOMING_AS_REMOTE = "treat-incoming-as-remote";
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/AbstractFileStore.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/AbstractFileStore.java
index dc3f416fee4..96f8b5f48cd 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/AbstractFileStore.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/intf/AbstractFileStore.java
@@ -29,6 +29,7 @@ import ch.systemsx.cisd.common.filesystem.IPathCopier;
import ch.systemsx.cisd.common.highwatermark.HostAwareFileWithHighwaterMark;
import ch.systemsx.cisd.common.utilities.FileUtilities;
import ch.systemsx.cisd.common.utilities.StoreItem;
+import ch.systemsx.cisd.datamover.DatamoverConstants;
/**
* The abstract super-class of classes that represent a file store.
@@ -112,13 +113,15 @@ public abstract class AbstractFileStore implements IFileStore
} else
{
return copier.copyToRemote(srcItem, destPath, destHostOrNull,
- destinationStore.tryGetRsyncModuleName());
+ destinationStore.tryGetRsyncModuleName(),
+ DatamoverConstants.RSYNC_PASSWORD_FILE_OUTGOING);
}
} else
{
assert destHostOrNull == null;
return copier.copyFromRemote(srcItem, srcHostOrNull, destPath,
- tryGetRsyncModuleName());
+ tryGetRsyncModuleName(),
+ DatamoverConstants.RSYNC_PASSWORD_FILE_INCOMING);
}
}
@@ -131,22 +134,24 @@ public abstract class AbstractFileStore implements IFileStore
{
if (srcHostOrNull != null && tryGetRsyncModuleName() != null)
{
- check(srcHostOrNull, tryGetRsyncModuleName());
+ check(srcHostOrNull, tryGetRsyncModuleName(),
+ DatamoverConstants.RSYNC_PASSWORD_FILE_INCOMING);
}
if (destHostOrNull != null && destinationStore.tryGetRsyncModuleName() != null)
{
- check(destHostOrNull, destinationStore.tryGetRsyncModuleName());
+ check(destHostOrNull, destinationStore.tryGetRsyncModuleName(),
+ DatamoverConstants.RSYNC_PASSWORD_FILE_OUTGOING);
}
}
- private void check(String host, String rsyncModule)
+ private void check(String host, String rsyncModule, String rsyncPasswordFileOrNull)
{
- final boolean connectionOK = copier.checkRsyncConnection(host, rsyncModule);
+ final boolean connectionOK =
+ copier.checkRsyncConnection(host, rsyncModule, rsyncPasswordFileOrNull);
if (connectionOK == false)
{
throw new ConfigurationFailureException(String.format(
- "Connection to rsync module %s::%s failed", srcHostOrNull,
- tryGetRsyncModuleName()));
+ "Connection to rsync module %s::%s failed", host, rsyncModule));
}
}
diff --git a/datamover/sourceTest/java/ch/systemsx/cisd/datamover/SelfTestTest.java b/datamover/sourceTest/java/ch/systemsx/cisd/datamover/SelfTestTest.java
index 26b6c6e56d7..7faee105f35 100644
--- a/datamover/sourceTest/java/ch/systemsx/cisd/datamover/SelfTestTest.java
+++ b/datamover/sourceTest/java/ch/systemsx/cisd/datamover/SelfTestTest.java
@@ -105,13 +105,15 @@ public class SelfTestTest
}
public Status copyFromRemote(File sourcePath, String sourceHost,
- File destinationDirectory, String rsyncModuleNameOrNull)
+ File destinationDirectory, String rsyncModuleNameOrNull,
+ String rsyncPasswordFileOrNull)
{
throw new AssertionError();
}
public Status copyToRemote(File sourcePath, File destinationDirectory,
- String destinationHostOrNull, String rsyncModuleNameOrNull)
+ String destinationHostOrNull, String rsyncModuleNameOrNull,
+ String rsyncPasswordFileOrNull)
{
throw new AssertionError();
}
@@ -131,7 +133,8 @@ public class SelfTestTest
return false;
}
- public boolean checkRsyncConnection(String host, String rsyncModule)
+ public boolean checkRsyncConnection(String host, String rsyncModule,
+ String rsyncPassworFileOrNull)
{
throw new AssertionError();
}
--
GitLab