From 8d6df43b7f77ce69b09e834c113c5bf9f24c718a Mon Sep 17 00:00:00 2001
From: juanf <juanf>
Date: Mon, 13 Mar 2017 15:17:56 +0000
Subject: [PATCH] SSDM-4843 : Configurable rsync on the DSS
SVN: 37887
---
.../openbis/dss/generic/DataStoreServer.java | 5 +++
.../server/plugins/standard/Copier.java | 3 +-
.../plugins/standard/IPathCopierFactory.java | 3 +-
.../standard/RsyncArchiveCopierFactory.java | 12 +++++--
.../plugins/standard/RsyncCopierFactory.java | 5 +--
.../archiver/ArchiveDestinationFactory.java | 7 ++--
.../utils/DssPropertyParametersUtil.java | 16 +++++++++
.../dss/generic/shared/utils/RSyncConfig.java | 35 +++++++++++++++++++
.../standard/DataSetCopierForUsersTest.java | 3 +-
.../plugins/standard/DataSetCopierTest.java | 12 ++++---
.../DataSetFileOperationsManagerTest.java | 14 +++++---
11 files changed, 95 insertions(+), 20 deletions(-)
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/RSyncConfig.java
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/DataStoreServer.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/DataStoreServer.java
index a41d38051d4..6d6b597251f 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/DataStoreServer.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/DataStoreServer.java
@@ -18,6 +18,7 @@ package ch.systemsx.cisd.openbis.dss.generic;
import java.io.File;
import java.lang.Thread.UncaughtExceptionHandler;
+import java.util.List;
import org.apache.log4j.Logger;
@@ -32,6 +33,7 @@ import ch.systemsx.cisd.openbis.dss.BuildAndEnvironmentInfo;
import ch.systemsx.cisd.openbis.dss.generic.server.CommandQueueLister;
import ch.systemsx.cisd.openbis.dss.generic.shared.QueueingDataSetStatusUpdaterService;
import ch.systemsx.cisd.openbis.dss.generic.shared.utils.DssPropertyParametersUtil;
+import ch.systemsx.cisd.openbis.dss.generic.shared.utils.RSyncConfig;
/**
* Main class starting {@link ch.systemsx.cisd.openbis.dss.generic.server.DataStoreServer}, {@link ETLDaemon}.
@@ -102,6 +104,9 @@ public class DataStoreServer
ExtendedProperties props = DssPropertyParametersUtil.loadProperties(DssPropertyParametersUtil.SERVICE_PROPERTIES_FILE);
File storeRootDir = DssPropertyParametersUtil.getStoreRootDir(props);
+ List<String> rsyncOps = DssPropertyParametersUtil.getRsyncOptions(props);
+ notificationLog.info("Rsync configured with additional options: " + rsyncOps);
+ RSyncConfig.getInstance(rsyncOps);
// Initialize the shredder and updater _before_ the DataSetCommandExecutor which uses them.
QueueingPathRemoverService.start(storeRootDir, ETLDaemon.shredderQueueFile);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/Copier.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/Copier.java
index f6bc517408e..1744977bc9e 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/Copier.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/Copier.java
@@ -39,6 +39,7 @@ import ch.systemsx.cisd.common.process.ProcessResult;
import ch.systemsx.cisd.common.properties.PropertyUtils;
import ch.systemsx.cisd.openbis.dss.generic.shared.IPostRegistrationDatasetHandler;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
+import ch.systemsx.cisd.openbis.dss.generic.shared.utils.RSyncConfig;
public class Copier implements Serializable, IPostRegistrationDatasetHandler
{
@@ -112,7 +113,7 @@ public class Copier implements Serializable, IPostRegistrationDatasetHandler
String rsyncModule = hostAwareFile.tryGetRsyncModule();
IPathCopier copier =
pathCopierFactory.create(rsyncExecutable, sshExecutable,
- DataSetCopier.SSH_TIMEOUT_MILLIS);
+ DataSetCopier.SSH_TIMEOUT_MILLIS, RSyncConfig.getInstance().getAdditionalCommandLineOptions());
copier.check();
if (host != null)
{
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/IPathCopierFactory.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/IPathCopierFactory.java
index c85f54517c2..dc020f35b69 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/IPathCopierFactory.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/IPathCopierFactory.java
@@ -17,10 +17,11 @@
package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard;
import java.io.File;
+import java.util.List;
import ch.systemsx.cisd.common.filesystem.IPathCopier;
public interface IPathCopierFactory
{
- IPathCopier create(File rsyncExecutable, File sshExecutableOrNull, long timeoutInMillis);
+ IPathCopier create(File rsyncExecutable, File sshExecutableOrNull, long timeoutInMillis, List<String> additionalCmdLineFlagsOrNull);
}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiveCopierFactory.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiveCopierFactory.java
index 8132fe6f114..8a76f64c9d4 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiveCopierFactory.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiveCopierFactory.java
@@ -18,6 +18,8 @@ package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard;
import java.io.File;
import java.io.Serializable;
+import java.util.Arrays;
+import java.util.List;
import org.apache.commons.lang.time.DateUtils;
@@ -39,10 +41,14 @@ public final class RsyncArchiveCopierFactory implements Serializable, IPathCopie
private static final long serialVersionUID = 1L;
@Override
- public IPathCopier create(File rsyncExecutable, File sshExecutableOrNull, long timeoutInMillis)
+ public IPathCopier create(File rsyncExecutable, File sshExecutableOrNull, long timeoutInMillis, List<String> additionalCmdLineFlagsOrNull)
{
- return new RsyncCopier(rsyncExecutable, sshExecutableOrNull, "--archive", "--delete",
- "--inplace", "--checksum", getTimeoutParameter(timeoutInMillis));
+ List<String> additionalCmdLineFlags = Arrays.asList("--archive", "--delete", "--inplace", "--checksum", getTimeoutParameter(timeoutInMillis));
+ if (additionalCmdLineFlagsOrNull != null)
+ {
+ additionalCmdLineFlags.addAll(additionalCmdLineFlagsOrNull);
+ }
+ return new RsyncCopier(rsyncExecutable, sshExecutableOrNull, additionalCmdLineFlags.toArray(new String[0]));
}
private String getTimeoutParameter(long timeoutInMillis)
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncCopierFactory.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncCopierFactory.java
index 3e92d913e73..ce99c65af6e 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncCopierFactory.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncCopierFactory.java
@@ -18,6 +18,7 @@ package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard;
import java.io.File;
import java.io.Serializable;
+import java.util.List;
import ch.systemsx.cisd.common.filesystem.IPathCopier;
import ch.systemsx.cisd.common.filesystem.rsync.RsyncCopier;
@@ -27,8 +28,8 @@ public final class RsyncCopierFactory implements Serializable, IPathCopierFactor
private static final long serialVersionUID = 1L;
@Override
- public IPathCopier create(File rsyncExecutable, File sshExecutableOrNull, long timeoutInMillis)
+ public IPathCopier create(File rsyncExecutable, File sshExecutableOrNull, long timeoutInMillis, List<String> additionalCmdLineFlagsOrNull)
{
- return new RsyncCopier(rsyncExecutable, sshExecutableOrNull, false, false);
+ return new RsyncCopier(rsyncExecutable, sshExecutableOrNull, false, false, additionalCmdLineFlagsOrNull.toArray(new String[0]));
}
}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/ArchiveDestinationFactory.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/ArchiveDestinationFactory.java
index 66022748023..9a84404a7a1 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/ArchiveDestinationFactory.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/ArchiveDestinationFactory.java
@@ -32,6 +32,7 @@ import ch.systemsx.cisd.openbis.dss.generic.server.RemoteDataSetFileOperationsEx
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.Copier;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.IPathCopierFactory;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.ISshCommandExecutorFactory;
+import ch.systemsx.cisd.openbis.dss.generic.shared.utils.RSyncConfig;
/**
* @author Jakub Straszewski
@@ -90,7 +91,8 @@ public class ArchiveDestinationFactory implements Serializable
File gfindExecutable = Copier.getExecutable(properties, AbstractDataSetFileOperationsManager.GFIND_EXEC);
IPathCopier copier =
- pathCopierFactory.create(rsyncExecutable, sshExecutable, timeoutInMillis);
+ pathCopierFactory.create(rsyncExecutable, sshExecutable, timeoutInMillis,
+ RSyncConfig.getInstance().getAdditionalCommandLineOptions());
copier.check();
String rsyncModule = hostAwareFile.tryGetRsyncModule();
String rsyncPasswordFile = properties.getProperty(AbstractDataSetFileOperationsManager.RSYNC_PASSWORD_FILE_KEY);
@@ -110,7 +112,8 @@ public class ArchiveDestinationFactory implements Serializable
File sshExecutable = null; // don't use ssh locally
File rsyncExecutable = Copier.getExecutable(properties, AbstractDataSetFileOperationsManager.RSYNC_EXEC);
IPathCopier copier =
- pathCopierFactory.create(rsyncExecutable, sshExecutable, timeoutInMillis);
+ pathCopierFactory.create(rsyncExecutable, sshExecutable, timeoutInMillis,
+ RSyncConfig.getInstance().getAdditionalCommandLineOptions());
copier.check();
String rsyncModule = hostAwareFile.tryGetRsyncModule();
String rsyncPasswordFile = properties.getProperty(AbstractDataSetFileOperationsManager.RSYNC_PASSWORD_FILE_KEY);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/DssPropertyParametersUtil.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/DssPropertyParametersUtil.java
index 4fb76749ef8..f552867e2b4 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/DssPropertyParametersUtil.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/DssPropertyParametersUtil.java
@@ -17,6 +17,8 @@
package ch.systemsx.cisd.openbis.dss.generic.shared.utils;
import java.io.File;
+import java.util.Arrays;
+import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
@@ -56,6 +58,8 @@ public class DssPropertyParametersUtil
public static final String SERVER_URL_KEY = "server-url";
+ public static final String RSYNC_OPTIONS = "rsync-options";
+
public static final String DATA_STREAM_TIMEOUT = "data-stream-timeout";
public static final String DATA_STREAM_MAX_TIMEOUT = "data-stream-max-timeout";
@@ -157,6 +161,18 @@ public class DssPropertyParametersUtil
STOREROOT_DIR_KEY)));
}
+ public static List<String> getRsyncOptions(Properties serviceProperties)
+ {
+ String rsyncOptions = serviceProperties.getProperty(RSYNC_OPTIONS, null);
+ if (rsyncOptions == null)
+ {
+ return null;
+ } else
+ {
+ return Arrays.asList(rsyncOptions.split(" "));
+ }
+ }
+
public static String getOpenBisServerUrl(Properties serviceProperties)
{
return PropertyUtils.getMandatoryProperty(serviceProperties, SERVER_URL_KEY);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/RSyncConfig.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/RSyncConfig.java
new file mode 100644
index 00000000000..6957877d980
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/RSyncConfig.java
@@ -0,0 +1,35 @@
+package ch.systemsx.cisd.openbis.dss.generic.shared.utils;
+
+import java.util.List;
+
+public class RSyncConfig
+{
+ private final List<String> additionalCommandLineOptions;
+
+ private static RSyncConfig instance = new RSyncConfig();
+
+ private RSyncConfig()
+ {
+ this.additionalCommandLineOptions = null;
+ }
+
+ private RSyncConfig(List<String> additionalCommandLineOptions)
+ {
+ this.additionalCommandLineOptions = additionalCommandLineOptions;
+ }
+
+ public List<String> getAdditionalCommandLineOptions()
+ {
+ return additionalCommandLineOptions;
+ }
+
+ public synchronized static RSyncConfig getInstance(List<String> additionalCommandLineOptions)
+ {
+ return instance = new RSyncConfig(additionalCommandLineOptions);
+ }
+
+ public synchronized static RSyncConfig getInstance()
+ {
+ return instance;
+ }
+}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetCopierForUsersTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetCopierForUsersTest.java
index 3d173079d27..24c94ab1602 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetCopierForUsersTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetCopierForUsersTest.java
@@ -47,6 +47,7 @@ import ch.systemsx.cisd.common.test.RecordingMatcher;
import ch.systemsx.cisd.common.utilities.ITimeProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.DataSetProcessingContext;
import ch.systemsx.cisd.openbis.dss.generic.shared.ProcessingStatus;
+import ch.systemsx.cisd.openbis.dss.generic.shared.utils.RSyncConfig;
import ch.systemsx.cisd.openbis.generic.shared.Constants;
import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
import ch.systemsx.cisd.openbis.generic.shared.dto.builders.DatasetDescriptionBuilder;
@@ -219,7 +220,7 @@ public class DataSetCopierForUsersTest extends AbstractFileSystemTestCase
{
{
one(pathFactory).create(rsyncExecutableDummy, sshExecutableDummy,
- DataSetCopier.SSH_TIMEOUT_MILLIS);
+ DataSetCopier.SSH_TIMEOUT_MILLIS, RSyncConfig.getInstance().getAdditionalCommandLineOptions());
will(returnValue(copier));
one(sshFactory).create(sshExecutableDummy, null);
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetCopierTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetCopierTest.java
index 8d9d93865bb..f7d43cf5611 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetCopierTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetCopierTest.java
@@ -50,13 +50,13 @@ import ch.systemsx.cisd.common.test.RecordingMatcher;
import ch.systemsx.cisd.common.utilities.ITimeProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.DataSetProcessingContext;
import ch.systemsx.cisd.openbis.dss.generic.shared.ProcessingStatus;
+import ch.systemsx.cisd.openbis.dss.generic.shared.utils.RSyncConfig;
import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
/**
* @author Franz-Josef Elmer
*/
-@Friend(toClasses =
-{ DataSetCopier.class, AbstractDropboxProcessingPlugin.class })
+@Friend(toClasses = { DataSetCopier.class, AbstractDropboxProcessingPlugin.class })
public class DataSetCopierTest extends AbstractFileSystemTestCase
{
private static final String RSYNC_EXECUTABLE = "rsync-executable";
@@ -236,7 +236,8 @@ public class DataSetCopierTest extends AbstractFileSystemTestCase
{
assertEquals(
"Path to executable 'rsync' is not a file: "
- + rsyncExecutableDummy.getAbsolutePath(), ex.getMessage());
+ + rsyncExecutableDummy.getAbsolutePath(),
+ ex.getMessage());
}
context.assertIsSatisfied();
@@ -254,7 +255,8 @@ public class DataSetCopierTest extends AbstractFileSystemTestCase
{
assertEquals(
"Path to executable 'ssh' is not a file: "
- + sshExecutableDummy.getAbsolutePath(), ex.getMessage());
+ + sshExecutableDummy.getAbsolutePath(),
+ ex.getMessage());
}
context.assertIsSatisfied();
@@ -775,7 +777,7 @@ public class DataSetCopierTest extends AbstractFileSystemTestCase
{
{
exactly(numberOfExpectedCreations).of(pathFactory).create(rsyncExecutableDummy,
- sshExecutableDummy, DataSetCopier.SSH_TIMEOUT_MILLIS);
+ sshExecutableDummy, DataSetCopier.SSH_TIMEOUT_MILLIS, RSyncConfig.getInstance().getAdditionalCommandLineOptions());
will(returnValue(copier));
exactly(numberOfExpectedCreations).of(sshFactory).create(sshExecutableDummy,
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManagerTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManagerTest.java
index 1919cb09da8..1dc67bbf205 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManagerTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManagerTest.java
@@ -41,6 +41,7 @@ import ch.systemsx.cisd.common.filesystem.IPathCopier;
import ch.systemsx.cisd.common.filesystem.ssh.ISshCommandExecutor;
import ch.systemsx.cisd.common.logging.LogInitializer;
import ch.systemsx.cisd.common.process.ProcessResult;
+import ch.systemsx.cisd.openbis.dss.generic.shared.utils.RSyncConfig;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DatasetLocation;
import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
@@ -780,7 +781,8 @@ public class DataSetFileOperationsManagerTest extends AbstractFileSystemTestCase
one(sshExecutor).executeCommandRemotely(
gfindExec.getPath() + " " + ds1ArchivedLocationFile.getPath()
- + " -type f -printf \"%p\\t%s\\n\"", timeoutMillis());
+ + " -type f -printf \"%p\\t%s\\n\"",
+ timeoutMillis());
String filePath1 =
ds1ArchivedLocationFile.getPath() + File.separator
+ "original/data1_2.txt";
@@ -797,7 +799,8 @@ public class DataSetFileOperationsManagerTest extends AbstractFileSystemTestCase
will(returnValue(BooleanStatus.createTrue()));
one(sshExecutor).executeCommandRemotely(
gfindExec.getPath() + " " + ds2ArchivedLocationFile.getPath()
- + " -type f -printf \"%p\\t%s\\n\"", timeoutMillis());
+ + " -type f -printf \"%p\\t%s\\n\"",
+ timeoutMillis());
String filePath2 =
ds2ArchivedLocationFile.getPath() + File.separator
+ "original/data2.txt";
@@ -840,7 +843,8 @@ public class DataSetFileOperationsManagerTest extends AbstractFileSystemTestCase
one(sshExecutor).executeCommandRemotely(
gfindExec.getPath() + " " + ds2ArchivedLocationFile.getPath()
- + " -type f -printf \"%p\\t%s\\n\"", timeoutMillis());
+ + " -type f -printf \"%p\\t%s\\n\"",
+ timeoutMillis());
will(returnValue(ERROR_RESULT));
}
});
@@ -1056,7 +1060,7 @@ public class DataSetFileOperationsManagerTest extends AbstractFileSystemTestCase
context.checking(new Expectations()
{
{
- one(copierFactory).create(rsyncExec, null, DEFAULT_TIMEOUT_MILLIS);
+ one(copierFactory).create(rsyncExec, null, DEFAULT_TIMEOUT_MILLIS, RSyncConfig.getInstance().getAdditionalCommandLineOptions());
will(returnValue(copier));
one(copier).check();
@@ -1070,7 +1074,7 @@ public class DataSetFileOperationsManagerTest extends AbstractFileSystemTestCase
context.checking(new Expectations()
{
{
- one(copierFactory).create(rsyncExec, sshExec, timeoutMillis());
+ one(copierFactory).create(rsyncExec, sshExec, timeoutMillis(), RSyncConfig.getInstance().getAdditionalCommandLineOptions());
will(returnValue(copier));
one(sshFactory).create(sshExec, hostOrNull);
--
GitLab