From 287afffe80c098bc6ef7c1cc411b0e21e7b97421 Mon Sep 17 00:00:00 2001
From: buczekp <buczekp>
Date: Tue, 5 Apr 2011 09:27:19 +0000
Subject: [PATCH] [LMS-2180] use rsync when copying files locally; don't use
--append when using rsync (failed to override files in destination that are
bigger than source files)
SVN: 20651
---
.../IDataSetFileOperationsExecutor.java | 3 ++
.../LocalDataSetFileOperationsExcecutor.java | 27 +++++++++++-
.../RemoteDataSetFileOperationsExecutor.java | 5 +++
.../DataSetFileOperationsManager.java | 25 ++++++++---
.../standard/RsyncArchiveCopierFactory.java | 42 +++++++++++++++++++
.../plugins/standard/RsyncArchiver.java | 9 ++--
.../DataSetFileOperationsManagerTest.java | 30 +++++++++++++
.../server/plugins/LocalAndRemoteCopier.java | 10 ++++-
8 files changed, 139 insertions(+), 12 deletions(-)
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiveCopierFactory.java
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/IDataSetFileOperationsExecutor.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/IDataSetFileOperationsExecutor.java
index b206ffc123c..1a8a89b2060 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/IDataSetFileOperationsExecutor.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/IDataSetFileOperationsExecutor.java
@@ -35,6 +35,9 @@ public interface IDataSetFileOperationsExecutor
void copyDataSetToDestination(File dataSet, File destination);
+ // uses rsync --delete for performance both locally and remotely
+ void syncDataSetWithDestination(File dataSet, File destination);
+
void retrieveDataSetFromDestination(File dataSet, File destination);
void renameTo(File newFile, File oldFile);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/LocalDataSetFileOperationsExcecutor.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/LocalDataSetFileOperationsExcecutor.java
index e8b7082926f..223a13b93cd 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/LocalDataSetFileOperationsExcecutor.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/LocalDataSetFileOperationsExcecutor.java
@@ -32,6 +32,7 @@ import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.filesystem.BooleanStatus;
import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.common.filesystem.IFileOperations;
+import ch.systemsx.cisd.common.filesystem.IPathCopier;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
@@ -40,11 +41,21 @@ public final class LocalDataSetFileOperationsExcecutor implements IDataSetFileOp
final static Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
LocalDataSetFileOperationsExcecutor.class);
+ private final IPathCopier copier;
+
+ private final String rsyncModuleNameOrNull;
+
+ private final String rsyncPasswordFileOrNull;
+
private final IFileOperations fileOperations;
- public LocalDataSetFileOperationsExcecutor(IFileOperations fileOperations)
+ public LocalDataSetFileOperationsExcecutor(IFileOperations fileOperations, IPathCopier copier,
+ String rsyncModuleNameOrNull, String rsyncPasswordFileOrNull)
{
this.fileOperations = fileOperations;
+ this.copier = copier;
+ this.rsyncModuleNameOrNull = rsyncModuleNameOrNull;
+ this.rsyncPasswordFileOrNull = rsyncPasswordFileOrNull;
}
public BooleanStatus checkSame(File dataSet, File destination)
@@ -63,7 +74,6 @@ public final class LocalDataSetFileOperationsExcecutor implements IDataSetFileOp
if (destination.isDirectory())
{
FileFilter nullFilter = null;
- // TODO ignore symlinks?
List<File> storeFiles = FileUtilities.listFiles(dataSet, nullFilter, true);
List<File> destFiles = FileUtilities.listFiles(destination, nullFilter, true);
@@ -154,6 +164,19 @@ public final class LocalDataSetFileOperationsExcecutor implements IDataSetFileOp
}
}
+ public void syncDataSetWithDestination(File dataSet, File destination)
+ {
+ // rsync --delete is more effective then deletion of destination directory & copy all
+ String host = null; // local
+ Status result =
+ copier.copyToRemote(dataSet, destination, host, rsyncModuleNameOrNull,
+ rsyncPasswordFileOrNull);
+ if (result.isError())
+ {
+ throw new ExceptionWithStatus(result);
+ }
+ }
+
public void retrieveDataSetFromDestination(File dataSet, File destination)
{
try
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/RemoteDataSetFileOperationsExecutor.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/RemoteDataSetFileOperationsExecutor.java
index c3932d0b4f9..72180f09bce 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/RemoteDataSetFileOperationsExecutor.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/RemoteDataSetFileOperationsExecutor.java
@@ -128,6 +128,11 @@ public final class RemoteDataSetFileOperationsExecutor implements IDataSetFileOp
}
}
+ public void syncDataSetWithDestination(File dataSet, File destination)
+ {
+ copyDataSetToDestination(dataSet, destination);
+ }
+
public void retrieveDataSetFromDestination(File dataSet, File destination)
{
Status result =
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManager.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManager.java
index 8745915c786..770dc5cedc4 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManager.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManager.java
@@ -92,9 +92,16 @@ public class DataSetFileOperationsManager
if (hostOrNull == null)
{
+ File sshExecutable = null; // don't use ssh locally
+ File rsyncExecutable = Copier.getExecutable(properties, RSYNC_EXEC);
+ IPathCopier copier = pathCopierFactory.create(rsyncExecutable, sshExecutable);
+ copier.check();
+ String rsyncModule = hostAwareFile.tryGetRsyncModule();
+ String rsyncPasswordFile = properties.getProperty(RSYNC_PASSWORD_FILE_KEY);
this.executor =
new LocalDataSetFileOperationsExcecutor(
- FileOperations.getMonitoredInstanceForCurrentThread());
+ FileOperations.getMonitoredInstanceForCurrentThread(), copier,
+ rsyncModule, rsyncPasswordFile);
} else
{
File sshExecutable = Copier.getExecutable(properties, SSH_EXEC);
@@ -124,10 +131,18 @@ public class DataSetFileOperationsManager
try
{
File destinationFolder = new File(destination, dataset.getDataSetLocation());
- createFolderIfNotExists(destinationFolder.getParentFile());
- operationLog.info("Copy dataset '" + dataset.getDatasetCode() + "' from '"
- + originalData.getPath() + "' to '" + destinationFolder.getParentFile());
- executor.copyDataSetToDestination(originalData, destinationFolder.getParentFile());
+ if (createFolderIfNotExists(destinationFolder.getParentFile())
+ || destinationExists(destinationFolder).isSuccess() == false)
+ {
+ operationLog.info("Copy dataset '" + dataset.getDatasetCode() + "' from '"
+ + originalData.getPath() + "' to '" + destinationFolder.getParentFile());
+ executor.copyDataSetToDestination(originalData, destinationFolder.getParentFile());
+ } else
+ {
+ operationLog.info("Update dataset '" + dataset.getDatasetCode() + "' from '"
+ + originalData.getPath() + "' to '" + destinationFolder.getParentFile());
+ executor.syncDataSetWithDestination(originalData, destinationFolder.getParentFile());
+ }
return Status.OK;
} catch (ExceptionWithStatus ex)
{
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiveCopierFactory.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiveCopierFactory.java
new file mode 100644
index 00000000000..a0898ecf996
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiveCopierFactory.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2010 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard;
+
+import java.io.File;
+import java.io.Serializable;
+
+import ch.systemsx.cisd.common.filesystem.IPathCopier;
+import ch.systemsx.cisd.common.filesystem.rsync.RsyncCopier;
+
+/**
+ * {@link IPathCopierFactory} that is more reliable than {@link RsyncCopierFactory} when it comes to
+ * deciding which files to transfer. {@link IPathCopier} created by {@link RsyncCopierFactory} uses
+ * "--append" flag causing files that are bigger in destination than in source to be ignored.
+ * {@link IPathCopier} created by this factory is supposed to ignore only those files that have same
+ * sizes and modification times.
+ */
+public final class RsyncArchiveCopierFactory implements Serializable, IPathCopierFactory
+{
+ private static final long serialVersionUID = 1L;
+
+ public IPathCopier create(File rsyncExecutable, File sshExecutableOrNull)
+ {
+ // TODO 2011-04-05, Piotr Buczek: should we use --no-whole-file?
+ return new RsyncCopier(rsyncExecutable, sshExecutableOrNull, "--archive", "--delete",
+ "--inplace");
+ }
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java
index 770df7c92be..934f3470106 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java
@@ -52,12 +52,12 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
public RsyncArchiver(Properties properties, File storeRoot)
{
- this(properties, storeRoot, new RsyncCopierFactory(), new SshCommandExecutorFactory());
+ this(properties, storeRoot, new RsyncArchiveCopierFactory(),
+ new SshCommandExecutorFactory());
}
@Private
- RsyncArchiver(Properties properties, File storeRoot,
- IPathCopierFactory pathCopierFactory,
+ RsyncArchiver(Properties properties, File storeRoot, IPathCopierFactory pathCopierFactory,
ISshCommandExecutorFactory sshCommandExecutorFactory)
{
super(properties, storeRoot, null, null);
@@ -131,7 +131,8 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
if (fileOperationsManager == null)
{
this.fileOperationsManager =
- new DataSetFileOperationsManager(properties, pathCopierFactory, sshCommandExecutorFactory);
+ new DataSetFileOperationsManager(properties, pathCopierFactory,
+ sshCommandExecutorFactory);
}
}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManagerTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManagerTest.java
index b4cdae4e5d0..112a24a9665 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManagerTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManagerTest.java
@@ -249,6 +249,18 @@ public class DataSetFileOperationsManagerTest extends AbstractFileSystemTestCase
/*
* archive 2nd time (could happen on crash of DSS, but shouldn't hurt)
*/
+ context.checking(new Expectations()
+ {
+ {
+ /*
+ * use rsync
+ */
+ one(copier).copyToRemote(ds1Location,
+ ds1ArchivedLocationFile.getParentFile().getAbsoluteFile(), null, null,
+ null);
+ will(returnValue(Status.OK));
+ }
+ });
status = dataSetCopier.copyToDestination(ds1Location, ds1);
assertSuccessful(status);
// check that data set is now in archive
@@ -482,6 +494,9 @@ public class DataSetFileOperationsManagerTest extends AbstractFileSystemTestCase
{
final Properties properties = new Properties();
properties.setProperty(DataSetFileOperationsManager.DESTINATION_KEY, destination.getPath());
+ properties.setProperty(DataSetFileOperationsManager.RSYNC_EXEC + "-executable",
+ rsyncExec.getPath());
+ prepareLocalCreateAndCheckCopier();
return properties;
}
@@ -521,6 +536,8 @@ public class DataSetFileOperationsManagerTest extends AbstractFileSystemTestCase
one(sshExecutor).exists(ds2ArchivedLocationFile.getParentFile().getPath(),
SSH_TIMEOUT_MILLIS);
will(returnValue(BooleanStatus.createTrue()));
+ one(sshExecutor).exists(ds2ArchivedLocationFile.getPath(), SSH_TIMEOUT_MILLIS);
+ will(returnValue(BooleanStatus.createTrue()));
one(copier).copyToRemote(ds2Location, ds2ArchivedLocationFile.getParentFile(),
HOST, null, null);
@@ -851,6 +868,19 @@ public class DataSetFileOperationsManagerTest extends AbstractFileSystemTestCase
// context.assertIsSatisfied();
// }
+ private void prepareLocalCreateAndCheckCopier()
+ {
+ context.checking(new Expectations()
+ {
+ {
+ one(copierFactory).create(rsyncExec, null);
+ will(returnValue(copier));
+
+ one(copier).check();
+ }
+ });
+ }
+
private void prepareRemoteCreateAndCheckCopier(final String hostOrNull,
final String rsyncModuleOrNull, final boolean checkingResult)
{
diff --git a/rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/dss/phosphonetx/server/plugins/LocalAndRemoteCopier.java b/rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/dss/phosphonetx/server/plugins/LocalAndRemoteCopier.java
index bb1c99fecbc..4630408b2cf 100644
--- a/rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/dss/phosphonetx/server/plugins/LocalAndRemoteCopier.java
+++ b/rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/dss/phosphonetx/server/plugins/LocalAndRemoteCopier.java
@@ -99,9 +99,17 @@ class LocalAndRemoteCopier implements Serializable, IPostRegistrationDatasetHand
destination = hostAwareFile.getFile();
if (hostOrNull == null)
{
+ File sshExecutable = null; // don't use ssh locally
+ File rsyncExecutable = Copier.getExecutable(properties, DataSetCopier.RSYNC_EXEC);
+ IPathCopier copier = pathCopierFactory.create(rsyncExecutable, sshExecutable);
+ copier.check();
+ String rsyncModule = hostAwareFile.tryGetRsyncModule();
+ String rsyncPasswordFile =
+ properties.getProperty(DataSetCopier.RSYNC_PASSWORD_FILE_KEY);
executor =
new LocalDataSetFileOperationsExcecutor(
- FileOperations.getMonitoredInstanceForCurrentThread());
+ FileOperations.getMonitoredInstanceForCurrentThread(), copier,
+ rsyncModule, rsyncPasswordFile);
} else
{
File sshExecutable = Copier.getExecutable(properties, DataSetCopier.SSH_EXEC);
--
GitLab