[LMS-2180] use rsync when copying files locally; don't use --append when using...

[LMS-2180] use rsync when copying files locally; don't use --append when using rsync (failed to override files in destination that are bigger than source files)

SVN: 20651

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/IDataSetFileOperationsExecutor.java

@@ -35,6 +35,9 @@ public interface IDataSetFileOperationsExecutor
 
     void copyDataSetToDestination(File dataSet, File destination);
 
+    // uses rsync --delete for performance both locally and remotely
+    void syncDataSetWithDestination(File dataSet, File destination);
+
     void retrieveDataSetFromDestination(File dataSet, File destination);
 
     void renameTo(File newFile, File oldFile);

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/LocalDataSetFileOperationsExcecutor.java

@@ -32,6 +32,7 @@ import ch.systemsx.cisd.common.exceptions.Status;
 import ch.systemsx.cisd.common.filesystem.BooleanStatus;
 import ch.systemsx.cisd.common.filesystem.FileUtilities;
 import ch.systemsx.cisd.common.filesystem.IFileOperations;
+import ch.systemsx.cisd.common.filesystem.IPathCopier;
 import ch.systemsx.cisd.common.logging.LogCategory;
 import ch.systemsx.cisd.common.logging.LogFactory;
 
@@ -40,11 +41,21 @@ public final class LocalDataSetFileOperationsExcecutor implements IDataSetFileOp
     final static Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
             LocalDataSetFileOperationsExcecutor.class);
 
+    private final IPathCopier copier;
+
+    private final String rsyncModuleNameOrNull;
+
+    private final String rsyncPasswordFileOrNull;
+
     private final IFileOperations fileOperations;
 
-    public LocalDataSetFileOperationsExcecutor(IFileOperations fileOperations)
+    public LocalDataSetFileOperationsExcecutor(IFileOperations fileOperations, IPathCopier copier,
+            String rsyncModuleNameOrNull, String rsyncPasswordFileOrNull)
     {
         this.fileOperations = fileOperations;
+        this.copier = copier;
+        this.rsyncModuleNameOrNull = rsyncModuleNameOrNull;
+        this.rsyncPasswordFileOrNull = rsyncPasswordFileOrNull;
     }
 
     public BooleanStatus checkSame(File dataSet, File destination)
@@ -63,7 +74,6 @@ public final class LocalDataSetFileOperationsExcecutor implements IDataSetFileOp
             if (destination.isDirectory())
             {
                 FileFilter nullFilter = null;
-                // TODO ignore symlinks?
                 List<File> storeFiles = FileUtilities.listFiles(dataSet, nullFilter, true);
                 List<File> destFiles = FileUtilities.listFiles(destination, nullFilter, true);
 
@@ -154,6 +164,19 @@ public final class LocalDataSetFileOperationsExcecutor implements IDataSetFileOp
         }
     }
 
+    public void syncDataSetWithDestination(File dataSet, File destination)
+    {
+        // rsync --delete is more effective then deletion of destination directory & copy all
+        String host = null; // local
+        Status result =
+                copier.copyToRemote(dataSet, destination, host, rsyncModuleNameOrNull,
+                        rsyncPasswordFileOrNull);
+        if (result.isError())
+        {
+            throw new ExceptionWithStatus(result);
+        }
+    }
+
     public void retrieveDataSetFromDestination(File dataSet, File destination)
     {
         try

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/RemoteDataSetFileOperationsExecutor.java

@@ -128,6 +128,11 @@ public final class RemoteDataSetFileOperationsExecutor implements IDataSetFileOp
         }
     }
 
+    public void syncDataSetWithDestination(File dataSet, File destination)
+    {
+        copyDataSetToDestination(dataSet, destination);
+    }
+
     public void retrieveDataSetFromDestination(File dataSet, File destination)
     {
         Status result =

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManager.java

@@ -92,9 +92,16 @@ public class DataSetFileOperationsManager
 
         if (hostOrNull == null)
         {
+            File sshExecutable = null; // don't use ssh locally
+            File rsyncExecutable = Copier.getExecutable(properties, RSYNC_EXEC);
+            IPathCopier copier = pathCopierFactory.create(rsyncExecutable, sshExecutable);
+            copier.check();
+            String rsyncModule = hostAwareFile.tryGetRsyncModule();
+            String rsyncPasswordFile = properties.getProperty(RSYNC_PASSWORD_FILE_KEY);
             this.executor =
                     new LocalDataSetFileOperationsExcecutor(
-                            FileOperations.getMonitoredInstanceForCurrentThread());
+                            FileOperations.getMonitoredInstanceForCurrentThread(), copier,
+                            rsyncModule, rsyncPasswordFile);
         } else
         {
             File sshExecutable = Copier.getExecutable(properties, SSH_EXEC);
@@ -124,10 +131,18 @@ public class DataSetFileOperationsManager
         try
         {
             File destinationFolder = new File(destination, dataset.getDataSetLocation());
-            createFolderIfNotExists(destinationFolder.getParentFile());
-            operationLog.info("Copy dataset '" + dataset.getDatasetCode() + "' from '"
-                    + originalData.getPath() + "' to '" + destinationFolder.getParentFile());
-            executor.copyDataSetToDestination(originalData, destinationFolder.getParentFile());
+            if (createFolderIfNotExists(destinationFolder.getParentFile())
+                    || destinationExists(destinationFolder).isSuccess() == false)
+            {
+                operationLog.info("Copy dataset '" + dataset.getDatasetCode() + "' from '"
+                        + originalData.getPath() + "' to '" + destinationFolder.getParentFile());
+                executor.copyDataSetToDestination(originalData, destinationFolder.getParentFile());
+            } else
+            {
+                operationLog.info("Update dataset '" + dataset.getDatasetCode() + "' from '"
+                        + originalData.getPath() + "' to '" + destinationFolder.getParentFile());
+                executor.syncDataSetWithDestination(originalData, destinationFolder.getParentFile());
+            }
             return Status.OK;
         } catch (ExceptionWithStatus ex)
         {

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiveCopierFactory.java

+/*
+ * Copyright 2010 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard;
+
+import java.io.File;
+import java.io.Serializable;
+
+import ch.systemsx.cisd.common.filesystem.IPathCopier;
+import ch.systemsx.cisd.common.filesystem.rsync.RsyncCopier;
+
+/**
+ * {@link IPathCopierFactory} that is more reliable than {@link RsyncCopierFactory} when it comes to
+ * deciding which files to transfer. {@link IPathCopier} created by {@link RsyncCopierFactory} uses
+ * "--append" flag causing files that are bigger in destination than in source to be ignored.
+ * {@link IPathCopier} created by this factory is supposed to ignore only those files that have same
+ * sizes and modification times.
+ */
+public final class RsyncArchiveCopierFactory implements Serializable, IPathCopierFactory
+{
+    private static final long serialVersionUID = 1L;
+
+    public IPathCopier create(File rsyncExecutable, File sshExecutableOrNull)
+    {
+        // TODO 2011-04-05, Piotr Buczek: should we use --no-whole-file?
+        return new RsyncCopier(rsyncExecutable, sshExecutableOrNull, "--archive", "--delete",
+                "--inplace");
+    }
+}
\ No newline at end of file

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java

@@ -52,12 +52,12 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
 
     public RsyncArchiver(Properties properties, File storeRoot)
     {
-        this(properties, storeRoot, new RsyncCopierFactory(), new SshCommandExecutorFactory());
+        this(properties, storeRoot, new RsyncArchiveCopierFactory(),
+                new SshCommandExecutorFactory());
     }
 
     @Private
-    RsyncArchiver(Properties properties, File storeRoot,
-            IPathCopierFactory pathCopierFactory,
+    RsyncArchiver(Properties properties, File storeRoot, IPathCopierFactory pathCopierFactory,
             ISshCommandExecutorFactory sshCommandExecutorFactory)
     {
         super(properties, storeRoot, null, null);
@@ -131,7 +131,8 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
         if (fileOperationsManager == null)
         {
             this.fileOperationsManager =
-                    new DataSetFileOperationsManager(properties, pathCopierFactory, sshCommandExecutorFactory);
+                    new DataSetFileOperationsManager(properties, pathCopierFactory,
+                            sshCommandExecutorFactory);
         }
     }
 

datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManagerTest.java

@@ -249,6 +249,18 @@ public class DataSetFileOperationsManagerTest extends AbstractFileSystemTestCase
         /*
          * archive 2nd time (could happen on crash of DSS, but shouldn't hurt)
          */
+        context.checking(new Expectations()
+            {
+                {
+                    /*
+                     * use rsync
+                     */
+                    one(copier).copyToRemote(ds1Location,
+                            ds1ArchivedLocationFile.getParentFile().getAbsoluteFile(), null, null,
+                            null);
+                    will(returnValue(Status.OK));
+                }
+            });
         status = dataSetCopier.copyToDestination(ds1Location, ds1);
         assertSuccessful(status);
         // check that data set is now in archive
@@ -482,6 +494,9 @@ public class DataSetFileOperationsManagerTest extends AbstractFileSystemTestCase
     {
         final Properties properties = new Properties();
         properties.setProperty(DataSetFileOperationsManager.DESTINATION_KEY, destination.getPath());
+        properties.setProperty(DataSetFileOperationsManager.RSYNC_EXEC + "-executable",
+                rsyncExec.getPath());
+        prepareLocalCreateAndCheckCopier();
         return properties;
     }
 
@@ -521,6 +536,8 @@ public class DataSetFileOperationsManagerTest extends AbstractFileSystemTestCase
                     one(sshExecutor).exists(ds2ArchivedLocationFile.getParentFile().getPath(),
                             SSH_TIMEOUT_MILLIS);
                     will(returnValue(BooleanStatus.createTrue()));
+                    one(sshExecutor).exists(ds2ArchivedLocationFile.getPath(), SSH_TIMEOUT_MILLIS);
+                    will(returnValue(BooleanStatus.createTrue()));
 
                     one(copier).copyToRemote(ds2Location, ds2ArchivedLocationFile.getParentFile(),
                             HOST, null, null);
@@ -851,6 +868,19 @@ public class DataSetFileOperationsManagerTest extends AbstractFileSystemTestCase
     // context.assertIsSatisfied();
     // }
 
+    private void prepareLocalCreateAndCheckCopier()
+    {
+        context.checking(new Expectations()
+            {
+                {
+                    one(copierFactory).create(rsyncExec, null);
+                    will(returnValue(copier));
+
+                    one(copier).check();
+                }
+            });
+    }
+
     private void prepareRemoteCreateAndCheckCopier(final String hostOrNull,
             final String rsyncModuleOrNull, final boolean checkingResult)
     {

rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/dss/phosphonetx/server/plugins/LocalAndRemoteCopier.java

@@ -99,9 +99,17 @@ class LocalAndRemoteCopier implements Serializable, IPostRegistrationDatasetHand
         destination = hostAwareFile.getFile();
         if (hostOrNull == null)
         {
+            File sshExecutable = null; // don't use ssh locally
+            File rsyncExecutable = Copier.getExecutable(properties, DataSetCopier.RSYNC_EXEC);
+            IPathCopier copier = pathCopierFactory.create(rsyncExecutable, sshExecutable);
+            copier.check();
+            String rsyncModule = hostAwareFile.tryGetRsyncModule();
+            String rsyncPasswordFile =
+                    properties.getProperty(DataSetCopier.RSYNC_PASSWORD_FILE_KEY);
             executor =
                     new LocalDataSetFileOperationsExcecutor(
-                            FileOperations.getMonitoredInstanceForCurrentThread());
+                            FileOperations.getMonitoredInstanceForCurrentThread(), copier,
+                            rsyncModule, rsyncPasswordFile);
         } else
         {
             File sshExecutable = Copier.getExecutable(properties, DataSetCopier.SSH_EXEC);