[LMS-2106] tests for remote operations and errors

SVN: 20314

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncDataSetCopier.java

@@ -48,18 +48,20 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
 public class RsyncDataSetCopier // TODO rename to DataSetFileOperationsManager
 {
 
+    private final static Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
+            RsyncDataSetCopier.class);
+
     @Private
     static final String DESTINATION_KEY = "destination";
 
     @Private
     static final String RSYNC_PASSWORD_FILE_KEY = "rsync-password-file";
 
-    private static final String CHECK_EXISTENCE_FAILED = "couldn't check existence";
-
-    private static final String DESTINATION_DOES_NOT_EXIST = "destination doesn't exist";
+    @Private
+    static final String CHECK_EXISTENCE_FAILED = "couldn't check existence";
 
-    private final static Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
-            RsyncDataSetCopier.class);
+    @Private
+    static final String DESTINATION_DOES_NOT_EXIST = "destination doesn't exist";
 
     @Private
     static final String RSYNC_EXEC = "rsync";
@@ -67,7 +69,8 @@ public class RsyncDataSetCopier // TODO rename to DataSetFileOperationsManager
     @Private
     static final String SSH_EXEC = "ssh";
 
-    private static final long SSH_TIMEOUT_MILLIS = 15 * 1000; // 15s
+    @Private
+    static final long SSH_TIMEOUT_MILLIS = 15 * 1000; // 15s
 
     private final IDataSetFileOperationsExecutor executor;
 

datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncDataSetCopierTest.java

@@ -18,9 +18,12 @@ package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
 import java.util.Properties;
 
 import org.apache.commons.io.FileUtils;
+import org.jmock.Expectations;
 import org.jmock.Mockery;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeClass;
@@ -29,11 +32,14 @@ import org.testng.annotations.Test;
 
 import ch.rinn.restrictions.Friend;
 import ch.systemsx.cisd.base.tests.AbstractFileSystemTestCase;
+import ch.systemsx.cisd.common.concurrent.ExecutionResult;
 import ch.systemsx.cisd.common.exceptions.Status;
+import ch.systemsx.cisd.common.filesystem.BooleanStatus;
 import ch.systemsx.cisd.common.filesystem.FileUtilities;
 import ch.systemsx.cisd.common.filesystem.IPathCopier;
 import ch.systemsx.cisd.common.filesystem.ssh.ISshCommandExecutor;
 import ch.systemsx.cisd.common.logging.LogInitializer;
+import ch.systemsx.cisd.common.process.ProcessResult;
 import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
 
 /**
@@ -42,6 +48,16 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
 @Friend(toClasses = RsyncDataSetCopier.class)
 public class RsyncDataSetCopierTest extends AbstractFileSystemTestCase
 {
+    private static final String DUMMY_ERROR_MESSAGE = "dummy error message";
+
+    private static final ProcessResult OK_RESULT = new ProcessResult(Arrays.asList(""), 0, null,
+            ExecutionResult.create(null), null, 0, (List<String>) null, null, null, null);
+
+    private static final ProcessResult ERROR_RESULT = new ProcessResult(Arrays.asList(""), 0, null,
+            ExecutionResult.createExceptional(new Exception(DUMMY_ERROR_MESSAGE)), null, 0,
+            (List<String>) null, null, null, null);
+
+    private static final long SSH_TIMEOUT_MILLIS = RsyncDataSetCopier.SSH_TIMEOUT_MILLIS;
 
     private static final String LOCATION_1 = "l1";
 
@@ -53,7 +69,7 @@ public class RsyncDataSetCopierTest extends AbstractFileSystemTestCase
 
     private static final String DS2_LOCATION = LOCATION_1 + File.separator + DS2_CODE;
 
-    private static final String DS1_DATA_FILE = "data.txt";
+    private static final String DS1_DATA_FILE = "data1.txt";
 
     private static final String DS2_DATA_FILE = "data2.txt";
 
@@ -79,16 +95,22 @@ public class RsyncDataSetCopierTest extends AbstractFileSystemTestCase
 
     private File ds2Data;
 
+    private File ds1ArchivedLocationFile;
+
+    private File ds1ArchivedDataFile;
+
+    private File ds2ArchivedLocationFile;
+
+    private File ds2ArchivedDataFile;
+
     private Mockery context;
 
     private IPathCopierFactory copierFactory;
 
-    @SuppressWarnings("unused")
     private IPathCopier copier;
 
-    private ISshCommandExecutorFactory sshExecutorFactory;
+    private ISshCommandExecutorFactory sshFactory;
 
-    @SuppressWarnings("unused")
     private ISshCommandExecutor sshExecutor;
 
     private File destination;
@@ -109,14 +131,14 @@ public class RsyncDataSetCopierTest extends AbstractFileSystemTestCase
         context = new Mockery();
         copierFactory = context.mock(IPathCopierFactory.class);
         copier = context.mock(IPathCopier.class);
-        sshExecutorFactory = context.mock(ISshCommandExecutorFactory.class);
+        sshFactory = context.mock(ISshCommandExecutorFactory.class);
         sshExecutor = context.mock(ISshCommandExecutor.class);
 
         storeRoot = new File(workingDirectory, "store");
         storeRoot.mkdirs();
         File share = new File(storeRoot, SHARE_ID);
 
-        ds1 = createDataSetDescription(DS1_CODE, LOCATION_1 + File.separator + DS1_CODE, true);
+        ds1 = createDataSetDescription(DS1_CODE, DS1_LOCATION, true);
         ds1Location = new File(share, DS1_LOCATION);
         File ds1Folder = new File(ds1Location, ORIGINAL);
         ds1Folder.mkdirs();
@@ -132,13 +154,18 @@ public class RsyncDataSetCopierTest extends AbstractFileSystemTestCase
 
         destination = new File(workingDirectory, "destination");
         destination.mkdirs();
+
+        ds1ArchivedLocationFile = new File(destination, ds1.getDataSetLocation());
+        ds1ArchivedDataFile =
+                new File(ds1ArchivedLocationFile, ORIGINAL + File.separator + ds1Data.getName());
+        ds2ArchivedLocationFile = new File(destination, ds2.getDataSetLocation());
+        ds2ArchivedDataFile =
+                new File(ds2ArchivedLocationFile, ORIGINAL + File.separator + ds2Data.getName());
+
         rsyncExec = new File(workingDirectory, "my-rsync");
         rsyncExec.createNewFile();
         sshExec = new File(workingDirectory, "my-rssh");
         sshExec.createNewFile();
-
-        storeRoot = new File(workingDirectory, "store");
-        storeRoot.mkdirs();
     }
 
     private DatasetDescription createDataSetDescription(String dataSetCode, String location,
@@ -171,27 +198,28 @@ public class RsyncDataSetCopierTest extends AbstractFileSystemTestCase
         context.assertIsSatisfied();
     }
 
+    /*
+     * --< LOCAL >----------------------------------------------------------------------------------
+     */
+
     @Test
     public void testLocalCopyToDestination()
     {
         Properties properties = createLocalDestinationProperties();
         RsyncDataSetCopier dataSetCopier =
-                new RsyncDataSetCopier(properties, copierFactory, sshExecutorFactory);
+                new RsyncDataSetCopier(properties, copierFactory, sshFactory);
         prepareForCheckingLastModifiedDate();
 
-        File copiedDataSet = ds1ArchivedLocationFile();
-        File copiedData = ds1ArchivedDataFile();
-
         // check that data set is not yet in archive
-        assertEquals(false, copiedDataSet.exists());
+        assertDs1NotInArchive();
 
         /*
          * archive 1st time
          */
         Status status = dataSetCopier.copyToDestination(ds1Location, ds1);
-        assertEquals(Status.OK, status);
+        assertSuccessful(status);
         // check that data set is now in archive
-        assertDs1InArchive(copiedDataSet, copiedData);
+        assertDs1InArchive();
         // check that data set is still in store
         assertDs1InStore();
 
@@ -199,9 +227,9 @@ public class RsyncDataSetCopierTest extends AbstractFileSystemTestCase
          * archive 2nd time (could happen on crash of DSS, but shouldn't hurt)
          */
         status = dataSetCopier.copyToDestination(ds1Location, ds1);
-        assertEquals(Status.OK, status);
+        assertSuccessful(status);
         // check that data set is now in archive
-        assertDs1InArchive(copiedDataSet, copiedData);
+        assertDs1InArchive();
         // check that data set is still in store
         assertDs1InStore();
 
@@ -209,28 +237,25 @@ public class RsyncDataSetCopierTest extends AbstractFileSystemTestCase
     }
 
     @Test
-    public void testLocalCopyToNonExistantDestination()
+    public void testLocalCopyToNonExistentDestination()
     {
         Properties properties = createLocalDestinationProperties();
         RsyncDataSetCopier dataSetCopier =
-                new RsyncDataSetCopier(properties, copierFactory, sshExecutorFactory);
+                new RsyncDataSetCopier(properties, copierFactory, sshFactory);
         prepareForCheckingLastModifiedDate();
 
         destination.delete(); // if destination folder doesn't exist it will be created
 
-        File copiedDataSet = ds1ArchivedLocationFile();
-        File copiedData = ds1ArchivedDataFile();
-
         // check that data set is not yet in archive
-        assertEquals(false, copiedDataSet.exists());
+        assertDs1NotInArchive();
 
         /*
          * archive
          */
         Status status = dataSetCopier.copyToDestination(ds1Location, ds1);
-        assertEquals(Status.OK, status);
+        assertSuccessful(status);
         // check that data set is now in archive
-        assertDs1InArchive(copiedDataSet, copiedData);
+        assertDs1InArchive();
         // check that data set is still in store
         assertDs1InStore();
 
@@ -242,36 +267,30 @@ public class RsyncDataSetCopierTest extends AbstractFileSystemTestCase
     {
         Properties properties = createLocalDestinationProperties();
         RsyncDataSetCopier dataSetCopier =
-                new RsyncDataSetCopier(properties, copierFactory, sshExecutorFactory);
+                new RsyncDataSetCopier(properties, copierFactory, sshFactory);
         prepareForCheckingLastModifiedDate();
 
-        File copiedDataSet1 = ds1ArchivedLocationFile();
-        File copiedDataSet2 = ds2ArchivedLocationFile();
-        File copiedData1 = ds1ArchivedDataFile();
-        File copiedData2 = ds2ArchivedDataFile();
-
         // check that both data sets are not yet in archive
-        assertEquals(false, copiedDataSet1.exists());
-        assertEquals(false, copiedDataSet2.exists());
+        assertDs1NotInArchive();
+        assertDs2NotInArchive();
 
         /*
          * copy 1st data set
          */
         Status status1 = dataSetCopier.copyToDestination(ds1Location, ds1);
-
-        assertEquals(Status.OK, status1);
-        assertDs1InArchive(copiedDataSet1, copiedData1);
+        assertSuccessful(status1);
+        assertDs1InArchive();
         // check that 2nd data set is not yet in archive
-        assertEquals(false, copiedDataSet2.exists());
+        assertDs2NotInArchive();
 
         /*
          * copy 2nd data set
          */
         Status status2 = dataSetCopier.copyToDestination(ds2Location, ds2);
-        assertEquals(Status.OK, status2);
-        assertDs2InArchive(copiedDataSet2, copiedData2);
+        assertSuccessful(status2);
+        assertDs2InArchive();
         // check that 1st data set is still in archive
-        assertDs1InArchive(copiedDataSet1, copiedData1);
+        assertDs1InArchive();
 
         // both data sets should be in the store
         assertDs1InStore();
@@ -288,19 +307,16 @@ public class RsyncDataSetCopierTest extends AbstractFileSystemTestCase
          */
         Properties properties = createLocalDestinationProperties();
         RsyncDataSetCopier dataSetCopier =
-                new RsyncDataSetCopier(properties, copierFactory, sshExecutorFactory);
+                new RsyncDataSetCopier(properties, copierFactory, sshFactory);
         prepareForCheckingLastModifiedDate();
 
-        File copiedDataSet = ds1ArchivedLocationFile();
-        File copiedData = ds1ArchivedDataFile();
-
         // check that data set is not yet in archive
-        assertEquals(false, copiedDataSet.exists());
+        assertDs1NotInArchive();
 
         Status status = dataSetCopier.copyToDestination(ds1Location, ds1);
 
-        assertEquals(Status.OK, status);
-        assertDs1InArchive(copiedDataSet, copiedData);
+        assertSuccessful(status);
+        assertDs1InArchive();
 
         /*
          * delete from store
@@ -312,29 +328,62 @@ public class RsyncDataSetCopierTest extends AbstractFileSystemTestCase
         {
             fail(e.getMessage());
         }
-        assertEquals(false, ds1Data.exists());
+        assertFalse(ds1Data.exists());
 
         /*
          * retrieve from archive - 1st time
          */
         Status statusRetrieve = dataSetCopier.retrieveFromDestination(ds1Location, ds1);
-        assertEquals(Status.OK, statusRetrieve);
+        assertSuccessful(statusRetrieve);
         assertDs1InStore();
-        assertDs1InArchive(copiedDataSet, copiedData);
+        assertDs1InArchive();
         assertDs2InStore(); // ds2 shouldn't be affected at all
 
         /*
          * retrieve from archive - 2nd time (possible e.g. after crash)
          */
         statusRetrieve = dataSetCopier.retrieveFromDestination(ds1Location, ds1);
-        assertEquals(Status.OK, statusRetrieve);
+        assertSuccessful(statusRetrieve);
         assertDs1InStore();
-        assertDs1InArchive(copiedDataSet, copiedData);
+        assertDs1InArchive();
         assertDs2InStore(); // ds2 shouldn't be affected at all
 
         context.assertIsSatisfied();
     }
 
+    @Test(dependsOnMethods = "testLocalCopyToDestination")
+    public void testLocalPresentInDestination()
+    {
+        Properties properties = createLocalDestinationProperties();
+        RsyncDataSetCopier dataSetCopier =
+                new RsyncDataSetCopier(properties, copierFactory, sshFactory);
+        prepareForCheckingLastModifiedDate();
+
+        // check that data set is not yet in archive
+        assertDs1NotInArchive();
+
+        /*
+         * before copying - doesn't exist
+         */
+        BooleanStatus boolStatus = dataSetCopier.isPresentInDestination(ds1Location, ds1);
+        assertFalse(boolStatus);
+
+        /*
+         * copy to archive
+         */
+        Status status = dataSetCopier.copyToDestination(ds1Location, ds1);
+        assertSuccessful(status);
+        assertDs1InArchive();
+
+        /*
+         * after copying - exists
+         */
+        boolStatus = dataSetCopier.isPresentInDestination(ds1Location, ds1);
+        assertTrue(boolStatus);
+
+        context.assertIsSatisfied();
+    }
+
     @Test(dependsOnMethods = "testLocalCopyToDestination")
     public void testLocalDeleteFromDestination()
     {
@@ -343,95 +392,448 @@ public class RsyncDataSetCopierTest extends AbstractFileSystemTestCase
          */
         Properties properties = createLocalDestinationProperties();
         RsyncDataSetCopier dataSetCopier =
-                new RsyncDataSetCopier(properties, copierFactory, sshExecutorFactory);
+                new RsyncDataSetCopier(properties, copierFactory, sshFactory);
         prepareForCheckingLastModifiedDate();
 
-        File copiedDataSet = ds1ArchivedLocationFile();
-        File copiedData = ds1ArchivedDataFile();
-
         // check that data set is not yet in archive
-        assertEquals(false, copiedDataSet.exists());
+        assertDs1NotInArchive();
 
         Status status = dataSetCopier.copyToDestination(ds1Location, ds1);
 
-        assertEquals(Status.OK, status);
-        assertDs1InArchive(copiedDataSet, copiedData);
+        assertSuccessful(status);
+        assertDs1InArchive();
 
         /*
          * delete from archive
          */
         Status statusDelete = dataSetCopier.deleteFromDestination(ds1);
-        assertEquals(Status.OK, statusDelete);
-        assertEquals(false, copiedDataSet.exists());
+        assertSuccessful(statusDelete);
+        assertDs1NotInArchive();
         assertDs1InStore(); // we didn't delete it from store
 
         context.assertIsSatisfied();
     }
 
-    private File ds1ArchivedLocationFile()
+    private Properties createLocalDestinationProperties()
     {
-        return new File(destination, ds1.getDataSetLocation());
+        final Properties properties = new Properties();
+        properties.setProperty(RsyncDataSetCopier.DESTINATION_KEY, destination.getPath());
+        return properties;
     }
 
-    private File ds2ArchivedLocationFile()
+    /*
+     * --< REMOTE >---------------------------------------------------------------------------------
+     */
+
+    @Test
+    public void testRemoteViaSshCopyToDestination()
     {
-        return new File(destination, ds2.getDataSetLocation());
+        Properties properties = createRemoteViaSshDestinationProperties();
+        prepareRemoteCreateAndCheckCopier(HOST, null, true);
+        RsyncDataSetCopier dataSetCopier =
+                new RsyncDataSetCopier(properties, copierFactory, sshFactory);
+        context.checking(new Expectations()
+            {
+                {
+                    /*
+                     * ds1: directory exists in archive -> first delete from directory
+                     */
+                    one(sshExecutor).exists(ds1ArchivedLocationFile.getPath(), SSH_TIMEOUT_MILLIS);
+                    will(returnValue(BooleanStatus.createTrue()));
+
+                    one(sshExecutor).executeCommandRemotely(
+                            "rm -rf " + ds1ArchivedLocationFile.getPath(), SSH_TIMEOUT_MILLIS);
+                    will(returnValue(OK_RESULT));
+
+                    one(copier).copyToRemote(ds1Location, ds1ArchivedLocationFile.getParentFile(),
+                            HOST, null, null);
+                    will(returnValue(Status.OK));
+
+                    /*
+                     * ds2: directory doesn't exist in archive -> only copy
+                     */
+                    one(sshExecutor).exists(ds2ArchivedLocationFile.getPath(), SSH_TIMEOUT_MILLIS);
+                    will(returnValue(BooleanStatus.createFalse()));
+
+                    one(copier).copyToRemote(ds2Location, ds2ArchivedLocationFile.getParentFile(),
+                            HOST, null, null);
+                    will(returnValue(Status.OK));
+                }
+            });
+        Status status1 = dataSetCopier.copyToDestination(ds1Location, ds1);
+        Status status2 = dataSetCopier.copyToDestination(ds2Location, ds2);
+        assertSuccessful(status1);
+        assertSuccessful(status2);
+
+        context.assertIsSatisfied();
     }
 
-    private File ds1ArchivedDataFile()
+    @Test
+    public void testRemoteViaSshCopyToDestinationWithErrors()
     {
-        return new File(ds1ArchivedLocationFile(), ORIGINAL + File.separator + ds1Data.getName());
+        Properties properties = createRemoteViaSshDestinationProperties();
+        prepareRemoteCreateAndCheckCopier(HOST, null, true);
+        RsyncDataSetCopier dataSetCopier =
+                new RsyncDataSetCopier(properties, copierFactory, sshFactory);
+        context.checking(new Expectations()
+            {
+                {
+                    one(sshExecutor).exists(ds1ArchivedLocationFile.getPath(), SSH_TIMEOUT_MILLIS);
+                    will(returnValue(BooleanStatus.createFalse()));
+                    one(copier).copyToRemote(ds1Location, ds1ArchivedLocationFile.getParentFile(),
+                            HOST, null, null);
+                    will(returnValue(Status.createError(DUMMY_ERROR_MESSAGE)));
+                }
+            });
+        Status status = dataSetCopier.copyToDestination(ds1Location, ds1);
+        assertError(status, DUMMY_ERROR_MESSAGE);
+
+        context.assertIsSatisfied();
     }
 
-    private File ds2ArchivedDataFile()
+    @Test
+    public void testRemoteViaSshRetrieveFromDestination()
     {
-        return new File(ds2ArchivedLocationFile(), ORIGINAL + File.separator + ds2Data.getName());
+        Properties properties = createRemoteViaSshDestinationProperties();
+        prepareRemoteCreateAndCheckCopier(HOST, null, true);
+        RsyncDataSetCopier dataSetCopier =
+                new RsyncDataSetCopier(properties, copierFactory, sshFactory);
+        context.checking(new Expectations()
+            {
+                {
+                    one(sshExecutor).exists(ds1ArchivedLocationFile.getPath(), SSH_TIMEOUT_MILLIS);
+                    will(returnValue(BooleanStatus.createTrue()));
+                    one(copier).copyFromRemote(ds1ArchivedLocationFile, HOST,
+                            ds1Location.getParentFile(), null, null);
+                    will(returnValue(Status.OK));
+                }
+            });
+        Status status = dataSetCopier.retrieveFromDestination(ds1Location, ds1);
+        assertSuccessful(status);
+
+        context.assertIsSatisfied();
     }
 
-    private void assertDs1InStore()
+    public void testRemoteViaSshRetrieveFromDestinationWithErrors()
     {
-        assertEquals(true, ds1Data.exists());
-        assertEquals(DATA1, FileUtilities.loadToString(ds1Data).trim());
+        Properties properties = createRemoteViaSshDestinationProperties();
+        prepareRemoteCreateAndCheckCopier(HOST, null, true);
+        RsyncDataSetCopier dataSetCopier =
+                new RsyncDataSetCopier(properties, copierFactory, sshFactory);
+        context.checking(new Expectations()
+            {
+                {
+                    /*
+                     * ds1: destination doesn't exist
+                     */
+                    one(sshExecutor).exists(ds1ArchivedLocationFile.getPath(), SSH_TIMEOUT_MILLIS);
+                    will(returnValue(BooleanStatus.createFalse()));
+
+                    /*
+                     * ds2: copy failed
+                     */
+                    one(sshExecutor).exists(ds2ArchivedLocationFile.getPath(), SSH_TIMEOUT_MILLIS);
+                    will(returnValue(BooleanStatus.createTrue()));
+                    one(copier).copyFromRemote(ds2ArchivedLocationFile, HOST,
+                            ds2Location.getParentFile(), null, null);
+                    will(returnValue(Status.createError(DUMMY_ERROR_MESSAGE)));
+                }
+            });
+
+        Status status1 = dataSetCopier.retrieveFromDestination(ds1Location, ds1);
+        assertError(status1, RsyncDataSetCopier.DESTINATION_DOES_NOT_EXIST);
+
+        Status status2 = dataSetCopier.retrieveFromDestination(ds2Location, ds2);
+        assertError(status2, DUMMY_ERROR_MESSAGE);
+
+        context.assertIsSatisfied();
     }
 
-    private void assertDs2InStore()
+    @Test
+    public void testRemoteViaSshIsPresentInDestination()
     {
-        assertEquals(true, ds2Data.exists());
-        assertEquals(DATA2, FileUtilities.loadToString(ds2Data).trim());
+        Properties properties = createRemoteViaSshDestinationProperties();
+        prepareRemoteCreateAndCheckCopier(HOST, null, true);
+        RsyncDataSetCopier dataSetCopier =
+                new RsyncDataSetCopier(properties, copierFactory, sshFactory);
+        context.checking(new Expectations()
+            {
+                {
+                    /*
+                     * ds1: present
+                     */
+                    one(sshExecutor).exists(ds1ArchivedLocationFile.getPath(), SSH_TIMEOUT_MILLIS);
+                    will(returnValue(BooleanStatus.createTrue()));
+
+                    /*
+                     * ds2: not present
+                     */
+                    one(sshExecutor).exists(ds2ArchivedLocationFile.getPath(), SSH_TIMEOUT_MILLIS);
+                    will(returnValue(BooleanStatus.createFalse()));
+                }
+            });
+        BooleanStatus status1 = dataSetCopier.isPresentInDestination(ds1Location, ds1);
+        BooleanStatus status2 = dataSetCopier.isPresentInDestination(ds2Location, ds2);
+        assertTrue(status1);
+        assertFalse(status2);
+
+        context.assertIsSatisfied();
     }
 
-    private void assertDs1InArchive(File copiedDataSet, File copiedData)
+    @Test
+    public void testRemoteViaSshIsPresentInDestinationWithError()
     {
-        assertEquals(true, copiedDataSet.isDirectory());
-        assertEquals(ds1Data.lastModified(), copiedDataSet.lastModified());
-        assertEquals(DATA1, FileUtilities.loadToString(copiedData).trim());
+        Properties properties = createRemoteViaSshDestinationProperties();
+        prepareRemoteCreateAndCheckCopier(HOST, null, true);
+        RsyncDataSetCopier dataSetCopier =
+                new RsyncDataSetCopier(properties, copierFactory, sshFactory);
+        context.checking(new Expectations()
+            {
+                {
+                    one(sshExecutor).exists(ds1ArchivedLocationFile.getPath(), SSH_TIMEOUT_MILLIS);
+                    will(returnValue(BooleanStatus.createError(DUMMY_ERROR_MESSAGE)));
+                }
+            });
+        BooleanStatus status = dataSetCopier.isPresentInDestination(ds1Location, ds1);
+        assertError(status, DUMMY_ERROR_MESSAGE);
+
+        context.assertIsSatisfied();
     }
 
-    private void assertDs2InArchive(File copiedDataSet2, File copiedData2)
+    @Test
+    public void testRemoteViaSshDeleteFromDestination()
     {
-        assertEquals(true, copiedDataSet2.isDirectory());
-        assertEquals(ds2Data.lastModified(), copiedData2.lastModified());
-        assertEquals(DATA2, FileUtilities.loadToString(copiedData2).trim());
+        Properties properties = createRemoteViaSshDestinationProperties();
+        prepareRemoteCreateAndCheckCopier(HOST, null, true);
+        RsyncDataSetCopier dataSetCopier =
+                new RsyncDataSetCopier(properties, copierFactory, sshFactory);
+        context.checking(new Expectations()
+            {
+                {
+                    /*
+                     * ds1: directory exists in archive -> delete from directory
+                     */
+                    one(sshExecutor).exists(ds1ArchivedLocationFile.getPath(), SSH_TIMEOUT_MILLIS);
+                    will(returnValue(BooleanStatus.createTrue()));
+                    one(sshExecutor).executeCommandRemotely(
+                            "rm -rf " + ds1ArchivedLocationFile.getPath(), SSH_TIMEOUT_MILLIS);
+                    will(returnValue(OK_RESULT));
+
+                    /*
+                     * ds2: directory doesn't exist in archive -> nothing to do
+                     */
+                    one(sshExecutor).exists(ds2ArchivedLocationFile.getPath(), SSH_TIMEOUT_MILLIS);
+                    will(returnValue(BooleanStatus.createFalse()));
+                }
+            });
+        Status status1 = dataSetCopier.deleteFromDestination(ds1);
+        Status status2 = dataSetCopier.deleteFromDestination(ds2);
+        assertSuccessful(status1);
+        assertSuccessful(status2);
+
+        context.assertIsSatisfied();
     }
 
-    private Properties createLocalDestinationProperties()
+    @Test
+    public void testRemoteViaSshDeleteFromDestinationWithErrors()
+    {
+        Properties properties = createRemoteViaSshDestinationProperties();
+        prepareRemoteCreateAndCheckCopier(HOST, null, true);
+        RsyncDataSetCopier dataSetCopier =
+                new RsyncDataSetCopier(properties, copierFactory, sshFactory);
+        context.checking(new Expectations()
+            {
+                {
+                    /*
+                     * ds1: fail to delete
+                     */
+                    one(sshExecutor).exists(ds1ArchivedLocationFile.getPath(), SSH_TIMEOUT_MILLIS);
+                    will(returnValue(BooleanStatus.createTrue()));
+                    one(sshExecutor).executeCommandRemotely(
+                            "rm -rf " + ds1ArchivedLocationFile.getPath(), SSH_TIMEOUT_MILLIS);
+                    will(returnValue(ERROR_RESULT));
+
+                    /*
+                     * ds2: fail to check existence
+                     */
+                    one(sshExecutor).exists(ds2ArchivedLocationFile.getPath(), SSH_TIMEOUT_MILLIS);
+                    will(returnValue(BooleanStatus.createError(DUMMY_ERROR_MESSAGE)));
+                }
+            });
+        Status status1 = dataSetCopier.deleteFromDestination(ds1);
+        Status status2 = dataSetCopier.deleteFromDestination(ds2);
+        assertError(status1, "couldn't delete");
+        assertError(status2, "couldn't check existence");
+
+        context.assertIsSatisfied();
+    }
+
+    // TODO 2011-03-14, Piotr Buczek: test rsync operations
+
+    // @Test
+    // public void testRemoteViaRsyncCopyToDestination()
+    // {
+    // Properties properties = createRemoteViaRsyncDestinationProperties();
+    // RsyncDataSetCopier dataSetCopier =
+    // new RsyncDataSetCopier(properties, copierFactory, sshFactory);
+    // prepareForCheckingLastModifiedDate();
+    //
+    // File copiedDataSet = ds1ArchivedLocationFile();
+    // File copiedData = ds1ArchivedDataFile();
+    //
+    // // check that data set is not yet in archive
+    // assertEquals(false, copiedDataSet.exists());
+    //
+    // /*
+    // * archive 1st time
+    // */
+    // Status status = dataSetCopier.copyToDestination(ds1Location, ds1);
+    // assertEquals(Status.OK, status);
+    // // check that data set is now in archive
+    // assertDs1InArchive(copiedDataSet, copiedData);
+    // // check that data set is still in store
+    // assertDs1InStore();
+    //
+    // /*
+    // * archive 2nd time (could happen on crash of DSS, but shouldn't hurt)
+    // */
+    // status = dataSetCopier.copyToDestination(ds1Location, ds1);
+    // assertEquals(Status.OK, status);
+    // // check that data set is now in archive
+    // assertDs1InArchive(copiedDataSet, copiedData);
+    // // check that data set is still in store
+    // assertDs1InStore();
+    //
+    // context.assertIsSatisfied();
+    // }
+
+    private void prepareRemoteCreateAndCheckCopier(final String hostOrNull,
+            final String rsyncModuleOrNull, final boolean checkingResult)
+    {
+        context.checking(new Expectations()
+            {
+                {
+                    one(copierFactory).create(rsyncExec, sshExec);
+                    will(returnValue(copier));
+
+                    one(sshFactory).create(sshExec, hostOrNull);
+                    will(returnValue(sshExecutor));
+
+                    one(copier).check();
+                    if (hostOrNull != null)
+                    {
+                        if (rsyncModuleOrNull != null)
+                        {
+                            one(copier).checkRsyncConnectionViaRsyncServer(hostOrNull,
+                                    rsyncModuleOrNull, rsyncModuleOrNull + "-password",
+                                    SSH_TIMEOUT_MILLIS);
+                        } else
+                        {
+                            one(copier).checkRsyncConnectionViaSsh(hostOrNull, null,
+                                    SSH_TIMEOUT_MILLIS);
+                        }
+                        will(returnValue(checkingResult));
+                    }
+                }
+            });
+    }
+
+    private static String HOST = "localhost";
+
+    private static String RSYNC_MODULE = "abc";
+
+    private Properties createRemoteViaSshDestinationProperties()
     {
         final Properties properties = new Properties();
-        properties.setProperty(RsyncDataSetCopier.DESTINATION_KEY, destination.getPath());
+        properties.setProperty(RsyncDataSetCopier.DESTINATION_KEY,
+                HOST + ":" + destination.getPath());
+        properties.setProperty(RsyncDataSetCopier.RSYNC_EXEC + "-executable", rsyncExec.getPath());
+        properties.setProperty(RsyncDataSetCopier.SSH_EXEC + "-executable", sshExec.getPath());
         return properties;
     }
 
     @SuppressWarnings("unused")
-    private Properties createRemoteDestinationProperties()
+    private Properties createRemoteViaRsyncDestinationProperties()
     {
         final Properties properties = new Properties();
-        properties.setProperty(RsyncDataSetCopier.DESTINATION_KEY,
-                "localhost:" + destination.getPath());
+        properties.setProperty(RsyncDataSetCopier.DESTINATION_KEY, HOST + ":" + RSYNC_MODULE + ":"
+                + destination.getPath());
+        properties.setProperty(RsyncDataSetCopier.RSYNC_PASSWORD_FILE_KEY, "abc-password");
         properties.setProperty(RsyncDataSetCopier.RSYNC_EXEC + "-executable", rsyncExec.getPath());
         properties.setProperty(RsyncDataSetCopier.SSH_EXEC + "-executable", sshExec.getPath());
         return properties;
     }
 
+    /*
+     * --< COMMON >---------------------------------------------------------------------------------
+     */
+
+    private void assertSuccessful(Status status)
+    {
+        assertEquals(Status.OK, status);
+    }
+
+    private void assertError(Status status, String expectedErrorMessage)
+    {
+        assertTrue(status.isError());
+        assertEquals(expectedErrorMessage, status.tryGetErrorMessage());
+    }
+
+    private void assertTrue(BooleanStatus boolStatus)
+    {
+        assertEquals(true, boolStatus.isSuccess());
+        assertEquals(false, boolStatus.isError());
+    }
+
+    private void assertFalse(BooleanStatus boolStatus)
+    {
+        assertEquals(false, boolStatus.isSuccess());
+        assertEquals(false, boolStatus.isError());
+    }
+
+    private void assertError(BooleanStatus boolStatus, String expectedErrorMessage)
+    {
+        assertEquals(false, boolStatus.isSuccess());
+        assertEquals(true, boolStatus.isError());
+        assertEquals(expectedErrorMessage, boolStatus.tryGetMessage());
+    }
+
+    private void assertDs1InStore()
+    {
+        assertEquals(true, ds1Data.exists());
+        assertEquals(DATA1, FileUtilities.loadToString(ds1Data).trim());
+    }
+
+    private void assertDs2InStore()
+    {
+        assertEquals(true, ds2Data.exists());
+        assertEquals(DATA2, FileUtilities.loadToString(ds2Data).trim());
+    }
+
+    private void assertDs1InArchive()
+    {
+        assertEquals(true, ds1ArchivedLocationFile.isDirectory());
+        assertEquals(ds1Data.lastModified(), ds1ArchivedDataFile.lastModified());
+        assertEquals(DATA1, FileUtilities.loadToString(ds1ArchivedDataFile).trim());
+    }
+
+    private void assertDs2InArchive()
+    {
+        assertEquals(true, ds2ArchivedLocationFile.isDirectory());
+        assertEquals(ds2Data.lastModified(), ds2ArchivedDataFile.lastModified());
+        assertEquals(DATA2, FileUtilities.loadToString(ds2ArchivedDataFile).trim());
+    }
+
+    private void assertDs1NotInArchive()
+    {
+        assertFalse(ds1ArchivedLocationFile.exists());
+    }
+
+    private void assertDs2NotInArchive()
+    {
+        assertFalse(ds2ArchivedLocationFile.exists());
+    }
+
     private void prepareForCheckingLastModifiedDate()
     {
         // Sleep long enough to test last modified date of target will be same as of source.
@@ -443,4 +845,5 @@ public class RsyncDataSetCopierTest extends AbstractFileSystemTestCase
             // ignored
         }
     }
+
 }