Skip to content
Snippets Groups Projects
Commit 4b187c6d authored by buczekp's avatar buczekp
Browse files

[LMS-1352] fixed and extended some tests 

SVN: 14600
parent 2ff8b5ce
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetCopier.java
+ 34
− 11
View file @ 4b187c6d
......@@ -28,6 +28,7 @@ import ch.systemsx.cisd.common.filesystem.BooleanStatus;
import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.common.filesystem.IPathCopier;
import ch.systemsx.cisd.common.filesystem.rsync.RsyncCopier;
import ch.systemsx.cisd.common.filesystem.ssh.ISshCommandExecutor;
import ch.systemsx.cisd.common.filesystem.ssh.SshCommandExecutor;
import ch.systemsx.cisd.common.highwatermark.HostAwareFile;
import ch.systemsx.cisd.common.highwatermark.HostAwareFileWithHighwaterMark;
......@@ -58,13 +59,15 @@ public class DataSetCopier extends AbstractDropboxProcessingPlugin
private static final String ALREADY_EXIST_MSG = "already exist";
private static final String COPYING_FAILED_MSG = "copying failed";
@Private
static final String COPYING_FAILED_MSG = "copying failed";
private static final String RSYNC_EXEC = "rsync";
private static final String SSH_EXEC = "ssh";
private static final long SSH_TIMEOUT_MILLIS = 15 * 1000; // 15s
@Private
static final long SSH_TIMEOUT_MILLIS = 15 * 1000; // 15s
@Private
static interface IPathCopierFactory
......@@ -82,6 +85,23 @@ public class DataSetCopier extends AbstractDropboxProcessingPlugin
}
}
@Private
static interface ISshCommandExecutorFactory
{
ISshCommandExecutor create(File sshExecutableOrNull, String host);
}
private static final class SshCommandExecutorFactory implements Serializable,
ISshCommandExecutorFactory
{
private static final long serialVersionUID = 1L;
public ISshCommandExecutor create(File sshExecutableOrNull, String host)
{
return new SshCommandExecutor(sshExecutableOrNull, host);
}
}
private static final class Copier implements Serializable, IPostRegistrationDatasetHandler
{
......@@ -93,7 +113,7 @@ public class DataSetCopier extends AbstractDropboxProcessingPlugin
private final File sshExecutable;
private final SshCommandExecutor sshCommandExecutor;
private final ISshCommandExecutor sshCommandExecutor;
private final Properties properties;
......@@ -105,19 +125,20 @@ public class DataSetCopier extends AbstractDropboxProcessingPlugin
private final String rsyncPasswordFile;
private final IPathCopierFactory factory;
private final IPathCopierFactory pathCopierFactory;
public Copier(Properties properties, IPathCopierFactory factory)
public Copier(Properties properties, IPathCopierFactory pathCopierFactory,
ISshCommandExecutorFactory sshCommandExecutorFactory)
{
this.properties = properties;
this.factory = factory;
this.pathCopierFactory = pathCopierFactory;
rsyncPasswordFile = properties.getProperty(RSYNC_PASSWORD_FILE_KEY);
rsyncExecutable = getExecutable(RSYNC_EXEC);
sshExecutable = getExecutable(SSH_EXEC);
HostAwareFile hostAwareFile =
HostAwareFileWithHighwaterMark.fromProperties(properties, DESTINATION_KEY);
host = hostAwareFile.tryGetHost();
sshCommandExecutor = new SshCommandExecutor(sshExecutable, host);
sshCommandExecutor = sshCommandExecutorFactory.create(sshExecutable, host);
rsyncModule = hostAwareFile.tryGetRsyncModule();
destination = hostAwareFile.getFile();
getCopier();
......@@ -127,7 +148,7 @@ public class DataSetCopier extends AbstractDropboxProcessingPlugin
{
if (copier == null)
{
copier = factory.create(rsyncExecutable, sshExecutable);
copier = pathCopierFactory.create(rsyncExecutable, sshExecutable);
copier.check();
if (host != null)
{
......@@ -214,13 +235,15 @@ public class DataSetCopier extends AbstractDropboxProcessingPlugin
public DataSetCopier(Properties properties, File storeRoot)
{
this(properties, storeRoot, new RsyncCopierFactory());
this(properties, storeRoot, new RsyncCopierFactory(), new SshCommandExecutorFactory());
}
@Private
DataSetCopier(Properties properties, File storeRoot, IPathCopierFactory factory)
DataSetCopier(Properties properties, File storeRoot, IPathCopierFactory pathCopierFactory,
ISshCommandExecutorFactory sshCommandExecutorFactory)
{
super(properties, storeRoot, new Copier(properties, factory));
super(properties, storeRoot, new Copier(properties, pathCopierFactory,
sshCommandExecutorFactory));
}
}
datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetCopierTest.java
+ 95
− 48
View file @ 4b187c6d
......@@ -33,10 +33,13 @@ import org.testng.annotations.Test;
import ch.rinn.restrictions.Friend;
import ch.systemsx.cisd.base.tests.AbstractFileSystemTestCase;
import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.filesystem.BooleanStatus;
import ch.systemsx.cisd.common.filesystem.IPathCopier;
import ch.systemsx.cisd.common.filesystem.ssh.ISshCommandExecutor;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.DataSetCopier.IPathCopierFactory;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.DataSetCopier.ISshCommandExecutorFactory;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.ProcessingStatus;
import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
/**
......@@ -51,10 +54,14 @@ public class DataSetCopierTest extends AbstractFileSystemTestCase
private Mockery context;
private IPathCopierFactory factory;
private IPathCopierFactory pathFactory;
private ISshCommandExecutorFactory sshFactory;
private IPathCopier copier;
private ISshCommandExecutor sshCommandExecutor;
private File storeRoot;
private File sshExecutableDummy;
......@@ -75,8 +82,10 @@ public class DataSetCopierTest extends AbstractFileSystemTestCase
public void beforeMethod() throws IOException
{
context = new Mockery();
factory = context.mock(DataSetCopier.IPathCopierFactory.class);
pathFactory = context.mock(DataSetCopier.IPathCopierFactory.class);
sshFactory = context.mock(DataSetCopier.ISshCommandExecutorFactory.class);
copier = context.mock(IPathCopier.class);
sshCommandExecutor = context.mock(ISshCommandExecutor.class);
storeRoot = new File(workingDirectory, "store");
storeRoot.mkdirs();
sshExecutableDummy = new File(workingDirectory, "my-ssh");
......@@ -111,7 +120,7 @@ public class DataSetCopierTest extends AbstractFileSystemTestCase
{
try
{
new DataSetCopier(new Properties(), storeRoot, factory);
new DataSetCopier(new Properties(), storeRoot, pathFactory, sshFactory);
fail("ConfigurationFailureException expected");
} catch (ConfigurationFailureException ex)
{
......@@ -128,7 +137,7 @@ public class DataSetCopierTest extends AbstractFileSystemTestCase
rsyncExecutableDummy.delete();
try
{
new DataSetCopier(properties, storeRoot, factory);
new DataSetCopier(properties, storeRoot, pathFactory, sshFactory);
fail("ConfigurationFailureException expected");
} catch (ConfigurationFailureException ex)
{
......@@ -145,7 +154,7 @@ public class DataSetCopierTest extends AbstractFileSystemTestCase
sshExecutableDummy.delete();
try
{
new DataSetCopier(properties, storeRoot, factory);
new DataSetCopier(properties, storeRoot, pathFactory, sshFactory);
fail("ConfigurationFailureException expected");
} catch (ConfigurationFailureException ex)
{
......@@ -163,7 +172,7 @@ public class DataSetCopierTest extends AbstractFileSystemTestCase
prepareCreateAndCheckCopier("host", null, false);
try
{
new DataSetCopier(properties, storeRoot, factory);
new DataSetCopier(properties, storeRoot, pathFactory, sshFactory);
fail("ConfigurationFailureException expected");
} catch (ConfigurationFailureException ex)
{
......@@ -181,7 +190,7 @@ public class DataSetCopierTest extends AbstractFileSystemTestCase
prepareCreateAndCheckCopier("host", "abc", false);
try
{
new DataSetCopier(properties, storeRoot, factory);
new DataSetCopier(properties, storeRoot, pathFactory, sshFactory);
fail("ConfigurationFailureException expected");
} catch (ConfigurationFailureException ex)
{
......@@ -205,7 +214,8 @@ public class DataSetCopierTest extends AbstractFileSystemTestCase
will(returnValue(Status.OK));
}
});
DataSetCopier dataSetCopier = new DataSetCopier(properties, storeRoot, factory);
DataSetCopier dataSetCopier =
new DataSetCopier(properties, storeRoot, pathFactory, sshFactory);
dataSetCopier.process(Arrays.asList(ds1, ds2));
......@@ -222,37 +232,45 @@ public class DataSetCopierTest extends AbstractFileSystemTestCase
{
one(copier).copyToRemote(ds1Data, new File("tmp/test"), null, null, null);
will(returnValue(Status.createError("error message")));
one(copier).copyToRemote(ds2Data, new File("tmp/test"), null, null, null);
will(returnValue(Status.OK));
}
});
DataSetCopier dataSetCopier = new DataSetCopier(properties, storeRoot, factory);
DataSetCopier dataSetCopier =
new DataSetCopier(properties, storeRoot, pathFactory, sshFactory);
try
{
dataSetCopier.process(Arrays.asList(ds1, ds2));
fail("EnvironmentFailureException expected");
} catch (EnvironmentFailureException ex)
{
assertEquals(
"Could not copy data set ds1 to destination folder 'tmp/test': error message",
ex.getMessage());
}
ProcessingStatus processingStatus = dataSetCopier.process(Arrays.asList(ds1, ds2));
// processing first data set fails but second one is processed successfully
Status errorStatus = Status.createError(DataSetCopier.COPYING_FAILED_MSG);
assertEquals(1, processingStatus.getErrorStatuses().size());
assertEquals(errorStatus, processingStatus.getErrorStatuses().get(0));
assertEquals(1, processingStatus.getDatasetsByStatus(errorStatus).size());
assertEquals(ds1, processingStatus.getDatasetsByStatus(errorStatus).get(0));
assertEquals(1, processingStatus.getDatasetsByStatus(Status.OK).size());
assertEquals(ds2, processingStatus.getDatasetsByStatus(Status.OK).get(0));
context.assertIsSatisfied();
}
@Test
public void testCopyRemotlyViaSSH()
public void testCopyRemotelyViaSsh()
{
properties.setProperty(DESTINATION_KEY, "host:tmp/test");
prepareCreateAndCheckCopier("host", null, true);
context.checking(new Expectations()
{
{
one(sshCommandExecutor).exists(new File("tmp/test/data.txt"),
DataSetCopier.SSH_TIMEOUT_MILLIS);
will(returnValue(BooleanStatus.createFalse()));
one(copier).copyToRemote(ds1Data, new File("tmp/test"), "host", null, null);
will(returnValue(Status.OK));
}
});
DataSetCopier dataSetCopier = new DataSetCopier(properties, storeRoot, factory);
DataSetCopier dataSetCopier =
new DataSetCopier(properties, storeRoot, pathFactory, sshFactory);
dataSetCopier.process(Arrays.asList(ds1));
......@@ -260,35 +278,45 @@ public class DataSetCopierTest extends AbstractFileSystemTestCase
}
@Test
public void testCopyRemotelyViaSSH()
public void testCopyRemotelyViaSshWithErrors()
{
properties.setProperty(DESTINATION_KEY, "host:tmp/test");
prepareCreateAndCheckCopier("host", null, true);
context.checking(new Expectations()
{
{
one(sshCommandExecutor).exists(new File("tmp/test/data.txt"),
DataSetCopier.SSH_TIMEOUT_MILLIS);
will(returnValue(BooleanStatus.createFalse()));
one(copier).copyToRemote(ds1Data, new File("tmp/test"), "host", null, null);
will(returnValue(Status.createError("error message")));
one(sshCommandExecutor).exists(new File("tmp/test/images"),
DataSetCopier.SSH_TIMEOUT_MILLIS);
will(returnValue(BooleanStatus.createFalse()));
one(copier).copyToRemote(ds2Data, new File("tmp/test"), "host", null, null);
will(returnValue(Status.OK));
}
});
DataSetCopier dataSetCopier = new DataSetCopier(properties, storeRoot, factory);
DataSetCopier dataSetCopier =
new DataSetCopier(properties, storeRoot, pathFactory, sshFactory);
try
{
dataSetCopier.process(Arrays.asList(ds1, ds2));
fail("EnvironmentFailureException expected");
} catch (EnvironmentFailureException ex)
{
assertEquals(
"Could not copy data set ds1 to destination folder 'tmp/test' on host 'host': "
+ "error message", ex.getMessage());
}
ProcessingStatus processingStatus = dataSetCopier.process(Arrays.asList(ds1, ds2));
// processing first data set fails but second one is processed successfully
Status errorStatus = Status.createError(DataSetCopier.COPYING_FAILED_MSG);
assertEquals(1, processingStatus.getErrorStatuses().size());
assertEquals(errorStatus, processingStatus.getErrorStatuses().get(0));
assertEquals(1, processingStatus.getDatasetsByStatus(errorStatus).size());
assertEquals(ds1, processingStatus.getDatasetsByStatus(errorStatus).get(0));
assertEquals(1, processingStatus.getDatasetsByStatus(Status.OK).size());
assertEquals(ds2, processingStatus.getDatasetsByStatus(Status.OK).get(0));
context.assertIsSatisfied();
}
@Test
public void testCopyRemotlyViaRsyncServer()
public void testCopyRemotelyViaRsyncServer()
{
properties.setProperty(DESTINATION_KEY, "host:abc:tmp/test");
properties.setProperty(RSYNC_PASSWORD_FILE_KEY, "abc-password");
......@@ -296,12 +324,16 @@ public class DataSetCopierTest extends AbstractFileSystemTestCase
context.checking(new Expectations()
{
{
one(sshCommandExecutor).exists(new File("tmp/test/data.txt"),
DataSetCopier.SSH_TIMEOUT_MILLIS);
will(returnValue(BooleanStatus.createFalse()));
one(copier).copyToRemote(ds1Data, new File("tmp/test"), "host", "abc",
"abc-password");
will(returnValue(Status.OK));
}
});
DataSetCopier dataSetCopier = new DataSetCopier(properties, storeRoot, factory);
DataSetCopier dataSetCopier =
new DataSetCopier(properties, storeRoot, pathFactory, sshFactory);
dataSetCopier.process(Arrays.asList(ds1));
......@@ -309,7 +341,7 @@ public class DataSetCopierTest extends AbstractFileSystemTestCase
}
@Test
public void testCopyRemotelyViaRsyncServer()
public void testCopyRemotelyViaRsyncServerWithErrors()
{
properties.setProperty(DESTINATION_KEY, "host:abc:tmp/test");
properties.setProperty(RSYNC_PASSWORD_FILE_KEY, "abc-password");
......@@ -317,22 +349,34 @@ public class DataSetCopierTest extends AbstractFileSystemTestCase
context.checking(new Expectations()
{
{
one(sshCommandExecutor).exists(new File("tmp/test/data.txt"),
DataSetCopier.SSH_TIMEOUT_MILLIS);
will(returnValue(BooleanStatus.createFalse()));
one(copier).copyToRemote(ds1Data, new File("tmp/test"), "host", "abc",
"abc-password");
will(returnValue(Status.createError("error message")));
one(sshCommandExecutor).exists(new File("tmp/test/images"),
DataSetCopier.SSH_TIMEOUT_MILLIS);
will(returnValue(BooleanStatus.createFalse()));
one(copier).copyToRemote(ds2Data, new File("tmp/test"), "host", "abc",
"abc-password");
will(returnValue(Status.OK));
}
});
DataSetCopier dataSetCopier = new DataSetCopier(properties, storeRoot, factory);
DataSetCopier dataSetCopier =
new DataSetCopier(properties, storeRoot, pathFactory, sshFactory);
try
{
dataSetCopier.process(Arrays.asList(ds1, ds2));
fail("EnvironmentFailureException expected");
} catch (EnvironmentFailureException ex)
{
assertEquals("Could not copy data set ds1 to destination folder 'tmp/test' "
+ "on host 'host' for rsync module 'abc': error message", ex.getMessage());
}
ProcessingStatus processingStatus = dataSetCopier.process(Arrays.asList(ds1, ds2));
// processing first data set fails but second one is processed successfully
Status errorStatus = Status.createError(DataSetCopier.COPYING_FAILED_MSG);
assertEquals(1, processingStatus.getErrorStatuses().size());
assertEquals(errorStatus, processingStatus.getErrorStatuses().get(0));
assertEquals(1, processingStatus.getDatasetsByStatus(errorStatus).size());
assertEquals(ds1, processingStatus.getDatasetsByStatus(errorStatus).get(0));
assertEquals(1, processingStatus.getDatasetsByStatus(Status.OK).size());
assertEquals(ds2, processingStatus.getDatasetsByStatus(Status.OK).get(0));
context.assertIsSatisfied();
}
......@@ -343,9 +387,12 @@ public class DataSetCopierTest extends AbstractFileSystemTestCase
context.checking(new Expectations()
{
{
one(factory).create(rsyncExecutableDummy, sshExecutableDummy);
one(pathFactory).create(rsyncExecutableDummy, sshExecutableDummy);
will(returnValue(copier));
one(sshFactory).create(sshExecutableDummy, hostOrNull);
will(returnValue(sshCommandExecutor));
one(copier).check();
if (hostOrNull != null)
{
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment