From 70fb8d7ee6467363c3232ddabb5975aac6841a08 Mon Sep 17 00:00:00 2001
From: cramakri <cramakri>
Date: Tue, 10 Jul 2012 09:25:37 +0000
Subject: [PATCH] BIS-21 SP-177 : Developing tests for this situation
SVN: 26047
---
.../AbstractJythonDataSetHandlerTest.java | 5 +-
.../JythonTopLevelDataSetRegistratorTest.java | 172 +++++++++++++++++-
.../v2-delete-after-registration.py | 25 +++
.../v2-delete-before-registration.py | 24 +++
4 files changed, 220 insertions(+), 6 deletions(-)
create mode 100644 datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/v2-delete-after-registration.py
create mode 100644 datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/v2-delete-before-registration.py
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/AbstractJythonDataSetHandlerTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/AbstractJythonDataSetHandlerTest.java
index 0e00657647d..6dd3eaf29d0 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/AbstractJythonDataSetHandlerTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/AbstractJythonDataSetHandlerTest.java
@@ -196,7 +196,7 @@ public abstract class AbstractJythonDataSetHandlerTest extends AbstractFileSyste
cleanUpDirectoryBeforeTheTest(recoveryStateDir);
DssRegistrationHealthMonitor.setOpenBisServiceForTest(openBisService);
-
+
JythonHookTestTool.createInTest().clear();
}
@@ -320,7 +320,8 @@ public abstract class AbstractJythonDataSetHandlerTest extends AbstractFileSyste
String validationScriptPropertyOrNull, HashMap<String, String> overrideOrNull)
{
Properties threadProperties = new Properties();
- threadProperties.setProperty(ThreadParameters.INCOMING_DIR, "incoming");
+ threadProperties.setProperty(ThreadParameters.INCOMING_DIR,
+ workingDirectory.getAbsolutePath());
threadProperties.setProperty(ThreadParameters.INCOMING_DATA_COMPLETENESS_CONDITION,
ThreadParameters.INCOMING_DATA_COMPLETENESS_CONDITION_MARKER_FILE);
threadProperties.setProperty(ThreadParameters.DELETE_UNIDENTIFIED_KEY, "false");
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetRegistratorTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetRegistratorTest.java
index 28b62fca036..6b867e47f01 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetRegistratorTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetRegistratorTest.java
@@ -33,6 +33,7 @@ import java.util.Properties;
import org.jmock.Expectations;
import org.jmock.api.Invocation;
+import org.jmock.internal.ExpectationBuilder;
import org.jmock.lib.action.CustomAction;
import org.python.core.PyException;
import org.testng.annotations.BeforeMethod;
@@ -50,6 +51,7 @@ import ch.systemsx.cisd.common.test.RecordingMatcher;
import ch.systemsx.cisd.common.utilities.IDelegatedAction;
import ch.systemsx.cisd.common.utilities.IPredicate;
import ch.systemsx.cisd.etlserver.IStorageProcessorTransactional;
+import ch.systemsx.cisd.etlserver.TopLevelDataSetRegistratorGlobalState;
import ch.systemsx.cisd.etlserver.IStorageProcessorTransactional.UnstoreDataAction;
import ch.systemsx.cisd.etlserver.ThreadParameters;
import ch.systemsx.cisd.openbis.dss.generic.shared.utils.DatasetLocationUtil;
@@ -127,16 +129,14 @@ public class JythonTopLevelDataSetRegistratorTest extends AbstractJythonDataSetH
return new File(script.getParentFile(), "v2-" + script.getName()).getPath();
}
- @DataProvider(name = "simpleTransactionTestCaseProvider")
- public Object[][] simpleTransactionCases()
+ private static <T> Object[][] asObjectArray(List<T> testCases)
{
- List<TestCaseParameters> testCases = simpleTransactionCasesList();
// here is crappy code for
// return parameters.map( (x) => new Object[]{x} )
Object[][] resultsList = new Object[testCases.size()][];
int index = 0;
- for (TestCaseParameters t : testCases)
+ for (T t : testCases)
{
resultsList[index++] = new Object[]
{ t };
@@ -145,6 +145,13 @@ public class JythonTopLevelDataSetRegistratorTest extends AbstractJythonDataSetH
return resultsList;
}
+ @DataProvider(name = "simpleTransactionTestCaseProvider")
+ public Object[][] simpleTransactionCases()
+ {
+ List<TestCaseParameters> testCases = simpleTransactionCasesList();
+ return asObjectArray(testCases);
+ }
+
public List<TestCaseParameters> simpleTransactionCasesList()
{
// creates data with more than only one dataset
@@ -859,6 +866,163 @@ public class JythonTopLevelDataSetRegistratorTest extends AbstractJythonDataSetH
jythonHookTestTool.assertNoMoreMessages();
}
+ /**
+ * Parameters for the single run of the testIncomingFileDeleted
+ *
+ * @author Chandrasekhar Ramakrishnan
+ */
+ private static class IncomingFileDeletedTestCaseParameters implements Cloneable
+ {
+ /**
+ * short description of the test. Will be presented in the test results view
+ */
+ protected String title;
+
+ /**
+ * The dropbox script file that should be used for this test case
+ */
+ protected String dropboxScriptPath = scriptPathV2("simple-testcase.py");
+
+ /**
+ * Specifies what properties should be overriden for this test case.
+ */
+ protected HashMap<String, String> overrideProperties;
+
+ /**
+ * When does this incoming file get deleted?
+ */
+ protected DeletionPoint deletionPoint = null;
+
+ private IncomingFileDeletedTestCaseParameters(String title)
+ {
+ this.title = title;
+ this.overrideProperties = new HashMap<String, String>();
+ }
+
+ @Override
+ public IncomingFileDeletedTestCaseParameters clone()
+ {
+ try
+ {
+ return (IncomingFileDeletedTestCaseParameters) super.clone();
+ } catch (CloneNotSupportedException e)
+ {
+ return null;
+ }
+ }
+
+ @Override
+ public String toString()
+ {
+ return title;
+ }
+
+ public TestCaseParameters toTestCaseParameters()
+ {
+ TestCaseParameters testCase = new TestCaseParameters(title);
+ testCase.shouldUseAutoRecovery = true;
+
+ return testCase;
+ }
+
+ public enum DeletionPoint
+ {
+ DURING_PRECOMMIT_COPY, BEFORE_OPENBIS_REGISTRATION, AFTER_OPENBIS_REGISTRATION;
+ }
+ }
+
+ @DataProvider(name = "incomingFileDeletedTestCaseProvider")
+ public Object[][] incomingFileDeletedCases()
+ {
+ List<IncomingFileDeletedTestCaseParameters> testCases = incomingFileDeletedCasesList();
+ return asObjectArray(testCases);
+ }
+
+ public List<IncomingFileDeletedTestCaseParameters> incomingFileDeletedCasesList()
+ {
+
+ ArrayList<IncomingFileDeletedTestCaseParameters> testCases =
+ new ArrayList<IncomingFileDeletedTestCaseParameters>();
+
+ // Delete the incoming file while the prestaging copy is being made
+ IncomingFileDeletedTestCaseParameters testCase =
+ new IncomingFileDeletedTestCaseParameters(
+ "Incoming file deleted during prestaging copy.");
+ testCase.deletionPoint =
+ IncomingFileDeletedTestCaseParameters.DeletionPoint.DURING_PRECOMMIT_COPY;
+ testCases.add(testCase);
+
+ // Delete the incoming file after the prestaging copy has been made, but before the metadata
+ // has been registered with the AS
+ testCase =
+ new IncomingFileDeletedTestCaseParameters(
+ "Incoming file deleted before metadata registration.");
+ testCase.dropboxScriptPath = scriptPathV2("delete-before-registration.py");
+ testCase.deletionPoint =
+ IncomingFileDeletedTestCaseParameters.DeletionPoint.BEFORE_OPENBIS_REGISTRATION;
+ testCases.add(testCase);
+
+ // Delete the incoming file after the metadata has been registered with the AS
+ testCase =
+ new IncomingFileDeletedTestCaseParameters(
+ "Incoming file deleted after metadata registration.");
+ testCase.dropboxScriptPath = scriptPathV2("delete-after-registration.py");
+ testCase.deletionPoint =
+ IncomingFileDeletedTestCaseParameters.DeletionPoint.AFTER_OPENBIS_REGISTRATION;
+ testCases.add(testCase);
+
+ return testCases;
+ }
+
+ @Test(dataProvider = "incomingFileDeletedTestCaseProvider")
+ public void testIncomingFileDeleted(final IncomingFileDeletedTestCaseParameters testCase)
+ {
+ // Not yet handled
+ if (testCase.deletionPoint == IncomingFileDeletedTestCaseParameters.DeletionPoint.DURING_PRECOMMIT_COPY)
+ {
+ return;
+ }
+ initializeStorageRecoveryManagerMock();
+ setUpHomeDataBaseExpectations();
+
+ Properties properties =
+ createThreadPropertiesRelativeToScriptsFolder(testCase.dropboxScriptPath,
+ testCase.overrideProperties);
+
+ // Create the handler
+ TopLevelDataSetRegistratorGlobalState globalState = createGlobalState(properties);
+ handler = new TestingDataSetHandlerV2(globalState, false, false);
+
+ createDataWithOneSubDataSet();
+
+ final RecordingMatcher<ch.systemsx.cisd.openbis.generic.shared.dto.AtomicEntityOperationDetails> atomicOperationDetails =
+ new RecordingMatcher<ch.systemsx.cisd.openbis.generic.shared.dto.AtomicEntityOperationDetails>();
+ context.checking(getTestIncomingFileDeletedExcpectations(testCase, atomicOperationDetails));
+ handler.handle(markerFile);
+
+ if (testCase.deletionPoint == IncomingFileDeletedTestCaseParameters.DeletionPoint.AFTER_OPENBIS_REGISTRATION)
+ {
+ assertEquals(1, MockStorageProcessor.instance.incomingDirs.size());
+ assertEquals(1, MockStorageProcessor.instance.calledCommitCount);
+ } else
+ {
+ assertEquals(0, MockStorageProcessor.instance.incomingDirs.size());
+ assertEquals(0, MockStorageProcessor.instance.calledCommitCount);
+
+ }
+
+ assertEquals("[]", Arrays.asList(stagingDirectory.list()).toString());
+ context.assertIsSatisfied();
+ }
+
+ private ExpectationBuilder getTestIncomingFileDeletedExcpectations(
+ final IncomingFileDeletedTestCaseParameters testCase,
+ final RecordingMatcher<AtomicEntityOperationDetails> atomicOperationDetails)
+ {
+ return getSimpleTransactionExpectations(testCase.toTestCaseParameters(),
+ atomicOperationDetails);
+ }
+
@Test
public void testTwoSimpleDataSets()
{
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/v2-delete-after-registration.py b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/v2-delete-after-registration.py
new file mode 100644
index 00000000000..1130ccdee95
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/v2-delete-after-registration.py
@@ -0,0 +1,25 @@
+from java.lang import IllegalArgumentException
+from java.io import File
+
+execfile("sourceTest/java/ch/systemsx/cisd/etlserver/registrator/all-hooks.py")
+
+def post_metadata_registration(context):
+ path_to_delete = context.getPersistentMap().get("path")
+ file_to_delete = File(path_to_delete)
+ file_to_delete.delete()
+ jythonHookTestTool.log("post_metadata_registration")
+
+def process(tr):
+ # Remember the original file so we can delete it before we do metadata registration
+ incomingDir = tr.getGlobalState().getThreadParameters().getIncomingDataDirectory()
+ original = File(incomingDir, tr.getIncoming().getName())
+ context = tr.getRegistrationContext()
+ context.getPersistentMap().put("path", original.getPath())
+
+ dataSet = tr.createNewDataSet()
+ incoming = tr.getIncoming()
+ tr.moveFile(incoming.getPath() + '/sub_data_set_1', dataSet)
+ dataSet.setDataSetType('O1')
+ dataSet.setExperiment(tr.getExperiment('/SPACE/PROJECT/EXP'))
+
+
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/v2-delete-before-registration.py b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/v2-delete-before-registration.py
new file mode 100644
index 00000000000..7a3206ddbfe
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/v2-delete-before-registration.py
@@ -0,0 +1,24 @@
+from java.lang import IllegalArgumentException
+from java.io import File
+
+execfile("sourceTest/java/ch/systemsx/cisd/etlserver/registrator/all-hooks.py")
+
+def pre_metadata_registration(context):
+ path_to_delete = context.getPersistentMap().get("path")
+ file_to_delete = File(path_to_delete)
+ file_to_delete.delete()
+ jythonHookTestTool.log("pre_metadata_registration")
+
+def process(tr):
+ # Remember the original file so we can delete it before we do metadata registration
+ incomingDir = tr.getGlobalState().getThreadParameters().getIncomingDataDirectory()
+ original = File(incomingDir, tr.getIncoming().getName())
+ context = tr.getRegistrationContext()
+ context.getPersistentMap().put("path", original.getPath())
+
+ dataSet = tr.createNewDataSet()
+ incoming = tr.getIncoming()
+ tr.moveFile(incoming.getPath() + '/sub_data_set_1', dataSet)
+ dataSet.setDataSetType('O1')
+ dataSet.setExperiment(tr.getExperiment('/SPACE/PROJECT/EXP'))
+
--
GitLab