diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetHandler.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetHandler.java
index e49b466834245b1f295b4d0465a97cc63dea47d7..a5c1cd2141907cb71f9fc0a886fc23c22f114e92 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetHandler.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetHandler.java
@@ -263,7 +263,7 @@ public class JythonTopLevelDataSetHandler<T extends DataSetInformation> extends
PyFunction function = tryJythonFunction(interpreter, POST_STORAGE_FUNCTION_NAME);
if (null != function)
{
- invokeTransactionFunctionWithContext(function, transaction);
+ invokeTransactionFunctionWithContext(function, service, transaction);
} else
{
function = tryJythonFunction(interpreter, COMMIT_TRANSACTION_FUNCTION_NAME);
@@ -281,7 +281,7 @@ public class JythonTopLevelDataSetHandler<T extends DataSetInformation> extends
PyFunction function = tryJythonFunction(interpreter, PRE_REGISTRATION_FUNCTION_NAME);
if (null != function)
{
- invokeTransactionFunctionWithContext(function, transaction);
+ invokeTransactionFunctionWithContext(function, service, transaction);
}
}
@@ -292,7 +292,7 @@ public class JythonTopLevelDataSetHandler<T extends DataSetInformation> extends
PyFunction function = tryJythonFunction(interpreter, POST_REGISTRATION_FUNCTION_NAME);
if (null != function)
{
- invokeTransactionFunctionWithContext(function, transaction);
+ invokeTransactionFunctionWithContext(function, service, transaction);
}
}
@@ -368,7 +368,7 @@ public class JythonTopLevelDataSetHandler<T extends DataSetInformation> extends
* Pulled out as a separate method so tests can hook in.
*/
protected void invokeTransactionFunctionWithContext(PyFunction function,
- DataSetRegistrationTransaction<T> transaction)
+ DataSetRegistrationService<T> service, DataSetRegistrationTransaction<T> transaction)
{
function.__call__(Py.java2py(transaction), Py.java2py(null));
}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/AbstractJythonDataSetHandlerTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/AbstractJythonDataSetHandlerTest.java
index cb40a6555dfe1d158496d1c6abd098c48d500281..bd88ade99516220093b69ed49daf02fdfe9d85f6 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/AbstractJythonDataSetHandlerTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/AbstractJythonDataSetHandlerTest.java
@@ -396,6 +396,12 @@ public abstract class AbstractJythonDataSetHandlerTest extends AbstractFileSyste
protected boolean didSecondaryTransactionErrorNotificationHappen = false;
+ protected boolean didPostRegistrationFunctionRunHappen = false;
+
+ protected boolean didPreRegistrationFunctionRunHappen = false;
+
+ protected boolean didPostStorageFunctionRunHappen = false;
+
public TestingDataSetHandler(TopLevelDataSetRegistratorGlobalState globalState,
boolean shouldRegistrationFail, boolean shouldReThrowRollbackException)
{
@@ -491,7 +497,7 @@ public abstract class AbstractJythonDataSetHandlerTest extends AbstractFileSyste
((JythonDataSetRegistrationService<DataSetInformation>) service)
.getInterpreter();
didRollbackTransactionFunctionRunHappen =
- interpreter.get("didTransactionRollbackHappen", Boolean.class);
+ readBoolean(interpreter, "didTransactionRollbackHappen");
}
@Override
@@ -505,9 +511,38 @@ public abstract class AbstractJythonDataSetHandlerTest extends AbstractFileSyste
((JythonDataSetRegistrationService<DataSetInformation>) service)
.getInterpreter();
didCommitTransactionFunctionRunHappen =
- interpreter.get("didTransactionCommitHappen", Boolean.class);
+ readBoolean(interpreter, "didTransactionCommitHappen");
}
+ @Override
+ protected void invokeTransactionFunctionWithContext(PyFunction function,
+ DataSetRegistrationService<DataSetInformation> service,
+ DataSetRegistrationTransaction<DataSetInformation> transaction)
+ {
+ super.invokeTransactionFunctionWithContext(function, service, transaction);
+ PythonInterpreter interpreter =
+ ((JythonDataSetRegistrationService<DataSetInformation>) service)
+ .getInterpreter();
+
+ didPreRegistrationFunctionRunHappen =
+ readBoolean(interpreter,"didPreRegistrationFunctionRunHappen");
+
+ didPostRegistrationFunctionRunHappen =
+ readBoolean(interpreter, "didPostRegistrationFunctionRunHappen");
+
+ didPostStorageFunctionRunHappen =
+ readBoolean(interpreter,"didPostStorageFunctionRunHappen");
+
+ }
+
+ //reads boolean or false if null from interpreter
+ private boolean readBoolean(PythonInterpreter interpreter, String variable)
+ {
+ Boolean retVal = interpreter.get(variable, Boolean.class);
+ if (retVal == null) return false;
+ return retVal;
+ }
+
@Override
public void didEncounterSecondaryTransactionErrors(
DataSetRegistrationService<DataSetInformation> service,
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetRegistratorTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetRegistratorTest.java
index 4818ee027f81833f41d2a09950c5700057ce8f0f..29b620f3bfa2b009887c67d68684e707d5d369d1 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetRegistratorTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetRegistratorTest.java
@@ -52,6 +52,7 @@ import ch.systemsx.cisd.openbis.dss.generic.shared.utils.DatasetLocationUtil;
import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.SearchCriteria;
import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.SearchCriteria.MatchClause;
import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.SearchCriteria.MatchClauseAttribute;
+import ch.systemsx.cisd.openbis.generic.shared.authorization.annotation.ShouldFlattenCollections;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ContainerDataSet;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetType;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.EntityProperty;
@@ -204,6 +205,18 @@ public class JythonTopLevelDataSetRegistratorTest extends AbstractJythonDataSetH
testCase.shouldValidationFail = true;
testCases.add(testCase);
+ testCase =
+ new TestCaseParameters(
+ "The simple validation without post_storage function defined.");
+ testCase.dropboxScriptPath = "simple-transaction-without-post-storage.py";
+ testCase.postStorageFunctionNotDefinedInADropbox = true;
+ testCases.add(testCase);
+
+ testCase = new TestCaseParameters("Dataset file not found.");
+ testCase.dropboxScriptPath = "file-not-found.py";
+ testCase.shouldNotFindDataSetFile = true;
+ testCases.add(testCase);
+
// TODO: Add more scenarios:
// - Test move to error
// - Test moving of the original file in case of validation error
@@ -230,19 +243,52 @@ public class JythonTopLevelDataSetRegistratorTest extends AbstractJythonDataSetH
*/
private static class TestCaseParameters
{
-
+ /**
+ * short description of the test. Will be presented in the test results view
+ */
protected String title;
+ /**
+ * The dropbox script file that should be used for this test case
+ */
+ protected String dropboxScriptPath = "simple-transaction.py";
+
+ /**
+ * Specifies what properties should be overriden for this test case.
+ */
protected HashMap<String, String> overrideProperties;
+ /**
+ * Describe what should happen with incoming data after execution of this test case.
+ */
protected String incomingDataSetAfterRegistration = "deleted";
+ /**
+ * Specifies the custom creator of datasets instead of createDataWithOneSubDataSet.
+ */
protected IDelegatedAction createDataSetDelegate = null;
+ /**
+ * True if the registration of metadata should fail
+ */
protected boolean shouldRegistrationFail = false;
+ /**
+ * True if assertValidDataSet method should return validation error on dataset
+ */
protected boolean shouldValidationFail = false;
+ /**
+ * True if the dropbox script should not find the specified datasetFile
+ */
+ protected boolean shouldNotFindDataSetFile = false;
+
+ /**
+ * True if commit_transaction function is defined in a jython dropbox script file, and
+ * post_storage function is not.
+ */
+ protected boolean postStorageFunctionNotDefinedInADropbox = false;
+
private TestCaseParameters(String title)
{
this.title = title;
@@ -259,9 +305,11 @@ public class JythonTopLevelDataSetRegistratorTest extends AbstractJythonDataSetH
public void testSimpleTransaction(final TestCaseParameters testCase)
{
setUpHomeDataBaseExpectations();
+
Properties properties =
- createThreadPropertiesRelativeToScriptsFolder("simple-transaction.py",
+ createThreadPropertiesRelativeToScriptsFolder(testCase.dropboxScriptPath,
testCase.overrideProperties);
+
if (testCase.shouldRegistrationFail || testCase.shouldValidationFail)
{
createHandler(properties, false, false);
@@ -289,21 +337,34 @@ public class JythonTopLevelDataSetRegistratorTest extends AbstractJythonDataSetH
one(openBisService).createDataSetCode();
will(returnValue(DATA_SET_CODE));
- atLeast(1).of(openBisService).tryToGetExperiment(
- new ExperimentIdentifierFactory(experiment.getIdentifier())
- .createIdentifier());
- will(returnValue(experiment));
- one(dataSetValidator).assertValidDataSet(DATA_SET_TYPE,
- new File(new File(stagingDirectory, DATA_SET_CODE), "sub_data_set_1"));
+ if (testCase.shouldNotFindDataSetFile)
+ {
+ broken = true;
+ }
- if (testCase.shouldValidationFail)
+ if (false == broken)
{
- Exception innerException = new Exception();
- will(throwException(new UserFailureException("Data set of type '"
- + DATA_SET_CODE + "' is invalid ", innerException)));
+ atLeast(1).of(openBisService).tryToGetExperiment(
+ new ExperimentIdentifierFactory(experiment.getIdentifier())
+ .createIdentifier());
+ will(returnValue(experiment));
+ }
- broken = true;
+ if (false == broken)
+ {
+ one(dataSetValidator).assertValidDataSet(
+ DATA_SET_TYPE,
+ new File(new File(stagingDirectory, DATA_SET_CODE),
+ "sub_data_set_1"));
+
+ if (testCase.shouldValidationFail)
+ {
+ Exception innerException = new Exception();
+ will(throwException(new UserFailureException("Data set of type '"
+ + DATA_SET_CODE + "' is invalid ", innerException)));
+ broken = true;
+ }
}
if (false == broken)
@@ -330,13 +391,31 @@ public class JythonTopLevelDataSetRegistratorTest extends AbstractJythonDataSetH
}
});
- handler.handle(markerFile);
+ if (testCase.shouldNotFindDataSetFile)
+ {
+ try
+ {
+ handler.handle(markerFile);
+ fail("Expected a FileNotFound exception.");
+ } catch (PyException pyException)
+ {
+ IOExceptionUnchecked tunnel = (IOExceptionUnchecked) pyException.getCause();
+ FileNotFoundException ex = (FileNotFoundException) tunnel.getCause();
+ assertTrue(ex.getMessage().startsWith("Neither '/non/existent/path' nor '"));
+ }
+ context.assertIsSatisfied();
+ return;
+ } else
+ {
+ handler.handle(markerFile);
+ }
+
checkInitialDirAfterRegistration(testCase.incomingDataSetAfterRegistration);
- if (!testCase.shouldValidationFail)
+ if (false == testCase.shouldValidationFail)
{
- // the incoming dir in storage processor is created at the beginning of transaction - so
- // after the successful registration
+ // the incoming dir in storage processor is created at the beginning of transaction
+ // so after the successful validation
assertEquals(1, MockStorageProcessor.instance.incomingDirs.size());
}
@@ -345,23 +424,15 @@ public class JythonTopLevelDataSetRegistratorTest extends AbstractJythonDataSetH
assertEquals(expectedCommitCount, MockStorageProcessor.instance.calledCommitCount);
+ assertJythonHooksExecuted(testCase);
+
if (testCase.shouldValidationFail)
{
} else if (testCase.shouldRegistrationFail)
{
assertEquals("[]", Arrays.asList(stagingDirectory.list()).toString());
-
- TestingDataSetHandler theHandler = (TestingDataSetHandler) handler;
- assertFalse(theHandler.didRollbackDataSetRegistrationFunctionRun);
- assertFalse(theHandler.didRollbackServiceFunctionRun);
- assertTrue(theHandler.didTransactionRollbackHappen);
- assertTrue(theHandler.didRollbackTransactionFunctionRunHappen);
} else
{
- TestingDataSetHandler theHandler = (TestingDataSetHandler) handler;
- assertTrue(theHandler.didCommitTransactionFunctionRunHappen);
- assertFalse(theHandler.didRollbackTransactionFunctionRunHappen);
-
assertEquals(1, MockStorageProcessor.instance.incomingDirs.size());
assertEquals(1, atomicatOperationDetails.recordedObject().getDataSetRegistrations()
.size());
@@ -395,34 +466,48 @@ public class JythonTopLevelDataSetRegistratorTest extends AbstractJythonDataSetH
context.assertIsSatisfied();
}
- @Test
- public void testFileNotFound()
+ private void assertJythonHooksExecuted(final TestCaseParameters testCase)
{
- setUpHomeDataBaseExpectations();
- Properties properties = createThreadPropertiesRelativeToScriptsFolder("file-not-found.py");
- createHandler(properties, false, true);
- createData();
-
- context.checking(new Expectations()
- {
- {
- one(openBisService).createDataSetCode();
- will(returnValue(DATA_SET_CODE));
- }
- });
+ TestingDataSetHandler theHandler = (TestingDataSetHandler) handler;
- try
+ if (testCase.shouldValidationFail)
{
- handler.handle(markerFile);
- fail("Expected a FileNotFound exception.");
- } catch (PyException pyException)
+ } else if (testCase.shouldRegistrationFail)
{
- IOExceptionUnchecked tunnel = (IOExceptionUnchecked) pyException.getCause();
- FileNotFoundException ex = (FileNotFoundException) tunnel.getCause();
- assertTrue(ex.getMessage().startsWith("Neither '/non/existent/path' nor '"));
+ assertFalse(theHandler.didRollbackDataSetRegistrationFunctionRun);
+ assertFalse(theHandler.didRollbackServiceFunctionRun);
+ assertTrue(theHandler.didTransactionRollbackHappen);
+ assertTrue(theHandler.didRollbackTransactionFunctionRunHappen);
+
+ assertTrue(theHandler.didPreRegistrationFunctionRunHappen);
+ assertFalse(theHandler.didPostRegistrationFunctionRunHappen);
+
+ assertFalse(theHandler.didCommitTransactionFunctionRunHappen);
+ assertFalse(theHandler.didPostStorageFunctionRunHappen);
+
+ } else
+ {
+ assertFalse(theHandler.didRollbackTransactionFunctionRunHappen);
+
+ assertTrue(theHandler.didPreRegistrationFunctionRunHappen);
+ assertTrue(theHandler.didPostRegistrationFunctionRunHappen);
+
+ if (testCase.postStorageFunctionNotDefinedInADropbox)
+ {
+ assertTrue(theHandler.didCommitTransactionFunctionRunHappen);
+ assertFalse(theHandler.didPostStorageFunctionRunHappen);
+ } else
+ {
+ assertFalse(theHandler.didCommitTransactionFunctionRunHappen);
+ assertTrue(theHandler.didPostStorageFunctionRunHappen);
+ }
}
+ }
+
+ @Test
+ public void testFileNotFound()
+ {
- context.assertIsSatisfied();
}
@Test
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/simple-transaction-without-post-storage.py b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/simple-transaction-without-post-storage.py
new file mode 100644
index 0000000000000000000000000000000000000000..fe7671fab6ca59eba33b7b5b4f64392200599d3a
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/simple-transaction-without-post-storage.py
@@ -0,0 +1,21 @@
+def rollback_transaction(service, transaction, algorithmRunner, throwable):
+ global didTransactionRollbackHappen
+ didTransactionRollbackHappen = True
+
+def commit_transaction(service, transaction):
+ global didTransactionCommitHappen
+ didTransactionCommitHappen = True
+
+def pre_metadata_registration(transaction, context):
+ global didPreRegistrationFunctionRunHappen
+ didPreRegistrationFunctionRunHappen = True
+
+def post_metadata_registration(transaction, context):
+ global didPostRegistrationFunctionRunHappen
+ didPostRegistrationFunctionRunHappen = True
+
+transaction = service.transaction(incoming, factory)
+dataSet = transaction.createNewDataSet()
+transaction.moveFile(incoming.getPath() + '/sub_data_set_1', dataSet)
+dataSet.setDataSetType('O1')
+dataSet.setExperiment(transaction.getExperiment('/SPACE/PROJECT/EXP'))
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/simple-transaction.py b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/simple-transaction.py
index d9227f3f0e8e1d43d367fdc5ca7382cabd08a377..06f747c519f6dd9e6041126f148c961b42173feb 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/simple-transaction.py
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/simple-transaction.py
@@ -1,11 +1,23 @@
def rollback_transaction(service, transaction, algorithmRunner, throwable):
- global didTransactionRollbackHappen
- didTransactionRollbackHappen = True
+ global didTransactionRollbackHappen
+ didTransactionRollbackHappen = True
def commit_transaction(service, transaction):
- global didTransactionCommitHappen
- didTransactionCommitHappen = True
-
+ global didTransactionCommitHappen
+ didTransactionCommitHappen = True
+
+def post_storage(transaction, context):
+ global didPostStorageFunctionRunHappen
+ didPostStorageFunctionRunHappen = True
+
+def pre_metadata_registration(transaction, context):
+ global didPreRegistrationFunctionRunHappen
+ didPreRegistrationFunctionRunHappen = True
+
+def post_metadata_registration(transaction, context):
+ global didPostRegistrationFunctionRunHappen
+ didPostRegistrationFunctionRunHappen = True
+
transaction = service.transaction(incoming, factory)
dataSet = transaction.createNewDataSet()
transaction.moveFile(incoming.getPath() + '/sub_data_set_1', dataSet)