diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/AbstractProgrammableTopLevelDataSetHandler.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/AbstractProgrammableTopLevelDataSetHandler.java
index 5f56e20965f33a0102e11b37524d2120849d4d9f..276accefbc8b3ed077188f301c3f76a615bc7338 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/AbstractProgrammableTopLevelDataSetHandler.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/AbstractProgrammableTopLevelDataSetHandler.java
@@ -22,6 +22,8 @@ import java.util.Calendar;
import java.util.Date;
import java.util.List;
+import org.python.core.PyException;
+
import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
import ch.systemsx.cisd.common.concurrent.ConcurrencyUtilities;
import ch.systemsx.cisd.common.exceptions.NotImplementedException;
@@ -83,8 +85,7 @@ public abstract class AbstractProgrammableTopLevelDataSetHandler<T extends DataS
abstract protected void handleDataSet(DataSetFile dataSetFile,
DataSetRegistrationService<T> service) throws Throwable;
- protected void executeProcessFunctionWithRetries(
- IJavaDataSetRegistrationDropboxV2<T> v2Programm,
+ protected void executeProcessFunctionWithRetries(IJavaDataSetRegistrationDropboxV2 v2Programm,
JythonDataSetRegistrationServiceV2<T> service, DataSetFile incomingDataSetFile)
{
DistinctExceptionsCollection errors = new DistinctExceptionsCollection();
@@ -661,7 +662,7 @@ public abstract class AbstractProgrammableTopLevelDataSetHandler<T extends DataS
protected abstract boolean shouldUseOldJythonHookFunctions();
- protected abstract IJavaDataSetRegistrationDropboxV2<T> getV2DropboxProgram(
+ protected abstract IJavaDataSetRegistrationDropboxV2 getV2DropboxProgram(
DataSetRegistrationService<T> service);
protected abstract IJavaDataSetRegistrationDropboxV1<T> getV1DropboxProgram();
@@ -676,7 +677,7 @@ public abstract class AbstractProgrammableTopLevelDataSetHandler<T extends DataS
*/
protected abstract class RecoveryHookAdaptor implements IPrePostRegistrationHook<T>
{
- protected abstract IJavaDataSetRegistrationDropboxV2<T> getV2DropboxProgramInternal();
+ protected abstract IJavaDataSetRegistrationDropboxV2 getV2DropboxProgramInternal();
protected final File incoming;
@@ -735,4 +736,15 @@ public abstract class AbstractProgrammableTopLevelDataSetHandler<T extends DataS
}
}
}
+
+ @Override
+ protected Throwable asSerializableException(Throwable throwable)
+ {
+ if (throwable instanceof PyException)
+ {
+ return new RuntimeException(throwable.toString());
+ }
+
+ return super.asSerializableException(throwable);
+ }
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetHandler.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetHandler.java
index 00305a4b5a3da014a09da4a87b75a509f8c36c5f..a3c4ed6facf5cdf7ab898f2b20f826bbe81c1ef8 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetHandler.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetHandler.java
@@ -21,7 +21,6 @@ import java.util.List;
import org.python.core.Py;
import org.python.core.PyBaseCode;
-import org.python.core.PyException;
import org.python.core.PyFunction;
import org.python.core.PyObject;
@@ -359,41 +358,6 @@ public class JythonTopLevelDataSetHandler<T extends DataSetInformation> extends
super.rollback(service, throwable);
}
- // getters for v2 hook functions required for auto-recovery
- public PyFunction tryGetPostRegistrationFunction(DataSetRegistrationService<T> service)
- {
- PythonInterpreter interpreter = getInterpreterFromService(service);
- PyFunction function =
- tryJythonFunction(interpreter, JythonHookFunction.POST_REGISTRATION_FUNCTION_NAME);
- return function;
- }
-
- public PyFunction tryGetPostStorageFunction(DataSetRegistrationService<T> service)
- {
- PythonInterpreter interpreter = getInterpreterFromService(service);
- PyFunction function =
- tryJythonFunction(interpreter, JythonHookFunction.POST_STORAGE_FUNCTION_NAME);
- return function;
- }
-
- public PyFunction getRollbackPreRegistrationFunction(DataSetRegistrationService<T> service)
- {
- PythonInterpreter interpreter = getInterpreterFromService(service);
- PyFunction function =
- tryJythonFunction(interpreter,
- JythonHookFunction.ROLLBACK_PRE_REGISTRATION_FUNCTION_NAME);
- return function;
- }
-
- public PyFunction getShouldRetryProcessFunction(DataSetRegistrationService<T> service)
- {
- PythonInterpreter interpreter = getInterpreterFromService(service);
- PyFunction function =
- tryJythonFunction(interpreter,
- JythonHookFunction.SHOULD_RETRY_PROCESS_FUNCTION_NAME);
- return function;
- }
-
/**
* If true than the old methods of jython hook functions will also be used (as a fallbacks in
* case of the new methods or missing, or normally)
@@ -532,17 +496,6 @@ public class JythonTopLevelDataSetHandler<T extends DataSetInformation> extends
return interpreter;
}
- @Override
- protected Throwable asSerializableException(Throwable throwable)
- {
- if (throwable instanceof PyException)
- {
- return new RuntimeException(throwable.toString());
- }
-
- return super.asSerializableException(throwable);
- }
-
/**
* V1 registration framework -- any file can go into faulty paths.
*/
@@ -553,10 +506,10 @@ public class JythonTopLevelDataSetHandler<T extends DataSetInformation> extends
}
@Override
- protected IJavaDataSetRegistrationDropboxV2<T> getV2DropboxProgram(
+ protected IJavaDataSetRegistrationDropboxV2 getV2DropboxProgram(
DataSetRegistrationService<T> service)
{
- return new JythonAsJavaDataSetRegistrationDropboxV2Wrapper<T>(
+ return new JythonAsJavaDataSetRegistrationDropboxV2Wrapper(
getInterpreterFromService(service));
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/IJavaDataSetRegistrationDropboxV2.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/IJavaDataSetRegistrationDropboxV2.java
index 9c8174f20a4b5fa29c36fd1c439f3ef28e5a83b2..88b6738c23057ca4d7f50521c6571f0aca90cdd5 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/IJavaDataSetRegistrationDropboxV2.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/IJavaDataSetRegistrationDropboxV2.java
@@ -18,12 +18,11 @@ package ch.systemsx.cisd.etlserver.registrator.api.v2;
import ch.systemsx.cisd.common.exceptions.NotImplementedException;
import ch.systemsx.cisd.etlserver.registrator.DataSetRegistrationContext;
-import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
/**
* @author Pawel Glyzewski
*/
-public interface IJavaDataSetRegistrationDropboxV2<T extends DataSetInformation>
+public interface IJavaDataSetRegistrationDropboxV2
{
public void process(IDataSetRegistrationTransactionV2 transaction);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JavaTopLevelDataSetHandlerV2.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JavaTopLevelDataSetHandlerV2.java
index e0d88e5f41883287db706215aebd324c87f66816..f6a789b19b44b9b30716a707368ba4912f1f85a9 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JavaTopLevelDataSetHandlerV2.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JavaTopLevelDataSetHandlerV2.java
@@ -21,7 +21,6 @@ import java.io.File;
import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
import ch.systemsx.cisd.common.exceptions.NotImplementedException;
-import ch.systemsx.cisd.common.interpreter.PythonInterpreter;
import ch.systemsx.cisd.common.utilities.IDelegatedActionWithResult;
import ch.systemsx.cisd.common.utilities.PropertyUtils;
import ch.systemsx.cisd.etlserver.ITopLevelDataSetRegistratorDelegate;
@@ -42,7 +41,7 @@ public class JavaTopLevelDataSetHandlerV2<T extends DataSetInformation> extends
// The key for the script in the properties file
public static final String PROGRAM_CLASS_KEY = "program-class";
- private Class<? extends IJavaDataSetRegistrationDropboxV2<T>> programClass;
+ private Class<? extends IJavaDataSetRegistrationDropboxV2> programClass;
/**
* @param globalState
@@ -58,8 +57,7 @@ public class JavaTopLevelDataSetHandlerV2<T extends DataSetInformation> extends
try
{
programClass =
- (Class<? extends IJavaDataSetRegistrationDropboxV2<T>>) Class
- .forName(className);
+ (Class<? extends IJavaDataSetRegistrationDropboxV2>) Class.forName(className);
} catch (ClassNotFoundException ex)
{
throw ConfigurationFailureException.fromTemplate("Class '%s' does not exist!",
@@ -85,7 +83,9 @@ public class JavaTopLevelDataSetHandlerV2<T extends DataSetInformation> extends
protected void handleDataSet(DataSetFile dataSetFile, DataSetRegistrationService<T> service)
throws Throwable
{
- IJavaDataSetRegistrationDropboxV2<T> v2Programm = getV2DropboxProgram(service);
+ waitUntilApplicationIsReady(dataSetFile);
+
+ IJavaDataSetRegistrationDropboxV2 v2Programm = getV2DropboxProgram(service);
if (v2Programm.isRetryFunctionDefined())
{
@@ -104,10 +104,10 @@ public class JavaTopLevelDataSetHandlerV2<T extends DataSetInformation> extends
{
return new RecoveryHookAdaptor(incoming)
{
- IJavaDataSetRegistrationDropboxV2<T> v2ProgramInternal;
+ IJavaDataSetRegistrationDropboxV2 v2ProgramInternal;
@Override
- protected IJavaDataSetRegistrationDropboxV2<T> getV2DropboxProgramInternal()
+ protected IJavaDataSetRegistrationDropboxV2 getV2DropboxProgramInternal()
{
if (v2ProgramInternal == null)
{
@@ -121,16 +121,14 @@ public class JavaTopLevelDataSetHandlerV2<T extends DataSetInformation> extends
}
/**
- * Create a Jython registration service that includes access to the interpreter.
- *
- * @param pythonInterpreter
+ * Create a V2 registration service.
*/
- protected DataSetRegistrationService<T> createJythonDataSetRegistrationServiceV2(
+ @Override
+ protected DataSetRegistrationService<T> createDataSetRegistrationService(
DataSetFile incomingDataSetFile,
DataSetInformation userProvidedDataSetInformationOrNull,
IDelegatedActionWithResult<Boolean> cleanAfterwardsAction,
- ITopLevelDataSetRegistratorDelegate delegate, PythonInterpreter pythonInterpreter,
- TopLevelDataSetRegistratorGlobalState globalState)
+ ITopLevelDataSetRegistratorDelegate delegate)
{
return new DataSetRegistrationServiceV2<T>(this, incomingDataSetFile,
userProvidedDataSetInformationOrNull, cleanAfterwardsAction, delegate);
@@ -143,7 +141,14 @@ public class JavaTopLevelDataSetHandlerV2<T extends DataSetInformation> extends
}
@Override
- protected IJavaDataSetRegistrationDropboxV2<T> getV2DropboxProgram(
+ protected boolean hasRecoveryMarkerFile(File incoming)
+ {
+ return getGlobalState().getStorageRecoveryManager().getProcessingMarkerFile(incoming)
+ .exists();
+ }
+
+ @Override
+ protected IJavaDataSetRegistrationDropboxV2 getV2DropboxProgram(
DataSetRegistrationService<T> service)
{
try
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JythonAsJavaDataSetRegistrationDropboxV2Wrapper.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JythonAsJavaDataSetRegistrationDropboxV2Wrapper.java
index 987f644647a6ef2a8cdc25ddcb27942e0d935db4..55988f477e0ef537e82bb73524c3c2c6d57c1830 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JythonAsJavaDataSetRegistrationDropboxV2Wrapper.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JythonAsJavaDataSetRegistrationDropboxV2Wrapper.java
@@ -29,13 +29,12 @@ import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.utilities.JythonUtils;
import ch.systemsx.cisd.etlserver.registrator.DataSetRegistrationContext;
import ch.systemsx.cisd.etlserver.registrator.JythonTopLevelDataSetHandler.JythonHookFunction;
-import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
/**
* @author Pawel Glyzewski
*/
-public class JythonAsJavaDataSetRegistrationDropboxV2Wrapper<T extends DataSetInformation>
- implements IJavaDataSetRegistrationDropboxV2<T>
+public class JythonAsJavaDataSetRegistrationDropboxV2Wrapper implements
+ IJavaDataSetRegistrationDropboxV2
{
protected static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
JythonAsJavaDataSetRegistrationDropboxV2Wrapper.class);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JythonTopLevelDataSetHandlerV2.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JythonTopLevelDataSetHandlerV2.java
index 19387ce972ddea307818b49d98b825a442dc4ce0..bad4ef963045355892ed6fd07c4486e4e6928e2e 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JythonTopLevelDataSetHandlerV2.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JythonTopLevelDataSetHandlerV2.java
@@ -98,8 +98,8 @@ public class JythonTopLevelDataSetHandlerV2<T extends DataSetInformation> extend
// Configure the evaluator
PythonInterpreter interpreter = service.getInterpreter();
- IJavaDataSetRegistrationDropboxV2<T> v2Programm =
- new JythonAsJavaDataSetRegistrationDropboxV2Wrapper<T>(interpreter);
+ IJavaDataSetRegistrationDropboxV2 v2Programm =
+ new JythonAsJavaDataSetRegistrationDropboxV2Wrapper(interpreter);
// Invoke the evaluator
interpreter.exec(scriptString);
@@ -143,10 +143,10 @@ public class JythonTopLevelDataSetHandlerV2<T extends DataSetInformation> extend
}
@Override
- protected IJavaDataSetRegistrationDropboxV2<T> getV2DropboxProgram(
+ protected IJavaDataSetRegistrationDropboxV2 getV2DropboxProgram(
DataSetRegistrationService<T> service)
{
- return new JythonAsJavaDataSetRegistrationDropboxV2Wrapper<T>(
+ return new JythonAsJavaDataSetRegistrationDropboxV2Wrapper(
getInterpreterFromService(service));
}
@@ -156,10 +156,10 @@ public class JythonTopLevelDataSetHandlerV2<T extends DataSetInformation> extend
{
return new RecoveryHookAdaptor(incoming)
{
- IJavaDataSetRegistrationDropboxV2<T> v2ProgramInternal;
+ IJavaDataSetRegistrationDropboxV2 v2ProgramInternal;
@Override
- protected IJavaDataSetRegistrationDropboxV2<T> getV2DropboxProgramInternal()
+ protected IJavaDataSetRegistrationDropboxV2 getV2DropboxProgramInternal()
{
if (v2ProgramInternal == null)
{
@@ -178,7 +178,7 @@ public class JythonTopLevelDataSetHandlerV2<T extends DataSetInformation> extend
verifyEvaluatorHookFunctions(internalInterpreter);
v2ProgramInternal =
- new JythonAsJavaDataSetRegistrationDropboxV2Wrapper<T>(
+ new JythonAsJavaDataSetRegistrationDropboxV2Wrapper(
internalInterpreter);
}
return v2ProgramInternal;