From 165f77c06592ab3d109e0cb2d87336d41c3e7aa0 Mon Sep 17 00:00:00 2001
From: gpawel <gpawel>
Date: Wed, 18 Jul 2012 08:38:07 +0000
Subject: [PATCH] SP-180 BIS-21 : Support pure Java V2 Dropboxes
SVN: 26141
---
...actProgrammableTopLevelDataSetHandler.java | 20 ++++++--
.../JythonTopLevelDataSetHandler.java | 51 +------------------
.../v2/IJavaDataSetRegistrationDropboxV2.java | 3 +-
.../api/v2/JavaTopLevelDataSetHandlerV2.java | 33 +++++++-----
...vaDataSetRegistrationDropboxV2Wrapper.java | 5 +-
.../v2/JythonTopLevelDataSetHandlerV2.java | 14 ++---
6 files changed, 47 insertions(+), 79 deletions(-)
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/AbstractProgrammableTopLevelDataSetHandler.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/AbstractProgrammableTopLevelDataSetHandler.java
index 5f56e20965f..276accefbc8 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/AbstractProgrammableTopLevelDataSetHandler.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/AbstractProgrammableTopLevelDataSetHandler.java
@@ -22,6 +22,8 @@ import java.util.Calendar;
import java.util.Date;
import java.util.List;
+import org.python.core.PyException;
+
import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
import ch.systemsx.cisd.common.concurrent.ConcurrencyUtilities;
import ch.systemsx.cisd.common.exceptions.NotImplementedException;
@@ -83,8 +85,7 @@ public abstract class AbstractProgrammableTopLevelDataSetHandler<T extends DataS
abstract protected void handleDataSet(DataSetFile dataSetFile,
DataSetRegistrationService<T> service) throws Throwable;
- protected void executeProcessFunctionWithRetries(
- IJavaDataSetRegistrationDropboxV2<T> v2Programm,
+ protected void executeProcessFunctionWithRetries(IJavaDataSetRegistrationDropboxV2 v2Programm,
JythonDataSetRegistrationServiceV2<T> service, DataSetFile incomingDataSetFile)
{
DistinctExceptionsCollection errors = new DistinctExceptionsCollection();
@@ -661,7 +662,7 @@ public abstract class AbstractProgrammableTopLevelDataSetHandler<T extends DataS
protected abstract boolean shouldUseOldJythonHookFunctions();
- protected abstract IJavaDataSetRegistrationDropboxV2<T> getV2DropboxProgram(
+ protected abstract IJavaDataSetRegistrationDropboxV2 getV2DropboxProgram(
DataSetRegistrationService<T> service);
protected abstract IJavaDataSetRegistrationDropboxV1<T> getV1DropboxProgram();
@@ -676,7 +677,7 @@ public abstract class AbstractProgrammableTopLevelDataSetHandler<T extends DataS
*/
protected abstract class RecoveryHookAdaptor implements IPrePostRegistrationHook<T>
{
- protected abstract IJavaDataSetRegistrationDropboxV2<T> getV2DropboxProgramInternal();
+ protected abstract IJavaDataSetRegistrationDropboxV2 getV2DropboxProgramInternal();
protected final File incoming;
@@ -735,4 +736,15 @@ public abstract class AbstractProgrammableTopLevelDataSetHandler<T extends DataS
}
}
}
+
+ @Override
+ protected Throwable asSerializableException(Throwable throwable)
+ {
+ if (throwable instanceof PyException)
+ {
+ return new RuntimeException(throwable.toString());
+ }
+
+ return super.asSerializableException(throwable);
+ }
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetHandler.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetHandler.java
index 00305a4b5a3..a3c4ed6facf 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetHandler.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetHandler.java
@@ -21,7 +21,6 @@ import java.util.List;
import org.python.core.Py;
import org.python.core.PyBaseCode;
-import org.python.core.PyException;
import org.python.core.PyFunction;
import org.python.core.PyObject;
@@ -359,41 +358,6 @@ public class JythonTopLevelDataSetHandler<T extends DataSetInformation> extends
super.rollback(service, throwable);
}
- // getters for v2 hook functions required for auto-recovery
- public PyFunction tryGetPostRegistrationFunction(DataSetRegistrationService<T> service)
- {
- PythonInterpreter interpreter = getInterpreterFromService(service);
- PyFunction function =
- tryJythonFunction(interpreter, JythonHookFunction.POST_REGISTRATION_FUNCTION_NAME);
- return function;
- }
-
- public PyFunction tryGetPostStorageFunction(DataSetRegistrationService<T> service)
- {
- PythonInterpreter interpreter = getInterpreterFromService(service);
- PyFunction function =
- tryJythonFunction(interpreter, JythonHookFunction.POST_STORAGE_FUNCTION_NAME);
- return function;
- }
-
- public PyFunction getRollbackPreRegistrationFunction(DataSetRegistrationService<T> service)
- {
- PythonInterpreter interpreter = getInterpreterFromService(service);
- PyFunction function =
- tryJythonFunction(interpreter,
- JythonHookFunction.ROLLBACK_PRE_REGISTRATION_FUNCTION_NAME);
- return function;
- }
-
- public PyFunction getShouldRetryProcessFunction(DataSetRegistrationService<T> service)
- {
- PythonInterpreter interpreter = getInterpreterFromService(service);
- PyFunction function =
- tryJythonFunction(interpreter,
- JythonHookFunction.SHOULD_RETRY_PROCESS_FUNCTION_NAME);
- return function;
- }
-
/**
* If true than the old methods of jython hook functions will also be used (as a fallbacks in
* case of the new methods or missing, or normally)
@@ -532,17 +496,6 @@ public class JythonTopLevelDataSetHandler<T extends DataSetInformation> extends
return interpreter;
}
- @Override
- protected Throwable asSerializableException(Throwable throwable)
- {
- if (throwable instanceof PyException)
- {
- return new RuntimeException(throwable.toString());
- }
-
- return super.asSerializableException(throwable);
- }
-
/**
* V1 registration framework -- any file can go into faulty paths.
*/
@@ -553,10 +506,10 @@ public class JythonTopLevelDataSetHandler<T extends DataSetInformation> extends
}
@Override
- protected IJavaDataSetRegistrationDropboxV2<T> getV2DropboxProgram(
+ protected IJavaDataSetRegistrationDropboxV2 getV2DropboxProgram(
DataSetRegistrationService<T> service)
{
- return new JythonAsJavaDataSetRegistrationDropboxV2Wrapper<T>(
+ return new JythonAsJavaDataSetRegistrationDropboxV2Wrapper(
getInterpreterFromService(service));
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/IJavaDataSetRegistrationDropboxV2.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/IJavaDataSetRegistrationDropboxV2.java
index 9c8174f20a4..88b6738c230 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/IJavaDataSetRegistrationDropboxV2.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/IJavaDataSetRegistrationDropboxV2.java
@@ -18,12 +18,11 @@ package ch.systemsx.cisd.etlserver.registrator.api.v2;
import ch.systemsx.cisd.common.exceptions.NotImplementedException;
import ch.systemsx.cisd.etlserver.registrator.DataSetRegistrationContext;
-import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
/**
* @author Pawel Glyzewski
*/
-public interface IJavaDataSetRegistrationDropboxV2<T extends DataSetInformation>
+public interface IJavaDataSetRegistrationDropboxV2
{
public void process(IDataSetRegistrationTransactionV2 transaction);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JavaTopLevelDataSetHandlerV2.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JavaTopLevelDataSetHandlerV2.java
index e0d88e5f418..f6a789b19b4 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JavaTopLevelDataSetHandlerV2.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JavaTopLevelDataSetHandlerV2.java
@@ -21,7 +21,6 @@ import java.io.File;
import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
import ch.systemsx.cisd.common.exceptions.NotImplementedException;
-import ch.systemsx.cisd.common.interpreter.PythonInterpreter;
import ch.systemsx.cisd.common.utilities.IDelegatedActionWithResult;
import ch.systemsx.cisd.common.utilities.PropertyUtils;
import ch.systemsx.cisd.etlserver.ITopLevelDataSetRegistratorDelegate;
@@ -42,7 +41,7 @@ public class JavaTopLevelDataSetHandlerV2<T extends DataSetInformation> extends
// The key for the script in the properties file
public static final String PROGRAM_CLASS_KEY = "program-class";
- private Class<? extends IJavaDataSetRegistrationDropboxV2<T>> programClass;
+ private Class<? extends IJavaDataSetRegistrationDropboxV2> programClass;
/**
* @param globalState
@@ -58,8 +57,7 @@ public class JavaTopLevelDataSetHandlerV2<T extends DataSetInformation> extends
try
{
programClass =
- (Class<? extends IJavaDataSetRegistrationDropboxV2<T>>) Class
- .forName(className);
+ (Class<? extends IJavaDataSetRegistrationDropboxV2>) Class.forName(className);
} catch (ClassNotFoundException ex)
{
throw ConfigurationFailureException.fromTemplate("Class '%s' does not exist!",
@@ -85,7 +83,9 @@ public class JavaTopLevelDataSetHandlerV2<T extends DataSetInformation> extends
protected void handleDataSet(DataSetFile dataSetFile, DataSetRegistrationService<T> service)
throws Throwable
{
- IJavaDataSetRegistrationDropboxV2<T> v2Programm = getV2DropboxProgram(service);
+ waitUntilApplicationIsReady(dataSetFile);
+
+ IJavaDataSetRegistrationDropboxV2 v2Programm = getV2DropboxProgram(service);
if (v2Programm.isRetryFunctionDefined())
{
@@ -104,10 +104,10 @@ public class JavaTopLevelDataSetHandlerV2<T extends DataSetInformation> extends
{
return new RecoveryHookAdaptor(incoming)
{
- IJavaDataSetRegistrationDropboxV2<T> v2ProgramInternal;
+ IJavaDataSetRegistrationDropboxV2 v2ProgramInternal;
@Override
- protected IJavaDataSetRegistrationDropboxV2<T> getV2DropboxProgramInternal()
+ protected IJavaDataSetRegistrationDropboxV2 getV2DropboxProgramInternal()
{
if (v2ProgramInternal == null)
{
@@ -121,16 +121,14 @@ public class JavaTopLevelDataSetHandlerV2<T extends DataSetInformation> extends
}
/**
- * Create a Jython registration service that includes access to the interpreter.
- *
- * @param pythonInterpreter
+ * Create a V2 registration service.
*/
- protected DataSetRegistrationService<T> createJythonDataSetRegistrationServiceV2(
+ @Override
+ protected DataSetRegistrationService<T> createDataSetRegistrationService(
DataSetFile incomingDataSetFile,
DataSetInformation userProvidedDataSetInformationOrNull,
IDelegatedActionWithResult<Boolean> cleanAfterwardsAction,
- ITopLevelDataSetRegistratorDelegate delegate, PythonInterpreter pythonInterpreter,
- TopLevelDataSetRegistratorGlobalState globalState)
+ ITopLevelDataSetRegistratorDelegate delegate)
{
return new DataSetRegistrationServiceV2<T>(this, incomingDataSetFile,
userProvidedDataSetInformationOrNull, cleanAfterwardsAction, delegate);
@@ -143,7 +141,14 @@ public class JavaTopLevelDataSetHandlerV2<T extends DataSetInformation> extends
}
@Override
- protected IJavaDataSetRegistrationDropboxV2<T> getV2DropboxProgram(
+ protected boolean hasRecoveryMarkerFile(File incoming)
+ {
+ return getGlobalState().getStorageRecoveryManager().getProcessingMarkerFile(incoming)
+ .exists();
+ }
+
+ @Override
+ protected IJavaDataSetRegistrationDropboxV2 getV2DropboxProgram(
DataSetRegistrationService<T> service)
{
try
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JythonAsJavaDataSetRegistrationDropboxV2Wrapper.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JythonAsJavaDataSetRegistrationDropboxV2Wrapper.java
index 987f644647a..55988f477e0 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JythonAsJavaDataSetRegistrationDropboxV2Wrapper.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JythonAsJavaDataSetRegistrationDropboxV2Wrapper.java
@@ -29,13 +29,12 @@ import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.utilities.JythonUtils;
import ch.systemsx.cisd.etlserver.registrator.DataSetRegistrationContext;
import ch.systemsx.cisd.etlserver.registrator.JythonTopLevelDataSetHandler.JythonHookFunction;
-import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
/**
* @author Pawel Glyzewski
*/
-public class JythonAsJavaDataSetRegistrationDropboxV2Wrapper<T extends DataSetInformation>
- implements IJavaDataSetRegistrationDropboxV2<T>
+public class JythonAsJavaDataSetRegistrationDropboxV2Wrapper implements
+ IJavaDataSetRegistrationDropboxV2
{
protected static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
JythonAsJavaDataSetRegistrationDropboxV2Wrapper.class);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JythonTopLevelDataSetHandlerV2.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JythonTopLevelDataSetHandlerV2.java
index 19387ce972d..bad4ef96304 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JythonTopLevelDataSetHandlerV2.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JythonTopLevelDataSetHandlerV2.java
@@ -98,8 +98,8 @@ public class JythonTopLevelDataSetHandlerV2<T extends DataSetInformation> extend
// Configure the evaluator
PythonInterpreter interpreter = service.getInterpreter();
- IJavaDataSetRegistrationDropboxV2<T> v2Programm =
- new JythonAsJavaDataSetRegistrationDropboxV2Wrapper<T>(interpreter);
+ IJavaDataSetRegistrationDropboxV2 v2Programm =
+ new JythonAsJavaDataSetRegistrationDropboxV2Wrapper(interpreter);
// Invoke the evaluator
interpreter.exec(scriptString);
@@ -143,10 +143,10 @@ public class JythonTopLevelDataSetHandlerV2<T extends DataSetInformation> extend
}
@Override
- protected IJavaDataSetRegistrationDropboxV2<T> getV2DropboxProgram(
+ protected IJavaDataSetRegistrationDropboxV2 getV2DropboxProgram(
DataSetRegistrationService<T> service)
{
- return new JythonAsJavaDataSetRegistrationDropboxV2Wrapper<T>(
+ return new JythonAsJavaDataSetRegistrationDropboxV2Wrapper(
getInterpreterFromService(service));
}
@@ -156,10 +156,10 @@ public class JythonTopLevelDataSetHandlerV2<T extends DataSetInformation> extend
{
return new RecoveryHookAdaptor(incoming)
{
- IJavaDataSetRegistrationDropboxV2<T> v2ProgramInternal;
+ IJavaDataSetRegistrationDropboxV2 v2ProgramInternal;
@Override
- protected IJavaDataSetRegistrationDropboxV2<T> getV2DropboxProgramInternal()
+ protected IJavaDataSetRegistrationDropboxV2 getV2DropboxProgramInternal()
{
if (v2ProgramInternal == null)
{
@@ -178,7 +178,7 @@ public class JythonTopLevelDataSetHandlerV2<T extends DataSetInformation> extend
verifyEvaluatorHookFunctions(internalInterpreter);
v2ProgramInternal =
- new JythonAsJavaDataSetRegistrationDropboxV2Wrapper<T>(
+ new JythonAsJavaDataSetRegistrationDropboxV2Wrapper(
internalInterpreter);
}
return v2ProgramInternal;
--
GitLab