LMS-2794: Add new methods to jython dropboxes. post_metadata_registration,...

LMS-2794: Add new methods to jython dropboxes. post_metadata_registration, pre_metadata_registration.
Also add method post_storage that should replace the commit_transaction

SVN: 24552

datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/AbstractOmniscientTopLevelDataSetRegistrator.java

@@ -592,6 +592,25 @@ public abstract class AbstractOmniscientTopLevelDataSetRegistrator<T extends Dat
     {
     }
 
+    /**
+     * A method called just before the registration of datasets in application server.
+     * <p>
+     * Subclasses can override and implement their own handling logic.
+     */
+    public void didPreRegistration(DataSetRegistrationService<T> service, DataSetRegistrationTransaction<T> transaction)
+    {
+    }
+    
+    /**
+     * A method called just after the successful registration of datasets in application server.
+     * <p>
+     * Subclasses can override and implement their own handling logic.
+     */
+    public void didPostRegistration(DataSetRegistrationService<T> service, DataSetRegistrationTransaction<T> transaction)
+    {
+    }
+    
+    
     /**
      * A method called when there is an error in one of the secondary transactions.
      * <p>

datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetRegistrationService.java

@@ -26,8 +26,6 @@ import ch.systemsx.cisd.base.exceptions.InterruptedExceptionUnchecked;
 import ch.systemsx.cisd.common.logging.LogCategory;
 import ch.systemsx.cisd.common.logging.LogFactory;
 import ch.systemsx.cisd.common.utilities.IDelegatedActionWithResult;
-import ch.systemsx.cisd.etlserver.DataSetRegistrationAlgorithm;
-import ch.systemsx.cisd.etlserver.DataSetRegistrationAlgorithmRunner;
 import ch.systemsx.cisd.etlserver.DssRegistrationLogDirectoryHelper;
 import ch.systemsx.cisd.etlserver.DssRegistrationLogger;
 import ch.systemsx.cisd.etlserver.IDataStoreStrategy;
@@ -49,7 +47,7 @@ import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
  * @author Chandrasekhar Ramakrishnan
  */
 public class DataSetRegistrationService<T extends DataSetInformation> implements
-        IDataSetRegistrationService
+        IDataSetRegistrationService, DataSetStorageAlgorithmRunner.IPrePostRegistrationHook<T>
 {
     private final AbstractOmniscientTopLevelDataSetRegistrator<T> registrator;
 
@@ -274,6 +272,17 @@ public class DataSetRegistrationService<T extends DataSetInformation> implements
         registrator.didCommitTransaction(this, transaction);
     }
 
+    public void executePreRegistration(DataSetRegistrationTransaction<T> transaction)
+    {
+        registrator.didPreRegistration(this, transaction);
+    }
+
+    public void executePostRegistration(DataSetRegistrationTransaction<T> transaction)
+    {
+        registrator.didPostRegistration(this, transaction);
+    }
+    
+    
     public void didEncounterSecondaryTransactionErrors(
             DataSetRegistrationTransaction<T> transaction,
             List<SecondaryTransactionFailure> secondaryErrors)

datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageAlgorithmRunner.java

@@ -25,9 +25,11 @@ import org.apache.log4j.Logger;
 
 import ch.systemsx.cisd.common.logging.LogCategory;
 import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.utilities.IDelegatedAction;
 import ch.systemsx.cisd.etlserver.DssRegistrationLogger;
 import ch.systemsx.cisd.etlserver.IStorageProcessorTransactional.IStorageProcessorTransaction;
 import ch.systemsx.cisd.etlserver.registrator.IDataSetOnErrorActionDecision.ErrorType;
+import ch.systemsx.cisd.etlserver.registrator.api.v1.impl.DataSetRegistrationTransaction;
 import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
 import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
 import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetRegistrationInformation;
@@ -63,6 +65,12 @@ public class DataSetStorageAlgorithmRunner<T extends DataSetInformation>
                 throws Throwable;
     }
 
+    public static interface IPrePostRegistrationHook<T extends DataSetInformation>
+    {
+        public void executePreRegistration(DataSetRegistrationTransaction<T> transaction);
+        public void executePostRegistration(DataSetRegistrationTransaction<T> transaction);
+    }
+    
     static private final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
             DataSetStorageAlgorithmRunner.class);
 
@@ -81,23 +89,29 @@ public class DataSetStorageAlgorithmRunner<T extends DataSetInformation>
 
     private final IRollbackStack rollbackStack;
 
+    private final DataSetRegistrationTransaction<T> transaction;
+    
+    private final IPrePostRegistrationHook<T> postPreRegistrationHooks;
+    
     private final DssRegistrationLogger dssRegistrationLog;
 
     private final IEncapsulatedOpenBISService openBISService;
 
     public DataSetStorageAlgorithmRunner(List<DataSetStorageAlgorithm<T>> dataSetStorageAlgorithms,
-            IRollbackDelegate<T> rollbackDelegate,
-            IDataSetInApplicationServerRegistrator<T> applicationServerRegistrator,
+            DataSetRegistrationTransaction<T> transaction,
             IRollbackStack rollbackStack, DssRegistrationLogger dssRegistrationLog,
-            IEncapsulatedOpenBISService openBISService)
+            IEncapsulatedOpenBISService openBISService,
+            IPrePostRegistrationHook<T> postPreRegistrationHooks)
     {
         this.dataSetStorageAlgorithms =
                 new ArrayList<DataSetStorageAlgorithm<T>>(dataSetStorageAlgorithms);
-        this.rollbackDelegate = rollbackDelegate;
-        this.applicationServerRegistrator = applicationServerRegistrator;
+        this.rollbackDelegate = transaction;
+        this.applicationServerRegistrator = transaction;
+        this.transaction = transaction;
         this.rollbackStack = rollbackStack;
         this.dssRegistrationLog = dssRegistrationLog;
         this.openBISService = openBISService;
+        this.postPreRegistrationHooks = postPreRegistrationHooks;
     }
 
     /**
@@ -193,10 +207,12 @@ public class DataSetStorageAlgorithmRunner<T extends DataSetInformation>
 
         try
         {
+            postPreRegistrationHooks.executePreRegistration(transaction);
+            
             // registers data set with yet non-existing store path.
-
             // Runs or throw a throwable
             registerDataSetsInApplicationServer();
+            
         } catch (final Throwable throwable)
         {
             rollbackDuringMetadataRegistration(throwable);
@@ -205,6 +221,15 @@ public class DataSetStorageAlgorithmRunner<T extends DataSetInformation>
 
         dssRegistrationLog.log("Data has been registered with the openBIS Application Server.");
 
+        try {
+            postPreRegistrationHooks.executePostRegistration(transaction);
+        }
+        catch (final Throwable throwable)
+        {
+            //TODO: log here
+            //maybe log but in general ignore
+        }
+        
         try
         {
             // Should always succeed

datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetHandler.java

@@ -58,6 +58,22 @@ public class JythonTopLevelDataSetHandler<T extends DataSetInformation> extends
      */
     private static final String COMMIT_TRANSACTION_FUNCTION_NAME = "commit_transaction";
 
+    /**
+     * The name of the function called after successful transaction commit.
+     */
+    private static final String POST_STORAGE_FUNCTION_NAME = "post_storage";
+
+    /**
+     * The name of the function called just before registration of datasets in application server.
+     */
+    private static final String PRE_REGISTRATION_FUNCTION_NAME = "pre_metadata_registration";
+
+    /**
+     * The name of the function called just after successful registration of datasets in application
+     * server.
+     */
+    private static final String POST_REGISTRATION_FUNCTION_NAME = "post_metadata_registration";
+
     /**
      * The name of the function called when secondary transactions, DynamicTransactionQuery objects,
      * fail.
@@ -151,7 +167,8 @@ public class JythonTopLevelDataSetHandler<T extends DataSetInformation> extends
      * @param pythonInterpreter
      */
     protected DataSetRegistrationService<T> createJythonDataSetRegistrationService(
-            DataSetFile incomingDataSetFile, DataSetInformation userProvidedDataSetInformationOrNull,
+            DataSetFile incomingDataSetFile,
+            DataSetInformation userProvidedDataSetInformationOrNull,
             IDelegatedActionWithResult<Boolean> cleanAfterwardsAction,
             ITopLevelDataSetRegistratorDelegate delegate, PythonInterpreter pythonInterpreter,
             TopLevelDataSetRegistratorGlobalState globalState)
@@ -191,6 +208,22 @@ public class JythonTopLevelDataSetHandler<T extends DataSetInformation> extends
         invokeCommitTransactionFunction(service, transaction);
     }
 
+    @Override
+    public void didPreRegistration(DataSetRegistrationService<T> service,
+            DataSetRegistrationTransaction<T> transaction)
+    {
+        super.didPreRegistration(service, transaction);
+        invokePreRegistrationFunction(service, transaction);
+    }
+
+    @Override
+    public void didPostRegistration(DataSetRegistrationService<T> service,
+            DataSetRegistrationTransaction<T> transaction)
+    {
+        super.didPostRegistration(service, transaction);
+        invokePostRegistrationFunction(service, transaction);
+    }
+
     @Override
     public void didEncounterSecondaryTransactionErrors(DataSetRegistrationService<T> service,
             DataSetRegistrationTransaction<T> transaction,
@@ -226,10 +259,40 @@ public class JythonTopLevelDataSetHandler<T extends DataSetInformation> extends
             DataSetRegistrationTransaction<T> transaction)
     {
         PythonInterpreter interpreter = getInterpreterFromService(service);
-        PyFunction function = tryJythonFunction(interpreter, COMMIT_TRANSACTION_FUNCTION_NAME);
+
+        PyFunction function = tryJythonFunction(interpreter, POST_STORAGE_FUNCTION_NAME);
         if (null != function)
         {
-            invokeCommitTransactionFunction(function, service, transaction);
+            invokeTransactionFunctionWithContext(function, transaction);
+        } else
+        {
+            function = tryJythonFunction(interpreter, COMMIT_TRANSACTION_FUNCTION_NAME);
+            if (null != function)
+            {
+                invokeServiceTransactionFunction(function, service, transaction);
+            }
+        }
+    }
+
+    private void invokePreRegistrationFunction(DataSetRegistrationService<T> service,
+            DataSetRegistrationTransaction<T> transaction)
+    {
+        PythonInterpreter interpreter = getInterpreterFromService(service);
+        PyFunction function = tryJythonFunction(interpreter, PRE_REGISTRATION_FUNCTION_NAME);
+        if (null != function)
+        {
+            invokeTransactionFunctionWithContext(function, transaction);
+        }
+    }
+
+    private void invokePostRegistrationFunction(DataSetRegistrationService<T> service,
+            DataSetRegistrationTransaction<T> transaction)
+    {
+        PythonInterpreter interpreter = getInterpreterFromService(service);
+        PyFunction function = tryJythonFunction(interpreter, POST_REGISTRATION_FUNCTION_NAME);
+        if (null != function)
+        {
+            invokeTransactionFunctionWithContext(function, transaction);
         }
     }
 
@@ -295,12 +358,21 @@ public class JythonTopLevelDataSetHandler<T extends DataSetInformation> extends
     /**
      * Pulled out as a separate method so tests can hook in.
      */
-    protected void invokeCommitTransactionFunction(PyFunction function,
+    protected void invokeServiceTransactionFunction(PyFunction function,
             DataSetRegistrationService<T> service, DataSetRegistrationTransaction<T> transaction)
     {
         function.__call__(Py.java2py(service), Py.java2py(transaction));
     }
 
+    /**
+     * Pulled out as a separate method so tests can hook in.
+     */
+    protected void invokeTransactionFunctionWithContext(PyFunction function,
+            DataSetRegistrationTransaction<T> transaction)
+    {
+        function.__call__(Py.java2py(transaction), Py.java2py(null));
+    }
+
     /**
      * Pulled out as a separate method so tests can hook in.
      */
@@ -367,7 +439,8 @@ public class JythonTopLevelDataSetHandler<T extends DataSetInformation> extends
         private final PythonInterpreter interpreter;
 
         public JythonDataSetRegistrationService(JythonTopLevelDataSetHandler<T> registrator,
-                DataSetFile incomingDataSetFile, DataSetInformation userProvidedDataSetInformationOrNull,
+                DataSetFile incomingDataSetFile,
+                DataSetInformation userProvidedDataSetInformationOrNull,
                 IDelegatedActionWithResult<Boolean> globalCleanAfterwardsAction,
                 ITopLevelDataSetRegistratorDelegate delegate, PythonInterpreter interpreter,
                 TopLevelDataSetRegistratorGlobalState globalState)

datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v1/impl/AbstractTransactionState.java

@@ -627,7 +627,7 @@ abstract class AbstractTransactionState<T extends DataSetInformation>
             }
 
             DataSetStorageAlgorithmRunner<T> runner =
-                    new DataSetStorageAlgorithmRunner<T>(algorithms, parent, parent, rollbackStack, registrationService.getDssRegistrationLog(), openBisService);
+                    new DataSetStorageAlgorithmRunner<T>(algorithms, parent, rollbackStack, registrationService.getDssRegistrationLog(), openBisService, registrationService);
            
             boolean someDataSetsRegistered = runner.prepareAndRunStorageAlgorithms();
 

datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/AbstractJythonDataSetHandlerTest.java

@@ -221,11 +221,13 @@ public abstract class AbstractJythonDataSetHandlerTest extends AbstractFileSyste
         return createThreadProperties(getRegistrationScriptsFolderPath() + scriptPath, null, null);
     }
 
-    protected Properties createThreadPropertiesRelativeToScriptsFolder(String scriptPath, HashMap<String, String> override)
+    protected Properties createThreadPropertiesRelativeToScriptsFolder(String scriptPath,
+            HashMap<String, String> override)
     {
-        return createThreadProperties(getRegistrationScriptsFolderPath() + scriptPath, null, override);
+        return createThreadProperties(getRegistrationScriptsFolderPath() + scriptPath, null,
+                override);
     }
-    
+
     protected Properties createThreadPropertiesRelativeToScriptsFolder(String scriptPath,
             String validationScriptPath)
     {
@@ -246,7 +248,8 @@ public abstract class AbstractJythonDataSetHandlerTest extends AbstractFileSyste
         threadProperties.setProperty(JythonTopLevelDataSetHandler.SCRIPT_PATH_KEY, scriptPath);
         if (null != validationScriptPropertyOrNull)
         {
-            threadProperties.setProperty(ch.systemsx.cisd.etlserver.ThreadParameters.VALIDATION_SCRIPT_KEY,
+            threadProperties.setProperty(
+                    ch.systemsx.cisd.etlserver.ThreadParameters.VALIDATION_SCRIPT_KEY,
                     validationScriptPropertyOrNull);
         }
         threadProperties.setProperty(TopLevelDataSetRegistratorGlobalState.STAGING_DIR,
@@ -258,7 +261,7 @@ public abstract class AbstractJythonDataSetHandlerTest extends AbstractFileSyste
 
         if (overrideOrNull != null)
         {
-            for (String key: overrideOrNull.keySet())
+            for (String key : overrideOrNull.keySet())
             {
                 threadProperties.setProperty(key, overrideOrNull.get(key));
             }
@@ -492,11 +495,11 @@ public abstract class AbstractJythonDataSetHandlerTest extends AbstractFileSyste
         }
 
         @Override
-        protected void invokeCommitTransactionFunction(PyFunction function,
+        protected void invokeServiceTransactionFunction(PyFunction function,
                 DataSetRegistrationService<DataSetInformation> service,
                 DataSetRegistrationTransaction<DataSetInformation> transaction)
         {
-            super.invokeCommitTransactionFunction(function, service, transaction);
+            super.invokeServiceTransactionFunction(function, service, transaction);
 
             PythonInterpreter interpreter =
                     ((JythonDataSetRegistrationService<DataSetInformation>) service)
@@ -543,8 +546,8 @@ public abstract class AbstractJythonDataSetHandlerTest extends AbstractFileSyste
         private final boolean shouldRegistrationFail;
 
         public TestDataRegistrationService(
-                JythonTopLevelDataSetHandler<DataSetInformation> registrator, DataSetFile aDataSetFile,
-                DataSetInformation userProvidedDataSetInformationOrNull,
+                JythonTopLevelDataSetHandler<DataSetInformation> registrator,
+                DataSetFile aDataSetFile, DataSetInformation userProvidedDataSetInformationOrNull,
                 IDelegatedActionWithResult<Boolean> globalCleanAfterwardsAction,
                 PythonInterpreter interpreter, boolean shouldRegistrationFail,
                 TopLevelDataSetRegistratorGlobalState globalState)