From 3921b2bb76585dbdcca3f790a70a51bc153b2e3b Mon Sep 17 00:00:00 2001
From: jakubs <jakubs>
Date: Mon, 21 May 2012 13:56:47 +0000
Subject: [PATCH] SP-45, BIS-21 move the recovery code from omniscient
registrator to jython V2
SVN: 25318
---
...tOmniscientTopLevelDataSetRegistrator.java | 241 ++---------------
.../DataSetStorageAlgorithmRunner.java | 5 +
.../DataSetStoragePrecommitRecoveryState.java | 12 +-
.../DataSetStorageRecoveryManager.java | 3 +-
.../v2/JythonTopLevelDataSetHandlerV2.java | 255 ++++++++++++++++++
.../JythonDropboxRecoveryTest.java | 2 +-
6 files changed, 290 insertions(+), 228 deletions(-)
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/AbstractOmniscientTopLevelDataSetRegistrator.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/AbstractOmniscientTopLevelDataSetRegistrator.java
index 3043c7526f8..8e910210b50 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/AbstractOmniscientTopLevelDataSetRegistrator.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/AbstractOmniscientTopLevelDataSetRegistrator.java
@@ -33,6 +33,7 @@ import org.apache.log4j.Logger;
import ch.systemsx.cisd.base.exceptions.InterruptedExceptionUnchecked;
import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.common.exceptions.NotImplementedException;
import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.exceptions.UserFailureException;
import ch.systemsx.cisd.common.filesystem.FastRecursiveHardLinkMaker;
@@ -79,10 +80,10 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.NewExternalData;
public abstract class AbstractOmniscientTopLevelDataSetRegistrator<T extends DataSetInformation>
extends AbstractTopLevelDataSetRegistrator
{
- static final Logger notificationLog = LogFactory.getLogger(LogCategory.NOTIFY,
+ protected static final Logger notificationLog = LogFactory.getLogger(LogCategory.NOTIFY,
AbstractOmniscientTopLevelDataSetRegistrator.class);
- static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
+ protected static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
AbstractOmniscientTopLevelDataSetRegistrator.class);
/**
@@ -248,7 +249,7 @@ public abstract class AbstractOmniscientTopLevelDataSetRegistrator<T extends Dat
}
- private final OmniscientTopLevelDataSetRegistratorState state;
+ protected final OmniscientTopLevelDataSetRegistratorState state;
private boolean stopped;
@@ -305,8 +306,13 @@ public abstract class AbstractOmniscientTopLevelDataSetRegistrator<T extends Dat
protected boolean hasRecoveryMarkerFile(File incoming)
{
- return getGlobalState().getStorageRecoveryManager().getProcessingMarkerFile(incoming)
- .exists();
+ return false;
+ }
+
+ protected void handleRecovery(final File incomingFileOriginal)
+ {
+ throw new NotImplementedException(
+ "Recovery feature is not implemented for this kind of registrator");
}
/**
@@ -470,240 +476,25 @@ public abstract class AbstractOmniscientTopLevelDataSetRegistrator<T extends Dat
return new RuntimeException(cause.toString());
}
- private void handleRecovery(final File incomingFileOriginal)
- {
- // get the marker file
- final File recoveryMarkerFile =
- state.getGlobalState().getStorageRecoveryManager()
- .getProcessingMarkerFile(incomingFileOriginal);
-
- // deserialize recovery state
- final DataSetStoragePrecommitRecoveryState<T> recoveryState =
- state.getGlobalState().getStorageRecoveryManager()
- .extractPrecommittedCheckpoint(recoveryMarkerFile);
-
- // then we should ensure that the recovery will actually take place itself!
- final DataSetStorageRecoveryInfo recoveryInfo =
- state.getGlobalState().getStorageRecoveryManager()
- .getRecoveryFileFromMarker(recoveryMarkerFile);
-
- final File recoveryFile = recoveryInfo.getRecoveryStateFile();
-
- if (false == recoveryFile.exists())
- {
- // TODO: is it safe to throw from here?
- operationLog.error("recovery file does not exist. " + recoveryFile);
- throw new IllegalStateException("Recovery file " + recoveryFile + " doesn't exist");
- }
-
- if (false == retryPeriodHasPassed(recoveryInfo))
- {
- operationLog.info("Found recovery inforation for "+incomingFileOriginal+". The recovery won't happen as the retry period has not yet passed" );
- return;
- }
-
- operationLog.info("will recover from broken registration. Found marker file "
- + recoveryMarkerFile + " and " + recoveryFile);
-
- IDelegatedActionWithResult<Boolean> recoveryMarkerFileCleanupAction =
- new IDelegatedActionWithResult<Boolean>()
- {
- public Boolean execute(boolean didOperationSucceed)
- {
- if (!didOperationSucceed
- && recoveryInfo.getTryCount() >= state.getGlobalState()
- .getStorageRecoveryManager().getMaximumRertyCount())
- {
- notificationLog.error("The dataset "
- + recoveryState.getIncomingDataSetFile()
- .getRealIncomingFile()
- + " has failed to register. Giving up.");
- deleteMarkerFile(recoveryState);
-
- System.err.println("This is happening " + recoveryMarkerFile);
- File errorRecoveryMarkerFile =
- new File(recoveryMarkerFile.getParent(),
- recoveryMarkerFile.getName() + ".ERROR");
- state.fileOperations.move(recoveryMarkerFile,
- errorRecoveryMarkerFile);
- } else
- {
- if (didOperationSucceed)
- {
- deleteMarkerFile(recoveryState);
-
- recoveryMarkerFile.delete();
- recoveryFile.delete();
- } else
- {
- // this replaces the recovery file with a new one with increased
- // count
- // FIXME: is this safe operation (how to assure, that it won't
- // corrupt the recoveryMarkerFile?)
-
- recoveryInfo.increaseTryCount();
- recoveryInfo.setLastTry(new Date());
- recoveryInfo.writeToFile(recoveryMarkerFile);
- }
- }
- return true;
- }
-
- private void deleteMarkerFile(
- final DataSetStoragePrecommitRecoveryState recoveryState)
- {
- File incomingMarkerFile =
- MarkerFileUtility.getMarkerFileFromIncoming(recoveryState
- .getIncomingDataSetFile().getRealIncomingFile());
- if (incomingMarkerFile.exists())
- {
-
- incomingMarkerFile.delete();
- }
- }
- };
-
- PostRegistrationCleanUpAction cleanupAction =
- new PostRegistrationCleanUpAction(recoveryState.getIncomingDataSetFile(),
- new DoNothingDelegatedAction());
-
- handleRecoveryState(recoveryState, cleanupAction, recoveryMarkerFileCleanupAction);
- }
-
/**
- * Check wheter the last retry + retry period < date.now
+ * Create an adaptor that offers access to the recovery hook functions.
*/
- private boolean retryPeriodHasPassed(final DataSetStorageRecoveryInfo recoveryInfo)
+ protected IPrePostRegistrationHook<T> createRecoveryHookAdaptor()
{
- Calendar c = Calendar.getInstance();
- c.setTime(recoveryInfo.getLastTry());
- c.add(Calendar.SECOND, state.getGlobalState().getStorageRecoveryManager()
- .getRetryPeriodInSeconds());
- return c.getTime().before(new Date());
- }
-
- private void handleRecoveryState(DataSetStoragePrecommitRecoveryState<T> recoveryState,
- final IDelegatedActionWithResult<Boolean> cleanAfterwardsAction,
- final IDelegatedActionWithResult<Boolean> recoveryMarkerCleanup)
- {
- // TODO: Jobs left to do here:
- // jython hooks
-
- DssRegistrationLogger logger = recoveryState.getRegistrationLogger(state);
-
- logger.log("The registration has been disturbed. Will try to recover...");
-
- // rollback delegate
- final List<Throwable> encounteredErrors = new ArrayList<Throwable>();
-
- // keeps track of whether we should keep or delete the recovery files.
- // we can delete if succesfully recovered, or rolledback, or gave up
- boolean shouldDeleteRecoveryFiles = false;
-
- IRollbackDelegate<T> rollbackDelegate = new IRollbackDelegate<T>()
- {
- public void didRollbackStorageAlgorithmRunner(
- DataSetStorageAlgorithmRunner<T> algorithm, Throwable ex,
- ErrorType errorType)
- {
- encounteredErrors.add(ex);
- }
- };
-
- // hookAdaptor
final AbstractOmniscientTopLevelDataSetRegistrator<T> myself = this;
IPrePostRegistrationHook<T> hookAdaptor = new IPrePostRegistrationHook<T>()
{
public void executePreRegistration(DataSetRegistrationTransaction<T> transaction)
{
- // myself.didPreRegistration(null, transaction);
+ myself.didPreRegistration(null, transaction);
}
public void executePostRegistration(DataSetRegistrationTransaction<T> transaction)
{
- // myself.didPostRegistration(null, transaction);
+ myself.didPostRegistration(null, transaction);
}
};
-
- DataSetStorageAlgorithmRunner<T> runner =
- new DataSetStorageAlgorithmRunner<T>(recoveryState.getIncomingDataSetFile(), // incoming
- recoveryState.getDataSetStorageAlgorithms(state), // algorithms
- rollbackDelegate, // rollback delegate,
- recoveryState.getRollbackStack(), // rollbackstack
- logger, // registrationLogger
- state.getGlobalState().getOpenBisService(), // openBisService
- hookAdaptor, // the hooks
- state.getGlobalState().getStorageRecoveryManager());
- boolean registrationSuccessful = false;
-
- operationLog.info("Recovery succesfully deserialized the state of the registration");
- try
- {
- List<String> dataSetCodes = recoveryState.getDataSetCodes();
- // TODO: we need to check for something more than just a registered datasets, for the
- // registrations that doesn't register anything.
- // OTOH: do we need to recover in this case? (as there is nothing to store)
- // maybe then we should only checkpoint for recovery if there ARE datasets registered?
-
- // TODO: verify why if this throws exception - the empty directory is created in a
- // store?
- List<ExternalData> registeredDataSets =
- state.getGlobalState().getOpenBisService().listDataSetsByCode(dataSetCodes);
- if (registeredDataSets.isEmpty())
- {
- operationLog
- .info("Recovery hasn't found registration artifacts in the application server. Registration of metadata was not succesfull.");
-
- IRollbackStackDelegate rollbackStackDelegate =
- new AbstractTransactionState.LiveTransactionRollbackDelegate(state
- .getGlobalState().getStagingDir());
-
- recoveryState.getRollbackStack().setLockedState(false);
-
- recoveryState.getRollbackStack().rollbackAll(rollbackStackDelegate);
- UnstoreDataAction action =
- state.getOnErrorActionDecision().computeUndoAction(
- ErrorType.OPENBIS_REGISTRATION_FAILURE, null);
- DataSetStorageRollbacker rollbacker =
- new DataSetStorageRollbacker(state, operationLog, action, recoveryState
- .getIncomingDataSetFile().getRealIncomingFile(), null, null,
- ErrorType.OPENBIS_REGISTRATION_FAILURE);
- operationLog.info(rollbacker.getErrorMessageForLog());
- rollbacker.doRollback(logger);
-
- shouldDeleteRecoveryFiles = true;
- //
- // invokeRollbackTransactionFunction
- //
- // // rollback during metadata registration
- } else
- {
- operationLog
- .info("Recovery has found datasets in the AS. The registration of metadata was succesfull.");
- runner.storeAfterRegistration();
- logger.registerSuccess();
- registrationSuccessful = true;
-
- shouldDeleteRecoveryFiles = true;
- }
- } catch (org.jmock.api.ExpectationError r)
- {
- // this exception can by only thrown by tests.
- // propagation of the exception is essential to test some functionalities
- throw r;
- } catch (Throwable error)
- {
- System.err.println("Caught an error! " + error);
- error.printStackTrace();
- // in this case we should ignore, and run the recovery again after some time
- encounteredErrors.add(error);
- }
-
- logger.logDssRegistrationResult(encounteredErrors);
-
- cleanAfterwardsAction.execute(registrationSuccessful);
-
- recoveryMarkerCleanup.execute(shouldDeleteRecoveryFiles);
+ return hookAdaptor;
}
/**
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageAlgorithmRunner.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageAlgorithmRunner.java
index 1d109cbd282..e48c1bb7a04 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageAlgorithmRunner.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageAlgorithmRunner.java
@@ -561,4 +561,9 @@ public class DataSetStorageAlgorithmRunner<T extends DataSetInformation>
{
return incomingDataSetFile;
}
+
+ public DataSetRegistrationContext getPersistentMap()
+ {
+ return transaction.getTransactionPersistentMap();
+ }
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStoragePrecommitRecoveryState.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStoragePrecommitRecoveryState.java
index ed69eda57bc..b4f5c95f6ca 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStoragePrecommitRecoveryState.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStoragePrecommitRecoveryState.java
@@ -43,9 +43,17 @@ public class DataSetStoragePrecommitRecoveryState<T extends DataSetInformation>
private final DataSetFile incomingDataSetFile;
+ private final DataSetRegistrationContext persistentMap;
+
+ public DataSetRegistrationContext getPersistentMap()
+ {
+ return persistentMap;
+ }
+
public DataSetStoragePrecommitRecoveryState(
List<DataSetStorageAlgorithm<T>> dataSetStorageAlgorithms,
- DssRegistrationLogger logger, IRollbackStack rollbackStack, DataSetFile incomingDataSetFile)
+ DssRegistrationLogger logger, IRollbackStack rollbackStack, DataSetFile incomingDataSetFile,
+ DataSetRegistrationContext persistentMap)
{
this.dataSetRecoveryStorageAlgorithms =
new ArrayList<DataSetStoragePrecommitRecoveryAlgorithm<T>>();
@@ -57,6 +65,8 @@ public class DataSetStoragePrecommitRecoveryState<T extends DataSetInformation>
this.rollbackStackBackingFiles = ((RollbackStack) rollbackStack).getBackingFiles();
this.incomingDataSetFile = incomingDataSetFile;
+
+ this.persistentMap = persistentMap;
}
public DataSetFile getIncomingDataSetFile()
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageRecoveryManager.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageRecoveryManager.java
index cc9e9f2c1b0..90d4e0f0aa0 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageRecoveryManager.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageRecoveryManager.java
@@ -58,7 +58,8 @@ public class DataSetStorageRecoveryManager implements IDataSetStorageRecoveryMan
DataSetStoragePrecommitRecoveryState<T> recoveryState =
new DataSetStoragePrecommitRecoveryState<T>(runner.getDataSetStorageAlgorithms(),
- runner.getDssRegistrationLogger(), runner.getRollbackStack(), incoming);
+ runner.getDssRegistrationLogger(), runner.getRollbackStack(), incoming,
+ runner.getPersistentMap());
runner.getRollbackStack().setLockedState(true);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JythonTopLevelDataSetHandlerV2.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JythonTopLevelDataSetHandlerV2.java
index 93ab021dc42..844a7c53db3 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JythonTopLevelDataSetHandlerV2.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JythonTopLevelDataSetHandlerV2.java
@@ -17,6 +17,10 @@
package ch.systemsx.cisd.etlserver.registrator.api.v2;
import java.io.File;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
import org.python.core.PyFunction;
import org.python.util.PythonInterpreter;
@@ -24,11 +28,26 @@ import org.python.util.PythonInterpreter;
import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
import ch.systemsx.cisd.common.utilities.IDelegatedActionWithResult;
import ch.systemsx.cisd.common.utilities.PythonUtils;
+import ch.systemsx.cisd.etlserver.DssRegistrationLogger;
import ch.systemsx.cisd.etlserver.ITopLevelDataSetRegistratorDelegate;
import ch.systemsx.cisd.etlserver.TopLevelDataSetRegistratorGlobalState;
+import ch.systemsx.cisd.etlserver.IStorageProcessorTransactional.UnstoreDataAction;
import ch.systemsx.cisd.etlserver.registrator.DataSetFile;
import ch.systemsx.cisd.etlserver.registrator.DataSetRegistrationService;
+import ch.systemsx.cisd.etlserver.registrator.DataSetStorageAlgorithmRunner;
+import ch.systemsx.cisd.etlserver.registrator.DataSetStoragePrecommitRecoveryState;
+import ch.systemsx.cisd.etlserver.registrator.DataSetStorageRecoveryInfo;
+import ch.systemsx.cisd.etlserver.registrator.DataSetStorageRollbacker;
+import ch.systemsx.cisd.etlserver.registrator.MarkerFileUtility;
+import ch.systemsx.cisd.etlserver.registrator.AbstractOmniscientTopLevelDataSetRegistrator.DoNothingDelegatedAction;
+import ch.systemsx.cisd.etlserver.registrator.AbstractOmniscientTopLevelDataSetRegistrator.PostRegistrationCleanUpAction;
+import ch.systemsx.cisd.etlserver.registrator.DataSetStorageAlgorithmRunner.IPrePostRegistrationHook;
+import ch.systemsx.cisd.etlserver.registrator.DataSetStorageAlgorithmRunner.IRollbackDelegate;
+import ch.systemsx.cisd.etlserver.registrator.IDataSetOnErrorActionDecision.ErrorType;
+import ch.systemsx.cisd.etlserver.registrator.api.v1.impl.AbstractTransactionState;
+import ch.systemsx.cisd.etlserver.registrator.api.v1.impl.RollbackStack.IRollbackStackDelegate;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ExternalData;
/**
* A top-level data set handler that runs a python (jython) script to register data sets.
@@ -127,4 +146,240 @@ public class JythonTopLevelDataSetHandlerV2<T extends DataSetInformation> extend
// If there is a recovery marker file, do not add the file to faulty paths.
return hasRecoveryMarkerFile(file);
}
+
+ @Override
+ protected boolean hasRecoveryMarkerFile(File incoming)
+ {
+ return getGlobalState().getStorageRecoveryManager().getProcessingMarkerFile(incoming)
+ .exists();
+ }
+
+ @Override
+ protected void handleRecovery(final File incomingFileOriginal)
+ {
+ // get the marker file
+ final File recoveryMarkerFile =
+ state.getGlobalState().getStorageRecoveryManager()
+ .getProcessingMarkerFile(incomingFileOriginal);
+
+ // deserialize recovery state
+ final DataSetStoragePrecommitRecoveryState<T> recoveryState =
+ state.getGlobalState().getStorageRecoveryManager()
+ .extractPrecommittedCheckpoint(recoveryMarkerFile);
+
+ // then we should ensure that the recovery will actually take place itself!
+ final DataSetStorageRecoveryInfo recoveryInfo =
+ state.getGlobalState().getStorageRecoveryManager()
+ .getRecoveryFileFromMarker(recoveryMarkerFile);
+
+ final File recoveryFile = recoveryInfo.getRecoveryStateFile();
+
+ if (false == recoveryFile.exists())
+ {
+ // TODO: is it safe to throw from here?
+ operationLog.error("recovery file does not exist. " + recoveryFile);
+ throw new IllegalStateException("Recovery file " + recoveryFile + " doesn't exist");
+ }
+
+ if (false == retryPeriodHasPassed(recoveryInfo))
+ {
+ operationLog.info("Found recovery inforation for " + incomingFileOriginal
+ + ". The recovery won't happen as the retry period has not yet passed");
+ return;
+ }
+
+ operationLog.info("will recover from broken registration. Found marker file "
+ + recoveryMarkerFile + " and " + recoveryFile);
+
+ IDelegatedActionWithResult<Boolean> recoveryMarkerFileCleanupAction =
+ new IDelegatedActionWithResult<Boolean>()
+ {
+ public Boolean execute(boolean didOperationSucceed)
+ {
+ if (!didOperationSucceed
+ && recoveryInfo.getTryCount() >= state.getGlobalState()
+ .getStorageRecoveryManager().getMaximumRertyCount())
+ {
+ notificationLog.error("The dataset "
+ + recoveryState.getIncomingDataSetFile()
+ .getRealIncomingFile()
+ + " has failed to register. Giving up.");
+ deleteMarkerFile();
+
+ System.err.println("This is happening " + recoveryMarkerFile);
+ File errorRecoveryMarkerFile =
+ new File(recoveryMarkerFile.getParent(),
+ recoveryMarkerFile.getName() + ".ERROR");
+ state.getFileOperations().move(recoveryMarkerFile,
+ errorRecoveryMarkerFile);
+ } else
+ {
+ if (didOperationSucceed)
+ {
+ deleteMarkerFile();
+
+ recoveryMarkerFile.delete();
+ recoveryFile.delete();
+ } else
+ {
+ // this replaces the recovery file with a new one with increased
+ // count
+ // FIXME: is this safe operation (how to assure, that it won't
+ // corrupt the recoveryMarkerFile?)
+
+ recoveryInfo.increaseTryCount();
+ recoveryInfo.setLastTry(new Date());
+ recoveryInfo.writeToFile(recoveryMarkerFile);
+ }
+ }
+ return true;
+ }
+
+ private void deleteMarkerFile()
+ {
+ File incomingMarkerFile =
+ MarkerFileUtility.getMarkerFileFromIncoming(recoveryState
+ .getIncomingDataSetFile().getRealIncomingFile());
+ if (incomingMarkerFile.exists())
+ {
+
+ incomingMarkerFile.delete();
+ }
+ }
+ };
+
+ PostRegistrationCleanUpAction cleanupAction =
+ new PostRegistrationCleanUpAction(recoveryState.getIncomingDataSetFile(),
+ new DoNothingDelegatedAction());
+
+ handleRecoveryState(recoveryState, cleanupAction, recoveryMarkerFileCleanupAction);
+ }
+
+
+ /**
+ * Check wheter the last retry + retry period < date.now
+ */
+ private boolean retryPeriodHasPassed(final DataSetStorageRecoveryInfo recoveryInfo)
+ {
+ Calendar c = Calendar.getInstance();
+ c.setTime(recoveryInfo.getLastTry());
+ c.add(Calendar.SECOND, state.getGlobalState().getStorageRecoveryManager()
+ .getRetryPeriodInSeconds());
+ return c.getTime().before(new Date());
+ }
+
+
+ private void handleRecoveryState(DataSetStoragePrecommitRecoveryState<T> recoveryState,
+ final IDelegatedActionWithResult<Boolean> cleanAfterwardsAction,
+ final IDelegatedActionWithResult<Boolean> recoveryMarkerCleanup)
+ {
+ // TODO: Jobs left to do here:
+ // jython hooks
+
+ DssRegistrationLogger logger = recoveryState.getRegistrationLogger(state);
+
+ logger.log("The registration has been disturbed. Will try to recover...");
+
+ // rollback delegate
+ final List<Throwable> encounteredErrors = new ArrayList<Throwable>();
+
+ // keeps track of whether we should keep or delete the recovery files.
+ // we can delete if succesfully recovered, or rolledback, or gave up
+ boolean shouldDeleteRecoveryFiles = false;
+
+ IRollbackDelegate<T> rollbackDelegate = new IRollbackDelegate<T>()
+ {
+ public void didRollbackStorageAlgorithmRunner(
+ DataSetStorageAlgorithmRunner<T> algorithm, Throwable ex,
+ ErrorType errorType)
+ {
+ encounteredErrors.add(ex);
+ }
+ };
+
+ // hookAdaptor
+ IPrePostRegistrationHook<T> hookAdaptor = createRecoveryHookAdaptor();
+
+ DataSetStorageAlgorithmRunner<T> runner =
+ new DataSetStorageAlgorithmRunner<T>(recoveryState.getIncomingDataSetFile(), // incoming
+ recoveryState.getDataSetStorageAlgorithms(state), // algorithms
+ rollbackDelegate, // rollback delegate,
+ recoveryState.getRollbackStack(), // rollbackstack
+ logger, // registrationLogger
+ state.getGlobalState().getOpenBisService(), // openBisService
+ hookAdaptor, // the hooks
+ state.getGlobalState().getStorageRecoveryManager());
+
+ boolean registrationSuccessful = false;
+
+ operationLog.info("Recovery succesfully deserialized the state of the registration");
+ try
+ {
+ List<String> dataSetCodes = recoveryState.getDataSetCodes();
+ // TODO: we need to check for something more than just a registered datasets, for the
+ // registrations that doesn't register anything.
+ // OTOH: do we need to recover in this case? (as there is nothing to store)
+ // maybe then we should only checkpoint for recovery if there ARE datasets registered?
+
+ // TODO: verify why if this throws exception - the empty directory is created in a
+ // store?
+ List<ExternalData> registeredDataSets =
+ state.getGlobalState().getOpenBisService().listDataSetsByCode(dataSetCodes);
+ if (registeredDataSets.isEmpty())
+ {
+ operationLog
+ .info("Recovery hasn't found registration artifacts in the application server. Registration of metadata was not succesfull.");
+
+ IRollbackStackDelegate rollbackStackDelegate =
+ new AbstractTransactionState.LiveTransactionRollbackDelegate(state
+ .getGlobalState().getStagingDir());
+
+ recoveryState.getRollbackStack().setLockedState(false);
+
+ recoveryState.getRollbackStack().rollbackAll(rollbackStackDelegate);
+ UnstoreDataAction action =
+ state.getOnErrorActionDecision().computeUndoAction(
+ ErrorType.OPENBIS_REGISTRATION_FAILURE, null);
+ DataSetStorageRollbacker rollbacker =
+ new DataSetStorageRollbacker(state, operationLog, action, recoveryState
+ .getIncomingDataSetFile().getRealIncomingFile(), null, null,
+ ErrorType.OPENBIS_REGISTRATION_FAILURE);
+ operationLog.info(rollbacker.getErrorMessageForLog());
+ rollbacker.doRollback(logger);
+
+ shouldDeleteRecoveryFiles = true;
+ //
+ // invokeRollbackTransactionFunction
+ //
+ // // rollback during metadata registration
+ } else
+ {
+ operationLog
+ .info("Recovery has found datasets in the AS. The registration of metadata was succesfull.");
+ runner.storeAfterRegistration();
+ logger.registerSuccess();
+ registrationSuccessful = true;
+
+ shouldDeleteRecoveryFiles = true;
+ }
+ } catch (org.jmock.api.ExpectationError r)
+ {
+ // this exception can by only thrown by tests.
+ // propagation of the exception is essential to test some functionalities
+ throw r;
+ } catch (Throwable error)
+ {
+ System.err.println("Caught an error! " + error);
+ error.printStackTrace();
+ // in this case we should ignore, and run the recovery again after some time
+ encounteredErrors.add(error);
+ }
+
+ logger.logDssRegistrationResult(encounteredErrors);
+
+ cleanAfterwardsAction.execute(registrationSuccessful);
+
+ recoveryMarkerCleanup.execute(shouldDeleteRecoveryFiles);
+ }
+
}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/JythonDropboxRecoveryTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/JythonDropboxRecoveryTest.java
index 709559e1025..39c00bd9d7e 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/JythonDropboxRecoveryTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/JythonDropboxRecoveryTest.java
@@ -82,7 +82,7 @@ public class JythonDropboxRecoveryTest extends AbstractJythonDataSetHandlerTest
testCase.recoveryRertyCount = 3;
testCases.add(testCase);
- testCase = new RecoveryTestCase("check retry count incremented");
+ testCase = new RecoveryTestCase("check can't retry immediatelly after last failure");
testCase.recoveryLastTry = new Date();
testCase.nextTryInTheFuture = true;
testCases.add(testCase);
--
GitLab