From abd3c793dfa7890374b0e6ac9b5861c11bfd76e0 Mon Sep 17 00:00:00 2001
From: jakubs <jakubs>
Date: Fri, 11 May 2012 15:39:44 +0000
Subject: [PATCH] SP-45, BIS-21 Autorecovery - rollback stack modifications
SVN: 25244
---
...tOmniscientTopLevelDataSetRegistrator.java | 18 ++++++-
.../DataSetRegistrationService.java | 2 +-
.../DataSetStorageRecoveryManager.java | 3 ++
.../etlserver/registrator/IRollbackStack.java | 12 +++++
.../api/v1/impl/AbstractTransactionState.java | 31 +++++++++--
.../impl/DataSetRegistrationTransaction.java | 53 +++++++++++++------
.../api/v1/impl/RollbackStack.java | 53 ++++++++++++++++++-
.../v2/JythonTopLevelDataSetHandlerV2.java | 1 -
.../api/v1/impl/RollbackStackTest.java | 42 +++++++++++++++
9 files changed, 190 insertions(+), 25 deletions(-)
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/AbstractOmniscientTopLevelDataSetRegistrator.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/AbstractOmniscientTopLevelDataSetRegistrator.java
index 93eb48b58e3..20972db6e79 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/AbstractOmniscientTopLevelDataSetRegistrator.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/AbstractOmniscientTopLevelDataSetRegistrator.java
@@ -58,7 +58,9 @@ import ch.systemsx.cisd.etlserver.registrator.DataSetStorageAlgorithmRunner.IPre
import ch.systemsx.cisd.etlserver.registrator.DataSetStorageAlgorithmRunner.IRollbackDelegate;
import ch.systemsx.cisd.etlserver.registrator.IDataSetOnErrorActionDecision.ErrorType;
import ch.systemsx.cisd.etlserver.registrator.api.v1.SecondaryTransactionFailure;
+import ch.systemsx.cisd.etlserver.registrator.api.v1.impl.AbstractTransactionState;
import ch.systemsx.cisd.etlserver.registrator.api.v1.impl.DataSetRegistrationTransaction;
+import ch.systemsx.cisd.etlserver.registrator.api.v1.impl.RollbackStack.IRollbackStackDelegate;
import ch.systemsx.cisd.openbis.dss.generic.shared.api.v1.validation.ValidationError;
import ch.systemsx.cisd.openbis.dss.generic.shared.api.v1.validation.ValidationScriptRunner;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
@@ -278,8 +280,10 @@ public abstract class AbstractOmniscientTopLevelDataSetRegistrator<T extends Dat
new ReentrantLock(), FileOperations.getMonitoredInstanceForCurrentThread(),
onErrorDecision);
+ state.fileOperations.mkdirs(getRollBackStackParentFolder());
+
DataSetRegistrationTransaction
- .rollbackDeadTransactions(globalState.getDssInternalTempDir());
+ .rollbackDeadTransactions(getRollBackStackParentFolder());
}
@@ -293,6 +297,11 @@ public abstract class AbstractOmniscientTopLevelDataSetRegistrator<T extends Dat
return state.registrationLock;
}
+ public File getRollBackStackParentFolder()
+ {
+ return getGlobalState().getDssInternalTempDir();
+ }
+
/**
* returns the recovery marker file if found, or null otherwise. It first checks if the incoming
* is the marker file, then if there is a marker file corresponding to this incoming file
@@ -527,6 +536,8 @@ public abstract class AbstractOmniscientTopLevelDataSetRegistrator<T extends Dat
DssRegistrationLogger logger = recoveryState.getRegistrationLogger(state);
+ logger.log("The registration has been disturbed. Will try to recover...");
+
// rollback delegate
final List<Throwable> encounteredErrors = new ArrayList<Throwable>();
@@ -578,7 +589,10 @@ public abstract class AbstractOmniscientTopLevelDataSetRegistrator<T extends Dat
if (registeredDataSets.isEmpty())
{
operationLog.info("Recovery hasn't found registration artifacts in the application server. Registration of metadata was not succesfull.");
- // recoveryState.getRollbackStack().rollbackAll(Live transaction state);./fe
+
+ IRollbackStackDelegate rollbackStackDelegate = new AbstractTransactionState.LiveTransactionRollbackDelegate(state.getGlobalState().getStagingDir());
+
+ recoveryState.getRollbackStack().rollbackAll(rollbackStackDelegate);
// encounteredErrors.add(ex);
//
// UnstoreDataAction action =
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetRegistrationService.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetRegistrationService.java
index 74e7cdf9ac2..177a2be39a3 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetRegistrationService.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetRegistrationService.java
@@ -152,7 +152,7 @@ public class DataSetRegistrationService<T extends DataSetInformation> implements
// Clone this service for the transaction to keep them independent
DataSetRegistrationTransaction<T> transaction =
- createTransaction(registrator.getGlobalState().getDssInternalTempDir(),
+ createTransaction(registrator.getRollBackStackParentFolder(),
workingDirectory, stagingDirectory, detailsFactory);
transactions.add(transaction);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageRecoveryManager.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageRecoveryManager.java
index 245dd7dd035..128c2843092 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageRecoveryManager.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageRecoveryManager.java
@@ -41,6 +41,8 @@ public class DataSetStorageRecoveryManager implements IDataSetStorageRecoveryMan
new DataSetStoragePrecommitRecoveryState<T>(runner.getDataSetStorageAlgorithms(),
runner.getDssRegistrationLogger(), runner.getRollbackStack(), incoming);
+ runner.getRollbackStack().setLockedState(true);
+
FileUtilities.writeToFile(serializedFile, recoveryState);
File processingMarkerFile = getProcessingMarkerFile(runner);
@@ -83,6 +85,7 @@ public class DataSetStorageRecoveryManager implements IDataSetStorageRecoveryMan
public <T extends DataSetInformation> void registrationCompleted(
DataSetStorageAlgorithmRunner<T> runner)
{
+ runner.getRollbackStack().setLockedState(false);
// Cleanup the state we have accumulated
File markerFile = getProcessingMarkerFile(runner);
FileUtilities.delete(markerFile);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/IRollbackStack.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/IRollbackStack.java
index 712de5d26ef..819f12e70b7 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/IRollbackStack.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/IRollbackStack.java
@@ -29,4 +29,16 @@ public interface IRollbackStack
* Push the command onto the stack and execute it.
*/
void pushAndExecuteCommand(ITransactionalCommand cmd);
+
+ /**
+ * Sets the locked state of this rollback stack. Changing this state to true results in creating
+ * or deleting the marker file.
+ */
+ public void setLockedState(boolean lockedState);
+
+ /**
+ * Returns whether this rollback stack is in locked state (i.e. it cannot execute any rollback
+ * actions)
+ */
+ public boolean isLockedState();
}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v1/impl/AbstractTransactionState.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v1/impl/AbstractTransactionState.java
index 2942d2ee1ee..89354b27b33 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v1/impl/AbstractTransactionState.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v1/impl/AbstractTransactionState.java
@@ -45,6 +45,7 @@ import ch.systemsx.cisd.etlserver.registrator.api.v1.IProject;
import ch.systemsx.cisd.etlserver.registrator.api.v1.ISample;
import ch.systemsx.cisd.etlserver.registrator.api.v1.ISpace;
import ch.systemsx.cisd.etlserver.registrator.api.v1.SecondaryTransactionFailure;
+import ch.systemsx.cisd.etlserver.registrator.api.v1.impl.RollbackStack.IRollbackStackDelegate;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
import ch.systemsx.cisd.openbis.dss.generic.shared.api.internal.v1.IDataSetImmutable;
import ch.systemsx.cisd.openbis.dss.generic.shared.api.internal.v1.IExperimentImmutable;
@@ -70,7 +71,7 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifierFa
*
* @author Chandrasekhar Ramakrishnan
*/
-abstract class AbstractTransactionState<T extends DataSetInformation>
+public abstract class AbstractTransactionState<T extends DataSetInformation>
{
protected final DataSetRegistrationTransaction<T> parent;
@@ -100,8 +101,9 @@ abstract class AbstractTransactionState<T extends DataSetInformation>
* @author Chandrasekhar Ramakrishnan
*/
static class LiveTransactionState<T extends DataSetInformation> extends
- AbstractTransactionState<T> implements RollbackStack.IRollbackStackDelegate
+ AbstractTransactionState<T>
{
+
// Default to polling every 10 seconds and waiting for up to 5 minutes
private static int fileSystemAvailablityWaitCount = 6 * 5;
@@ -692,7 +694,7 @@ abstract class AbstractTransactionState<T extends DataSetInformation>
*/
public void rollback()
{
- rollbackStack.rollbackAll(this);
+ rollbackStack.rollbackAll(new LiveTransactionRollbackDelegate(stagingDirectory));
registeredDataSets.clear();
for (DynamicTransactionQuery query : queriesToCommit.values())
{
@@ -847,6 +849,29 @@ abstract class AbstractTransactionState<T extends DataSetInformation>
{
return false;
}
+ }
+
+ /**
+ * Rollback stack delegate that checks whether the given filesystem is accessible before letting
+ * the rollback continue.
+ */
+ public static class LiveTransactionRollbackDelegate implements IRollbackStackDelegate
+ {
+ private final File stagingDirectory;
+
+ private final int fileSystemAvailablityWaitCount;
+
+ private final int fileSystemAvailablityPollingWaitTimeMs;
+
+ /**
+ * @param stagingDirectory Expects a staging directory
+ */
+ public LiveTransactionRollbackDelegate(File stagingDirectory)
+ {
+ this.stagingDirectory = stagingDirectory;
+ this.fileSystemAvailablityWaitCount = LiveTransactionState.fileSystemAvailablityWaitCount;
+ this.fileSystemAvailablityPollingWaitTimeMs = LiveTransactionState.fileSystemAvailablityPollingWaitTimeMs;
+ }
public void willContinueRollbackAll(RollbackStack stack)
{
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v1/impl/DataSetRegistrationTransaction.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v1/impl/DataSetRegistrationTransaction.java
index 2d74f6fa30e..c2240a884df 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v1/impl/DataSetRegistrationTransaction.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v1/impl/DataSetRegistrationTransaction.java
@@ -98,11 +98,7 @@ public class DataSetRegistrationTransaction<T extends DataSetInformation> implem
static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
DataSetRegistrationTransaction.class);
- /**
- * Check if there are any uncompleted transactions and roll them back. To be called during
- * startup of a thread.
- */
- public static synchronized void rollbackDeadTransactions(File rollBackStackParentFolder)
+ public static synchronized RollbackStack[] findRollbackStacks(File rollBackStackParentFolder)
{
File[] rollbackQueue1Files = rollBackStackParentFolder.listFiles(new FilenameFilter()
{
@@ -110,25 +106,50 @@ public class DataSetRegistrationTransaction<T extends DataSetInformation> implem
{
return name.endsWith(ROLLBACK_QUEUE1_FILE_NAME_SUFFIX);
}
-
});
- for (File rollbackStackQueue1 : rollbackQueue1Files)
+ RollbackStack[] rollbackStacks = new RollbackStack[rollbackQueue1Files.length];
+
+ for (int i = 0; i < rollbackQueue1Files.length; i++)
{
+ File rollbackStackQueue1 = rollbackQueue1Files[i];
RollbackStack stack = createExistingRollbackStack(rollbackStackQueue1);
- operationLog.info("Found dead rollback stack: " + rollbackStackQueue1
- + ". Rolling back.");
+ rollbackStacks[i] = stack;
+ }
+ return rollbackStacks;
+ }
+
+ /**
+ * Check if there are any uncompleted transactions and roll them back. To be called during
+ * startup of a thread.
+ */
+ public static synchronized void rollbackDeadTransactions(File rollBackStackParentFolder)
+ {
- try
+ RollbackStack[] stacks = findRollbackStacks(rollBackStackParentFolder);
+ for (RollbackStack stack : stacks)
+ {
+ if (stack.isLockedState())
{
- stack.rollbackAll();
- } catch (Throwable ex)
+ operationLog.info("Found rollback stack in locked state: "
+ + stack.getBackingFiles()[0] + ". Not Rolling back.");
+ } else
{
- // This should ever happen since rollbackAll should handle execptions, but is here
- // as a safeguard.
- operationLog.error("Encountered error rolling back transaction:", ex);
+ operationLog.info("Found dead rollback stack: " + stack.getBackingFiles()[0]
+ + ". Rolling back.");
+
+ try
+ {
+ stack.rollbackAll();
+ } catch (Throwable ex)
+ {
+ // This should ever happen since rollbackAll should handle execptions, but is
+ // here
+ // as a safeguard.
+ operationLog.error("Encountered error rolling back transaction:", ex);
+ }
+ stack.discard();
}
- stack.discard();
}
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v1/impl/RollbackStack.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v1/impl/RollbackStack.java
index 6a8d8662dee..e633626329a 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v1/impl/RollbackStack.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v1/impl/RollbackStack.java
@@ -17,8 +17,8 @@
package ch.systemsx.cisd.etlserver.registrator.api.v1.impl;
import java.io.File;
+import java.io.IOException;
import java.io.Serializable;
-import java.util.Arrays;
import java.util.Queue;
import org.apache.commons.lang.builder.ToStringBuilder;
@@ -61,6 +61,8 @@ public class RollbackStack implements IRollbackStack
private final File queue2File;
+ private final File lockedMarkerFile;
+
// These are not final because they get swapped around.
private PersistentExtendedBlockingQueueDecorator<StackElement> liveLifo;
@@ -78,6 +80,9 @@ public class RollbackStack implements IRollbackStack
this.queue1File = queue1File;
this.queue2File = queue2File;
+ this.lockedMarkerFile =
+ new File(queue1File.getParentFile(), queue1File.getName() + ".LOCKED");
+
PersistentExtendedBlockingQueueDecorator<StackElement> queue1 =
ExtendedBlockingQueueFactory.createSmartQueue(queue1File, false);
PersistentExtendedBlockingQueueDecorator<StackElement> queue2 =
@@ -152,7 +157,7 @@ public class RollbackStack implements IRollbackStack
/**
* Rollback the top of the stack and pop it from the stack
*/
- public ITransactionalCommand rollbackAndPop()
+ ITransactionalCommand rollbackAndPop()
{
StackElement elt = peek();
try
@@ -240,6 +245,11 @@ public class RollbackStack implements IRollbackStack
*/
public void rollbackAll(IRollbackStackDelegate delegate)
{
+ if (isLockedState())
+ {
+ throw new IllegalStateException("Rollback stack is in the locked state. Triggering rollback forbidden.");
+ }
+
getOperationLog().info("Rolling back stack " + this);
// Pop and rollback all
while (size() > 0)
@@ -254,6 +264,10 @@ public class RollbackStack implements IRollbackStack
*/
public void discard()
{
+ if (isLockedState())
+ {
+ throw new IllegalStateException("Discarding of locked rollback stack is illegal. Set locked to false first.");
+ }
// Close the persistent queues
liveLifo.close();
tempLifo.close();
@@ -264,6 +278,7 @@ public class RollbackStack implements IRollbackStack
// Delete the files
queue1File.delete();
queue2File.delete();
+
}
/**
@@ -276,6 +291,40 @@ public class RollbackStack implements IRollbackStack
{ queue1File, queue2File };
}
+ public void setLockedState(boolean lockedState)
+ {
+ if (!lockedState && isLockedState())
+ {
+ deleteLockedMarkerFile();
+ } else if (lockedState && false == isLockedState())
+ {
+ createLockedMarkerFile();
+ }
+ }
+
+ public boolean isLockedState()
+ {
+ return lockedMarkerFile.exists();
+ }
+
+ private void deleteLockedMarkerFile()
+ {
+ lockedMarkerFile.delete();
+ }
+
+ private void createLockedMarkerFile()
+ {
+ try
+ {
+ lockedMarkerFile.createNewFile();
+ } catch (IOException ex)
+ {
+ getOperationLog().fatal(
+ "Failed to create rollback stack lock marker file "
+ + lockedMarkerFile.getAbsolutePath());
+ }
+ }
+
/**
* A stack element combines the command with an order. The order is used to implement the
* ordering in the queue.
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JythonTopLevelDataSetHandlerV2.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JythonTopLevelDataSetHandlerV2.java
index b67dd32475a..cc818aca303 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JythonTopLevelDataSetHandlerV2.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v2/JythonTopLevelDataSetHandlerV2.java
@@ -96,5 +96,4 @@ public class JythonTopLevelDataSetHandlerV2<T extends DataSetInformation> extend
{
return false;
}
-
}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/api/v1/impl/RollbackStackTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/api/v1/impl/RollbackStackTest.java
index 386d33fed6d..cb5692c0a9f 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/api/v1/impl/RollbackStackTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/api/v1/impl/RollbackStackTest.java
@@ -48,6 +48,48 @@ public class RollbackStackTest extends AbstractTestWithRollbackStack
assertTrue(cmd2.rolledbackBeforePredecessor);
assertEquals(cmd2.predecessor, cmd1);
}
+
+ @Test
+ public void testCantRollbackIfLocked()
+ {
+ // Create some commands
+ TrackingCommand cmd1 = new TrackingCommand();
+ TrackingCommand cmd2 = new TrackingCommand(cmd1);
+
+ // Add them to the stack
+ rollbackStack.pushAndExecuteCommand(cmd1);
+ rollbackStack.pushAndExecuteCommand(cmd2);
+ assertEquals(TrackingCommandStatus.EXECUTED, cmd1.status);
+ assertEquals(TrackingCommandStatus.EXECUTED, cmd2.status);
+
+ assertEquals(
+ "RollbackStack[{StackElement [command=TrackingCommand [status=EXECUTED], order=0],StackElement [command=TrackingCommand [status=EXECUTED], order=1]}]",
+ rollbackStack.toString());
+
+ rollbackStack.setLockedState(true);
+
+ try {
+ rollbackStack.rollbackAll();
+ fail("The rollbackAll should fail when the rollback stack is in locked state");
+ } catch (Exception ex)
+ {
+ //should catch the exception, because the rollback stack is in the locked state
+ }
+
+ assertEquals(TrackingCommandStatus.EXECUTED, cmd1.status);
+ assertEquals(TrackingCommandStatus.EXECUTED, cmd2.status);
+
+ rollbackStack.setLockedState(false);
+
+ // Rollback and check that the rollback occurred correctly
+ rollbackStack.rollbackAll();
+
+ assertEquals(TrackingCommandStatus.ROLLEDBACK, cmd1.status);
+ assertEquals(TrackingCommandStatus.ROLLEDBACK, cmd2.status);
+ assertTrue(cmd2.rolledbackBeforePredecessor);
+ assertEquals(cmd2.predecessor, cmd1);
+ }
+
@Test
public void testResume()
--
GitLab