From 2d7ff9bc3349b7d80bf82303056033106fc20247 Mon Sep 17 00:00:00 2001
From: jakubs <jakubs>
Date: Mon, 21 May 2012 08:48:16 +0000
Subject: [PATCH] SP-45, BIS-21 add a new directory where the recovery marker
files will be stored
SVN: 25302
---
...TopLevelDataSetRegistratorGlobalState.java | 35 +++++++++++--------
.../DataSetStorageRecoveryManager.java | 22 ++++++------
.../IDataSetStorageRecoveryManager.java | 2 ++
.../AbstractJythonDataSetHandlerTest.java | 9 ++---
4 files changed, 40 insertions(+), 28 deletions(-)
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/TopLevelDataSetRegistratorGlobalState.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/TopLevelDataSetRegistratorGlobalState.java
index 93d19f4a31f..b0bd6e575ac 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/TopLevelDataSetRegistratorGlobalState.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/TopLevelDataSetRegistratorGlobalState.java
@@ -55,8 +55,6 @@ public class TopLevelDataSetRegistratorGlobalState
private final File dssRegistrationLogDir;
- private final File dropboxRecoveryStateDir;
-
private final File preStagingDir;
private final File stagingDir;
@@ -125,8 +123,6 @@ public class TopLevelDataSetRegistratorGlobalState
this.storeRootDir = storeRootDir;
this.dssInternalTempDir = dssInternalTempDir;
this.dssRegistrationLogDir = dssRegistrationLogDir;
- this.dropboxRecoveryStateDir =
- new File(dssRecoveryStateDir, threadParameters.getThreadName());
this.preStagingDir =
getPreStagingDir(storeRootDir, shareId, threadParameters.getThreadProperties());
this.stagingDir =
@@ -145,12 +141,22 @@ public class TopLevelDataSetRegistratorGlobalState
this.preRegistrationScriptOrNull = preRegistrationScriptOrNull;
this.postRegistrationScriptOrNull = postRegistrationScriptOrNull;
this.validationScriptsOrNull = validationScriptsOrNull;
+
+ File dropboxRecoveryStateDir =
+ new File(dssRecoveryStateDir, threadParameters.getThreadName());
+ File recoveryMarkerFilesDirectory =
+ new File(getRecoveryMarkerDir(storeRootDir, shareId,
+ threadParameters.getThreadProperties()), threadParameters.getThreadName());
+
+ dropboxRecoveryStateDir.mkdirs();
+ recoveryMarkerFilesDirectory.mkdirs();
+
this.storageRecoveryManager = storageRecoveryManager;
- this.storageRecoveryManager.setDropboxRecoveryStateDir(this.dropboxRecoveryStateDir);
+ this.storageRecoveryManager.setDropboxRecoveryStateDir(dropboxRecoveryStateDir);
+ this.storageRecoveryManager.setRecoveryMarkerFilesDir(recoveryMarkerFilesDirectory);
// Initialize the DSS Registration Log Directory
new DssRegistrationLogDirectoryHelper(dssRegistrationLogDir).initializeSubdirectories();
- dropboxRecoveryStateDir.mkdirs();
}
public String getDssCode()
@@ -184,14 +190,6 @@ public class TopLevelDataSetRegistratorGlobalState
return dssRegistrationLogDir;
}
- /**
- * Get the directory that holds the recovery state for this dropbox.
- */
- public File getDropboxRecoveryStateDir()
- {
- return dropboxRecoveryStateDir;
- }
-
/**
* Get the directory used for pre-staging. This holds a hardlink copy of the incoming data.
*/
@@ -306,6 +304,8 @@ public class TopLevelDataSetRegistratorGlobalState
public static final String PRE_COMMIT_DIR = "pre-commit-dir";
+ public static final String RECOVERY_MARKER_DIR = "recovery-marker-dir";
+
private static File getStagingDir(File storeRoot, String shareId, Properties threadProperties)
{
return getShareLocalDir(storeRoot, shareId, threadProperties, STAGING_DIR, "staging");
@@ -322,6 +322,13 @@ public class TopLevelDataSetRegistratorGlobalState
return getShareLocalDir(storeRoot, shareId, threadProperties, PRE_COMMIT_DIR, "pre-commit");
}
+ private static File getRecoveryMarkerDir(File storeRoot, String shareId,
+ Properties threadProperties)
+ {
+ return getShareLocalDir(storeRoot, shareId, threadProperties, RECOVERY_MARKER_DIR,
+ "recovery-marker");
+ }
+
/**
* Get a directory local to the share, respecting the user override, if one is specified, and
* defaulting to the defaultDirName.
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageRecoveryManager.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageRecoveryManager.java
index 12e4005d141..07a0930a33e 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageRecoveryManager.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageRecoveryManager.java
@@ -36,12 +36,14 @@ public class DataSetStorageRecoveryManager implements IDataSetStorageRecoveryMan
private static final String PRECOMMIT_SERIALIZED = ".PRECOMMIT_SERIALIZED";
private static final String PROCESSING_MARKER = ".PROCESSING_MARKER";
-
+
static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
DataSetStorageRecoveryManager.class);
private File dropboxRecoveryStateDir;
+ private File recoveryMarkerFilesDir;
+
public <T extends DataSetInformation> void checkpointPrecommittedState(
DataSetStorageAlgorithmRunner<T> runner)
{
@@ -81,9 +83,9 @@ public class DataSetStorageRecoveryManager implements IDataSetStorageRecoveryMan
*/
public File getProcessingMarkerFile(File incoming)
{
- return new File(incoming.getParentFile(), incoming.getName() + PROCESSING_MARKER);
+ return new File(recoveryMarkerFilesDir, incoming.getName() + PROCESSING_MARKER);
}
-
+
private <T extends DataSetInformation> File getSerializedFile(
DataSetStorageAlgorithmRunner<T> runner)
{
@@ -118,9 +120,9 @@ public class DataSetStorageRecoveryManager implements IDataSetStorageRecoveryMan
{
File markerFile = getProcessingMarkerFile(runner);
File recoveryState = getSerializedFile(runner);
-
- operationLog.info("Cleanup recovery with marker file "+ markerFile);
-
+
+ operationLog.info("Cleanup recovery with marker file " + markerFile);
+
runner.getRollbackStack().setLockedState(false);
// Cleanup the state we have accumulated
FileUtilities.delete(markerFile);
@@ -136,14 +138,14 @@ public class DataSetStorageRecoveryManager implements IDataSetStorageRecoveryMan
return true;
}
- public static boolean isRecoveryFile(File file)
+ public void setDropboxRecoveryStateDir(File dropboxRecoveryStateDir)
{
- return file.getName().endsWith(PROCESSING_MARKER);
+ this.dropboxRecoveryStateDir = dropboxRecoveryStateDir;
}
- public void setDropboxRecoveryStateDir(File dropboxRecoveryStateDir)
+ public void setRecoveryMarkerFilesDir(File recoveryMarkerFileDir)
{
- this.dropboxRecoveryStateDir = dropboxRecoveryStateDir;
+ this.recoveryMarkerFilesDir = recoveryMarkerFileDir;
}
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/IDataSetStorageRecoveryManager.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/IDataSetStorageRecoveryManager.java
index 25a55409e69..5ce38dd5de0 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/IDataSetStorageRecoveryManager.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/IDataSetStorageRecoveryManager.java
@@ -68,4 +68,6 @@ public interface IDataSetStorageRecoveryManager
boolean canRecoverFromError(Throwable ex);
void setDropboxRecoveryStateDir(File dropboxRecoveryStateDir);
+
+ void setRecoveryMarkerFilesDir(File recoveryMarkerFileDir);
}
\ No newline at end of file
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/AbstractJythonDataSetHandlerTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/AbstractJythonDataSetHandlerTest.java
index da2c9418b77..fc7b653941c 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/AbstractJythonDataSetHandlerTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/AbstractJythonDataSetHandlerTest.java
@@ -115,7 +115,6 @@ public abstract class AbstractJythonDataSetHandlerTest extends AbstractFileSyste
protected static final String SAMPLE_PERM_ID = "sample-perm-id";
-
@BeforeTest
public void init()
{
@@ -179,9 +178,9 @@ public abstract class AbstractJythonDataSetHandlerTest extends AbstractFileSyste
shouldReThrowException);
}
}
-
+
protected TopLevelDataSetRegistratorGlobalState createGlobalState(Properties threadProperties)
- {
+ {
ThreadParameters threadParameters =
new ThreadParameters(threadProperties, "jython-handler-test");
@@ -212,6 +211,8 @@ public abstract class AbstractJythonDataSetHandlerTest extends AbstractFileSyste
{
one(storageRecoveryManager).setDropboxRecoveryStateDir(
new File(workingDirectory, "jython-handler-test"));
+ one(storageRecoveryManager).setRecoveryMarkerFilesDir(new File(
+ new File(workingDirectory, "recovery-marker"), "jython-handler-test"));
}
});
}
@@ -402,7 +403,7 @@ public abstract class AbstractJythonDataSetHandlerTest extends AbstractFileSyste
};
}
}
-
+
protected void assertStorageProcess(AtomicEntityOperationDetails recordedObject,
String dataSetCode, String dataSetDirectory, int testId)
{
--
GitLab