diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/TopLevelDataSetRegistratorGlobalState.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/TopLevelDataSetRegistratorGlobalState.java
index 93d19f4a31f3f021654092d2c317df18bfbc5779..b0bd6e575acfdf06e7dea2c36e6855b6736eb44e 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/TopLevelDataSetRegistratorGlobalState.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/TopLevelDataSetRegistratorGlobalState.java
@@ -55,8 +55,6 @@ public class TopLevelDataSetRegistratorGlobalState
private final File dssRegistrationLogDir;
- private final File dropboxRecoveryStateDir;
-
private final File preStagingDir;
private final File stagingDir;
@@ -125,8 +123,6 @@ public class TopLevelDataSetRegistratorGlobalState
this.storeRootDir = storeRootDir;
this.dssInternalTempDir = dssInternalTempDir;
this.dssRegistrationLogDir = dssRegistrationLogDir;
- this.dropboxRecoveryStateDir =
- new File(dssRecoveryStateDir, threadParameters.getThreadName());
this.preStagingDir =
getPreStagingDir(storeRootDir, shareId, threadParameters.getThreadProperties());
this.stagingDir =
@@ -145,12 +141,22 @@ public class TopLevelDataSetRegistratorGlobalState
this.preRegistrationScriptOrNull = preRegistrationScriptOrNull;
this.postRegistrationScriptOrNull = postRegistrationScriptOrNull;
this.validationScriptsOrNull = validationScriptsOrNull;
+
+ File dropboxRecoveryStateDir =
+ new File(dssRecoveryStateDir, threadParameters.getThreadName());
+ File recoveryMarkerFilesDirectory =
+ new File(getRecoveryMarkerDir(storeRootDir, shareId,
+ threadParameters.getThreadProperties()), threadParameters.getThreadName());
+
+ dropboxRecoveryStateDir.mkdirs();
+ recoveryMarkerFilesDirectory.mkdirs();
+
this.storageRecoveryManager = storageRecoveryManager;
- this.storageRecoveryManager.setDropboxRecoveryStateDir(this.dropboxRecoveryStateDir);
+ this.storageRecoveryManager.setDropboxRecoveryStateDir(dropboxRecoveryStateDir);
+ this.storageRecoveryManager.setRecoveryMarkerFilesDir(recoveryMarkerFilesDirectory);
// Initialize the DSS Registration Log Directory
new DssRegistrationLogDirectoryHelper(dssRegistrationLogDir).initializeSubdirectories();
- dropboxRecoveryStateDir.mkdirs();
}
public String getDssCode()
@@ -184,14 +190,6 @@ public class TopLevelDataSetRegistratorGlobalState
return dssRegistrationLogDir;
}
- /**
- * Get the directory that holds the recovery state for this dropbox.
- */
- public File getDropboxRecoveryStateDir()
- {
- return dropboxRecoveryStateDir;
- }
-
/**
* Get the directory used for pre-staging. This holds a hardlink copy of the incoming data.
*/
@@ -306,6 +304,8 @@ public class TopLevelDataSetRegistratorGlobalState
public static final String PRE_COMMIT_DIR = "pre-commit-dir";
+ public static final String RECOVERY_MARKER_DIR = "recovery-marker-dir";
+
private static File getStagingDir(File storeRoot, String shareId, Properties threadProperties)
{
return getShareLocalDir(storeRoot, shareId, threadProperties, STAGING_DIR, "staging");
@@ -322,6 +322,13 @@ public class TopLevelDataSetRegistratorGlobalState
return getShareLocalDir(storeRoot, shareId, threadProperties, PRE_COMMIT_DIR, "pre-commit");
}
+ private static File getRecoveryMarkerDir(File storeRoot, String shareId,
+ Properties threadProperties)
+ {
+ return getShareLocalDir(storeRoot, shareId, threadProperties, RECOVERY_MARKER_DIR,
+ "recovery-marker");
+ }
+
/**
* Get a directory local to the share, respecting the user override, if one is specified, and
* defaulting to the defaultDirName.
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageRecoveryManager.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageRecoveryManager.java
index 12e4005d14142a7ca5f2741158e79f6f74052b0b..07a0930a33e7b40517cc7b22d8e3dc18ffcfecfb 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageRecoveryManager.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageRecoveryManager.java
@@ -36,12 +36,14 @@ public class DataSetStorageRecoveryManager implements IDataSetStorageRecoveryMan
private static final String PRECOMMIT_SERIALIZED = ".PRECOMMIT_SERIALIZED";
private static final String PROCESSING_MARKER = ".PROCESSING_MARKER";
-
+
static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
DataSetStorageRecoveryManager.class);
private File dropboxRecoveryStateDir;
+ private File recoveryMarkerFilesDir;
+
public <T extends DataSetInformation> void checkpointPrecommittedState(
DataSetStorageAlgorithmRunner<T> runner)
{
@@ -81,9 +83,9 @@ public class DataSetStorageRecoveryManager implements IDataSetStorageRecoveryMan
*/
public File getProcessingMarkerFile(File incoming)
{
- return new File(incoming.getParentFile(), incoming.getName() + PROCESSING_MARKER);
+ return new File(recoveryMarkerFilesDir, incoming.getName() + PROCESSING_MARKER);
}
-
+
private <T extends DataSetInformation> File getSerializedFile(
DataSetStorageAlgorithmRunner<T> runner)
{
@@ -118,9 +120,9 @@ public class DataSetStorageRecoveryManager implements IDataSetStorageRecoveryMan
{
File markerFile = getProcessingMarkerFile(runner);
File recoveryState = getSerializedFile(runner);
-
- operationLog.info("Cleanup recovery with marker file "+ markerFile);
-
+
+ operationLog.info("Cleanup recovery with marker file " + markerFile);
+
runner.getRollbackStack().setLockedState(false);
// Cleanup the state we have accumulated
FileUtilities.delete(markerFile);
@@ -136,14 +138,14 @@ public class DataSetStorageRecoveryManager implements IDataSetStorageRecoveryMan
return true;
}
- public static boolean isRecoveryFile(File file)
+ public void setDropboxRecoveryStateDir(File dropboxRecoveryStateDir)
{
- return file.getName().endsWith(PROCESSING_MARKER);
+ this.dropboxRecoveryStateDir = dropboxRecoveryStateDir;
}
- public void setDropboxRecoveryStateDir(File dropboxRecoveryStateDir)
+ public void setRecoveryMarkerFilesDir(File recoveryMarkerFileDir)
{
- this.dropboxRecoveryStateDir = dropboxRecoveryStateDir;
+ this.recoveryMarkerFilesDir = recoveryMarkerFileDir;
}
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/IDataSetStorageRecoveryManager.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/IDataSetStorageRecoveryManager.java
index 25a55409e69bbf0b5b022048df3e633249a29874..5ce38dd5de0a97d107ab48dc1285f4f3d0a9ac7b 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/IDataSetStorageRecoveryManager.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/IDataSetStorageRecoveryManager.java
@@ -68,4 +68,6 @@ public interface IDataSetStorageRecoveryManager
boolean canRecoverFromError(Throwable ex);
void setDropboxRecoveryStateDir(File dropboxRecoveryStateDir);
+
+ void setRecoveryMarkerFilesDir(File recoveryMarkerFileDir);
}
\ No newline at end of file
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/AbstractJythonDataSetHandlerTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/AbstractJythonDataSetHandlerTest.java
index da2c9418b77c15c28472a1c803500190ea5285a9..fc7b653941c796b6c499600e942199ea32dd3915 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/AbstractJythonDataSetHandlerTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/AbstractJythonDataSetHandlerTest.java
@@ -115,7 +115,6 @@ public abstract class AbstractJythonDataSetHandlerTest extends AbstractFileSyste
protected static final String SAMPLE_PERM_ID = "sample-perm-id";
-
@BeforeTest
public void init()
{
@@ -179,9 +178,9 @@ public abstract class AbstractJythonDataSetHandlerTest extends AbstractFileSyste
shouldReThrowException);
}
}
-
+
protected TopLevelDataSetRegistratorGlobalState createGlobalState(Properties threadProperties)
- {
+ {
ThreadParameters threadParameters =
new ThreadParameters(threadProperties, "jython-handler-test");
@@ -212,6 +211,8 @@ public abstract class AbstractJythonDataSetHandlerTest extends AbstractFileSyste
{
one(storageRecoveryManager).setDropboxRecoveryStateDir(
new File(workingDirectory, "jython-handler-test"));
+ one(storageRecoveryManager).setRecoveryMarkerFilesDir(new File(
+ new File(workingDirectory, "recovery-marker"), "jython-handler-test"));
}
});
}
@@ -402,7 +403,7 @@ public abstract class AbstractJythonDataSetHandlerTest extends AbstractFileSyste
};
}
}
-
+
protected void assertStorageProcess(AtomicEntityOperationDetails recordedObject,
String dataSetCode, String dataSetDirectory, int testId)
{