From 07615a4661b44b182aee481f8cef45cbadb233db Mon Sep 17 00:00:00 2001
From: jakubs <jakubs>
Date: Fri, 11 Jan 2013 15:43:22 +0000
Subject: [PATCH] SP-461 BIS-293 delete pre-staging directory on registration
failure
SVN: 28064
---
.../etlserver/registrator/DataSetFile.java | 9 +++++-
...tOmniscientTopLevelDataSetRegistrator.java | 7 ++---
.../v1/DataSetRegistrationService.java | 4 +--
.../v1/DataSetStorageRollbacker.java | 30 ++++++++++++-------
...tOmniscientTopLevelDataSetRegistrator.java | 6 ++--
...actProgrammableTopLevelDataSetHandler.java | 4 +--
.../v2/DataSetRegistrationService.java | 4 +--
.../v2/DataSetStorageRollbacker.java | 30 ++++++++++++-------
.../DataSetStorageRollbackerTest.java | 7 +++--
9 files changed, 63 insertions(+), 38 deletions(-)
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetFile.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetFile.java
index a5a072e7553..2b17954badd 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetFile.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetFile.java
@@ -56,8 +56,15 @@ public class DataSetFile implements Serializable
return (logicalIncomingFile != null) ? logicalIncomingFile : realIncomingFile;
}
+ public boolean isLogicalFileSpecified()
+ {
+ return logicalIncomingFile != null
+ && (false == (realIncomingFile.equals(logicalIncomingFile)));
+ }
+
/**
- * @returns something like "original file: /local/path/inbox/file.txt logical file: /local/path/pre-staging/file.txt"
+ * @returns something like
+ * "original file: /local/path/inbox/file.txt logical file: /local/path/pre-staging/file.txt"
*/
@Override
public String toString()
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v1/AbstractOmniscientTopLevelDataSetRegistrator.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v1/AbstractOmniscientTopLevelDataSetRegistrator.java
index cce088c64ed..6afedcc0e5d 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v1/AbstractOmniscientTopLevelDataSetRegistrator.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v1/AbstractOmniscientTopLevelDataSetRegistrator.java
@@ -410,6 +410,7 @@ public abstract class AbstractOmniscientTopLevelDataSetRegistrator<T extends Dat
new DssUniqueFilenameGenerator(globalState.getThreadParameters().getThreadName(),
incomingDataSetFile.getName(), null).generateFilename();
File preStagingDir = new File(preStagingRootDir, incomingDirName);
+
preStagingDir.mkdir();
// Try to find a hardlink maker
@@ -548,8 +549,7 @@ public abstract class AbstractOmniscientTopLevelDataSetRegistrator<T extends Dat
ErrorType.REGISTRATION_SCRIPT_ERROR, ex);
DataSetStorageRollbacker rollbacker =
new DataSetStorageRollbacker(getRegistratorState(), operationLog, action,
- incomingDataSetFile.getRealIncomingFile(), null, ex,
- ErrorType.REGISTRATION_SCRIPT_ERROR);
+ incomingDataSetFile, null, ex, ErrorType.REGISTRATION_SCRIPT_ERROR);
operationLog.info(rollbacker.getErrorMessageForLog());
service.getDssRegistrationLog().log("Processing failed : " + ex.toString());
@@ -589,8 +589,7 @@ public abstract class AbstractOmniscientTopLevelDataSetRegistrator<T extends Dat
ErrorType.INVALID_DATA_SET, null);
DataSetStorageRollbacker rollbacker =
new DataSetStorageRollbacker(getRegistratorState(), operationLog, action,
- incomingDataSetFile.getRealIncomingFile(), null, null,
- ErrorType.INVALID_DATA_SET);
+ incomingDataSetFile, null, null, ErrorType.INVALID_DATA_SET);
sb.append(rollbacker.getErrorMessageForLog());
String logMessage = sb.toString();
operationLog.info(logMessage);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v1/DataSetRegistrationService.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v1/DataSetRegistrationService.java
index 6543c7e53d3..b51e468894f 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v1/DataSetRegistrationService.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v1/DataSetRegistrationService.java
@@ -251,7 +251,7 @@ public class DataSetRegistrationService<T extends DataSetInformation> implements
{
DataSetStorageRollbacker rollbacker =
new DataSetStorageRollbacker(registratorContext, operationLog,
- UnstoreDataAction.MOVE_TO_ERROR, incomingDataSetFile.getRealIncomingFile(),
+ UnstoreDataAction.MOVE_TO_ERROR, incomingDataSetFile,
dataSetTypeCodeOrNull, null);
return rollbacker.doRollback(dssRegistrationLog);
}
@@ -268,7 +268,7 @@ public class DataSetRegistrationService<T extends DataSetInformation> implements
registratorContext.getOnErrorActionDecision().computeUndoAction(errorType, ex);
DataSetStorageRollbacker rollbacker =
new DataSetStorageRollbacker(registratorContext, operationLog, action,
- incomingDataSetFile.getRealIncomingFile(), null, ex, errorType);
+ incomingDataSetFile, null, ex, errorType);
operationLog.info(rollbacker.getErrorMessageForLog());
rollbacker.doRollback(dssRegistrationLog);
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v1/DataSetStorageRollbacker.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v1/DataSetStorageRollbacker.java
index f2e340079ca..b936c974311 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v1/DataSetStorageRollbacker.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v1/DataSetStorageRollbacker.java
@@ -31,6 +31,7 @@ import ch.systemsx.cisd.etlserver.DssRegistrationLogger;
import ch.systemsx.cisd.etlserver.FileRenamer;
import ch.systemsx.cisd.etlserver.IStorageProcessorTransactional.UnstoreDataAction;
import ch.systemsx.cisd.etlserver.TransferredDataSetHandler;
+import ch.systemsx.cisd.etlserver.registrator.DataSetFile;
import ch.systemsx.cisd.etlserver.registrator.v1.AbstractOmniscientTopLevelDataSetRegistrator.OmniscientTopLevelDataSetRegistratorState;
import ch.systemsx.cisd.etlserver.registrator.v1.IDataSetOnErrorActionDecision.ErrorType;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
@@ -50,7 +51,7 @@ public class DataSetStorageRollbacker
private final UnstoreDataAction unstoreAction;
- private final File incomingDataSetFile;
+ private final DataSetFile incomingDataSetFile;
private final String dataSetTypeCodeOrNull;
@@ -71,7 +72,7 @@ public class DataSetStorageRollbacker
* @param errorOrNull
*/
public DataSetStorageRollbacker(OmniscientTopLevelDataSetRegistratorState registratorContext,
- Logger operationLog, UnstoreDataAction unstoreAction, File incomingDataSetFile,
+ Logger operationLog, UnstoreDataAction unstoreAction, DataSetFile incomingDataSetFile,
String dataSetTypeCodeOrNull, Throwable errorOrNull)
{
this(registratorContext, operationLog, unstoreAction, incomingDataSetFile,
@@ -89,7 +90,7 @@ public class DataSetStorageRollbacker
* @param errorOrNull
*/
public DataSetStorageRollbacker(OmniscientTopLevelDataSetRegistratorState registratorContext,
- Logger operationLog, UnstoreDataAction unstoreAction, File incomingDataSetFile,
+ Logger operationLog, UnstoreDataAction unstoreAction, DataSetFile incomingDataSetFile,
String dataSetTypeCodeOrNull, Throwable errorOrNull, ErrorType errorTypeOrNull)
{
super();
@@ -123,6 +124,14 @@ public class DataSetStorageRollbacker
public File doRollback(DssRegistrationLogger dssRegistrationLog)
{
dssRegistrationLog.log(getErrorMessageForLog());
+
+ // delete pre-staging copy always - even if not deleting the real incoming directory
+ if (incomingDataSetFile.isLogicalFileSpecified())
+ {
+ FileUtilities.deleteRecursively(incomingDataSetFile.getLogicalIncomingFile()
+ .getParentFile());
+ }
+
if (unstoreAction == UnstoreDataAction.MOVE_TO_ERROR)
{
File newLocation = moveIncomingToError();
@@ -140,16 +149,16 @@ public class DataSetStorageRollbacker
{
dssRegistrationLog.log("File has been deleted.");
- FileUtilities.deleteRecursively(incomingDataSetFile,
+ FileUtilities.deleteRecursively(incomingDataSetFile.getRealIncomingFile(),
new Log4jSimpleLogger(operationLog));
return null;
}
StringBuilder untouchedMessage = new StringBuilder();
untouchedMessage.append("File has been left untouched ");
- untouchedMessage.append(incomingDataSetFile.getAbsolutePath());
+ untouchedMessage.append(incomingDataSetFile.getRealIncomingFile().getAbsolutePath());
dssRegistrationLog.log(untouchedMessage.toString());
- return incomingDataSetFile;
+ return incomingDataSetFile.getRealIncomingFile();
}
public File moveIncomingToError()
@@ -171,13 +180,14 @@ public class DataSetStorageRollbacker
TransferredDataSetHandler.ERROR_DATA_STRATEGY, registratorContext
.getStorageProcessor().getStoreRootDirectory(), registratorContext
.getFileOperations(), dataSetInfo, dataSetInfo.getDataSetType(),
- incomingDataSetFile);
+ incomingDataSetFile.getRealIncomingFile());
baseDirectoryHolder =
new BaseDirectoryHolder(TransferredDataSetHandler.ERROR_DATA_STRATEGY,
- baseDirectory, incomingDataSetFile);
+ baseDirectory, incomingDataSetFile.getRealIncomingFile());
// Move the incoming there
- FileRenamer.renameAndLog(incomingDataSetFile, baseDirectoryHolder.getTargetFile());
+ FileRenamer.renameAndLog(incomingDataSetFile.getRealIncomingFile(),
+ baseDirectoryHolder.getTargetFile());
return baseDirectoryHolder.getTargetFile();
}
@@ -187,7 +197,7 @@ public class DataSetStorageRollbacker
private void writeThrowable()
{
assert errorOrNull != null;
- final String fileName = incomingDataSetFile.getName() + ".exception";
+ final String fileName = incomingDataSetFile.getRealIncomingFile().getName() + ".exception";
final File file = new File(baseDirectoryHolder.getTargetFile().getParentFile(), fileName);
FileWriter writer = null;
try
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/AbstractOmniscientTopLevelDataSetRegistrator.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/AbstractOmniscientTopLevelDataSetRegistrator.java
index 798903b795f..b9a0bd24a9f 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/AbstractOmniscientTopLevelDataSetRegistrator.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/AbstractOmniscientTopLevelDataSetRegistrator.java
@@ -571,8 +571,7 @@ public abstract class AbstractOmniscientTopLevelDataSetRegistrator<T extends Dat
ErrorType.REGISTRATION_SCRIPT_ERROR, ex);
DataSetStorageRollbacker rollbacker =
new DataSetStorageRollbacker(getRegistratorState(), operationLog, action,
- incomingDataSetFile.getRealIncomingFile(), null, ex,
- ErrorType.REGISTRATION_SCRIPT_ERROR);
+ incomingDataSetFile, null, ex, ErrorType.REGISTRATION_SCRIPT_ERROR);
operationLog.info(rollbacker.getErrorMessageForLog());
service.getDssRegistrationLog().log("Processing failed : " + ex.toString());
@@ -612,8 +611,7 @@ public abstract class AbstractOmniscientTopLevelDataSetRegistrator<T extends Dat
ErrorType.INVALID_DATA_SET, null);
DataSetStorageRollbacker rollbacker =
new DataSetStorageRollbacker(getRegistratorState(), operationLog, action,
- incomingDataSetFile.getRealIncomingFile(), null, null,
- ErrorType.INVALID_DATA_SET);
+ incomingDataSetFile, null, null, ErrorType.INVALID_DATA_SET);
sb.append(rollbacker.getErrorMessageForLog());
String logMessage = sb.toString();
operationLog.info(logMessage);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/AbstractProgrammableTopLevelDataSetHandler.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/AbstractProgrammableTopLevelDataSetHandler.java
index 50bfc19a34a..38bf43f4ef7 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/AbstractProgrammableTopLevelDataSetHandler.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/AbstractProgrammableTopLevelDataSetHandler.java
@@ -479,8 +479,8 @@ public abstract class AbstractProgrammableTopLevelDataSetHandler<T extends DataS
state.getOnErrorActionDecision().computeUndoAction(
ErrorType.OPENBIS_REGISTRATION_FAILURE, null);
DataSetStorageRollbacker rollbacker =
- new DataSetStorageRollbacker(state, operationLog, action, recoveryState
- .getIncomingDataSetFile().getRealIncomingFile(), null, null,
+ new DataSetStorageRollbacker(state, operationLog, action,
+ recoveryState.getIncomingDataSetFile(), null, null,
ErrorType.OPENBIS_REGISTRATION_FAILURE);
operationLog.info(rollbacker.getErrorMessageForLog());
rollbacker.doRollback(logger);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/DataSetRegistrationService.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/DataSetRegistrationService.java
index 18d7fd75505..9f5c98f2d4c 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/DataSetRegistrationService.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/DataSetRegistrationService.java
@@ -232,7 +232,7 @@ public class DataSetRegistrationService<T extends DataSetInformation> implements
{
DataSetStorageRollbacker rollbacker =
new DataSetStorageRollbacker(registratorContext, operationLog,
- UnstoreDataAction.MOVE_TO_ERROR, incomingDataSetFile.getRealIncomingFile(),
+ UnstoreDataAction.MOVE_TO_ERROR, incomingDataSetFile,
dataSetTypeCodeOrNull, null);
return rollbacker.doRollback(dssRegistrationLog);
}
@@ -249,7 +249,7 @@ public class DataSetRegistrationService<T extends DataSetInformation> implements
registratorContext.getOnErrorActionDecision().computeUndoAction(errorType, ex);
DataSetStorageRollbacker rollbacker =
new DataSetStorageRollbacker(registratorContext, operationLog, action,
- incomingDataSetFile.getRealIncomingFile(), null, ex, errorType);
+ incomingDataSetFile, null, ex, errorType);
operationLog.info(rollbacker.getErrorMessageForLog());
rollbacker.doRollback(dssRegistrationLog);
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/DataSetStorageRollbacker.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/DataSetStorageRollbacker.java
index e6f83c01f96..8a69ed42f61 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/DataSetStorageRollbacker.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/DataSetStorageRollbacker.java
@@ -31,6 +31,7 @@ import ch.systemsx.cisd.etlserver.DssRegistrationLogger;
import ch.systemsx.cisd.etlserver.FileRenamer;
import ch.systemsx.cisd.etlserver.IStorageProcessorTransactional.UnstoreDataAction;
import ch.systemsx.cisd.etlserver.TransferredDataSetHandler;
+import ch.systemsx.cisd.etlserver.registrator.DataSetFile;
import ch.systemsx.cisd.etlserver.registrator.v2.AbstractOmniscientTopLevelDataSetRegistrator.OmniscientTopLevelDataSetRegistratorState;
import ch.systemsx.cisd.etlserver.registrator.v2.IDataSetOnErrorActionDecision.ErrorType;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
@@ -50,7 +51,7 @@ public class DataSetStorageRollbacker
private final UnstoreDataAction unstoreAction;
- private final File incomingDataSetFile;
+ private final DataSetFile incomingDataSetFile;
private final String dataSetTypeCodeOrNull;
@@ -71,7 +72,7 @@ public class DataSetStorageRollbacker
* @param errorOrNull
*/
public DataSetStorageRollbacker(OmniscientTopLevelDataSetRegistratorState registratorContext,
- Logger operationLog, UnstoreDataAction unstoreAction, File incomingDataSetFile,
+ Logger operationLog, UnstoreDataAction unstoreAction, DataSetFile incomingDataSetFile,
String dataSetTypeCodeOrNull, Throwable errorOrNull)
{
this(registratorContext, operationLog, unstoreAction, incomingDataSetFile,
@@ -89,7 +90,7 @@ public class DataSetStorageRollbacker
* @param errorOrNull
*/
public DataSetStorageRollbacker(OmniscientTopLevelDataSetRegistratorState registratorContext,
- Logger operationLog, UnstoreDataAction unstoreAction, File incomingDataSetFile,
+ Logger operationLog, UnstoreDataAction unstoreAction, DataSetFile incomingDataSetFile,
String dataSetTypeCodeOrNull, Throwable errorOrNull, ErrorType errorTypeOrNull)
{
super();
@@ -123,6 +124,14 @@ public class DataSetStorageRollbacker
public File doRollback(DssRegistrationLogger dssRegistrationLog)
{
dssRegistrationLog.log(getErrorMessageForLog());
+
+ // delete pre-staging copy always - even if not deleting the real incoming directory
+ if (incomingDataSetFile.isLogicalFileSpecified())
+ {
+ FileUtilities.deleteRecursively(incomingDataSetFile.getLogicalIncomingFile()
+ .getParentFile());
+ }
+
if (unstoreAction == UnstoreDataAction.MOVE_TO_ERROR)
{
File newLocation = moveIncomingToError();
@@ -140,16 +149,16 @@ public class DataSetStorageRollbacker
{
dssRegistrationLog.log("File has been deleted.");
- FileUtilities.deleteRecursively(incomingDataSetFile,
+ FileUtilities.deleteRecursively(incomingDataSetFile.getRealIncomingFile(),
new Log4jSimpleLogger(operationLog));
return null;
}
StringBuilder untouchedMessage = new StringBuilder();
untouchedMessage.append("File has been left untouched ");
- untouchedMessage.append(incomingDataSetFile.getAbsolutePath());
+ untouchedMessage.append(incomingDataSetFile.getRealIncomingFile().getAbsolutePath());
dssRegistrationLog.log(untouchedMessage.toString());
- return incomingDataSetFile;
+ return incomingDataSetFile.getRealIncomingFile();
}
public File moveIncomingToError()
@@ -171,13 +180,14 @@ public class DataSetStorageRollbacker
TransferredDataSetHandler.ERROR_DATA_STRATEGY, registratorContext
.getStorageProcessor().getStoreRootDirectory(), registratorContext
.getFileOperations(), dataSetInfo, dataSetInfo.getDataSetType(),
- incomingDataSetFile);
+ incomingDataSetFile.getRealIncomingFile());
baseDirectoryHolder =
new BaseDirectoryHolder(TransferredDataSetHandler.ERROR_DATA_STRATEGY,
- baseDirectory, incomingDataSetFile);
+ baseDirectory, incomingDataSetFile.getRealIncomingFile());
// Move the incoming there
- FileRenamer.renameAndLog(incomingDataSetFile, baseDirectoryHolder.getTargetFile());
+ FileRenamer.renameAndLog(incomingDataSetFile.getRealIncomingFile(),
+ baseDirectoryHolder.getTargetFile());
return baseDirectoryHolder.getTargetFile();
}
@@ -187,7 +197,7 @@ public class DataSetStorageRollbacker
private void writeThrowable()
{
assert errorOrNull != null;
- final String fileName = incomingDataSetFile.getName() + ".exception";
+ final String fileName = incomingDataSetFile.getRealIncomingFile().getName() + ".exception";
final File file = new File(baseDirectoryHolder.getTargetFile().getParentFile(), fileName);
FileWriter writer = null;
try
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageRollbackerTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageRollbackerTest.java
index 20ee623273d..e2287c63fba 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageRollbackerTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageRollbackerTest.java
@@ -37,9 +37,9 @@ import ch.systemsx.cisd.etlserver.ThreadParameters;
import ch.systemsx.cisd.etlserver.TopLevelDataSetRegistratorGlobalState;
import ch.systemsx.cisd.etlserver.registrator.recovery.DataSetStorageRecoveryManager;
import ch.systemsx.cisd.etlserver.registrator.v1.AbstractOmniscientTopLevelDataSetRegistrator;
+import ch.systemsx.cisd.etlserver.registrator.v1.AbstractOmniscientTopLevelDataSetRegistrator.OmniscientTopLevelDataSetRegistratorState;
import ch.systemsx.cisd.etlserver.registrator.v1.DataSetRegistrationService;
import ch.systemsx.cisd.etlserver.registrator.v1.DataSetStorageRollbacker;
-import ch.systemsx.cisd.etlserver.registrator.v1.AbstractOmniscientTopLevelDataSetRegistrator.OmniscientTopLevelDataSetRegistratorState;
import ch.systemsx.cisd.etlserver.registrator.v1.IDataSetOnErrorActionDecision.ErrorType;
import ch.systemsx.cisd.etlserver.validation.IDataSetValidator;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
@@ -65,7 +65,7 @@ public class DataSetStorageRollbackerTest extends AbstractFileSystemTestCase
private IDataSetValidator dataSetValidator;
- private File incomingDataSetFile;
+ private DataSetFile incomingDataSetFile;
private TestDataSetRegistrator testRegistrator;
@@ -89,7 +89,8 @@ public class DataSetStorageRollbackerTest extends AbstractFileSystemTestCase
@Test
public void testLogging()
{
- incomingDataSetFile = createDirectory(workingDirectory, "data_set");
+ File realIncoming = createDirectory(workingDirectory, "data_set");
+ incomingDataSetFile = new DataSetFile(realIncoming);
OmniscientTopLevelDataSetRegistratorState globalState =
testRegistrator.getRegistratorState();
DataSetStorageRollbacker rollbacker =
--
GitLab