diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetFile.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetFile.java
index a5a072e755366a2f6fcc23f08591f868b1de0e8a..2b17954baddf6fb6c2cb4eda3e59834ddd1830e7 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetFile.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetFile.java
@@ -56,8 +56,15 @@ public class DataSetFile implements Serializable
return (logicalIncomingFile != null) ? logicalIncomingFile : realIncomingFile;
}
+ public boolean isLogicalFileSpecified()
+ {
+ return logicalIncomingFile != null
+ && (false == (realIncomingFile.equals(logicalIncomingFile)));
+ }
+
/**
- * @returns something like "original file: /local/path/inbox/file.txt logical file: /local/path/pre-staging/file.txt"
+ * @returns something like
+ * "original file: /local/path/inbox/file.txt logical file: /local/path/pre-staging/file.txt"
*/
@Override
public String toString()
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v1/AbstractOmniscientTopLevelDataSetRegistrator.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v1/AbstractOmniscientTopLevelDataSetRegistrator.java
index cce088c64ed09c3357e5365b8449c90e5d23cb3b..6afedcc0e5df0484e6ef54b2e1e5088d77dd04f1 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v1/AbstractOmniscientTopLevelDataSetRegistrator.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v1/AbstractOmniscientTopLevelDataSetRegistrator.java
@@ -410,6 +410,7 @@ public abstract class AbstractOmniscientTopLevelDataSetRegistrator<T extends Dat
new DssUniqueFilenameGenerator(globalState.getThreadParameters().getThreadName(),
incomingDataSetFile.getName(), null).generateFilename();
File preStagingDir = new File(preStagingRootDir, incomingDirName);
+
preStagingDir.mkdir();
// Try to find a hardlink maker
@@ -548,8 +549,7 @@ public abstract class AbstractOmniscientTopLevelDataSetRegistrator<T extends Dat
ErrorType.REGISTRATION_SCRIPT_ERROR, ex);
DataSetStorageRollbacker rollbacker =
new DataSetStorageRollbacker(getRegistratorState(), operationLog, action,
- incomingDataSetFile.getRealIncomingFile(), null, ex,
- ErrorType.REGISTRATION_SCRIPT_ERROR);
+ incomingDataSetFile, null, ex, ErrorType.REGISTRATION_SCRIPT_ERROR);
operationLog.info(rollbacker.getErrorMessageForLog());
service.getDssRegistrationLog().log("Processing failed : " + ex.toString());
@@ -589,8 +589,7 @@ public abstract class AbstractOmniscientTopLevelDataSetRegistrator<T extends Dat
ErrorType.INVALID_DATA_SET, null);
DataSetStorageRollbacker rollbacker =
new DataSetStorageRollbacker(getRegistratorState(), operationLog, action,
- incomingDataSetFile.getRealIncomingFile(), null, null,
- ErrorType.INVALID_DATA_SET);
+ incomingDataSetFile, null, null, ErrorType.INVALID_DATA_SET);
sb.append(rollbacker.getErrorMessageForLog());
String logMessage = sb.toString();
operationLog.info(logMessage);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v1/DataSetRegistrationService.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v1/DataSetRegistrationService.java
index 6543c7e53d369fa4ba063306d2144e77d2512481..b51e468894f467f3079c91062b34149df913da96 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v1/DataSetRegistrationService.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v1/DataSetRegistrationService.java
@@ -251,7 +251,7 @@ public class DataSetRegistrationService<T extends DataSetInformation> implements
{
DataSetStorageRollbacker rollbacker =
new DataSetStorageRollbacker(registratorContext, operationLog,
- UnstoreDataAction.MOVE_TO_ERROR, incomingDataSetFile.getRealIncomingFile(),
+ UnstoreDataAction.MOVE_TO_ERROR, incomingDataSetFile,
dataSetTypeCodeOrNull, null);
return rollbacker.doRollback(dssRegistrationLog);
}
@@ -268,7 +268,7 @@ public class DataSetRegistrationService<T extends DataSetInformation> implements
registratorContext.getOnErrorActionDecision().computeUndoAction(errorType, ex);
DataSetStorageRollbacker rollbacker =
new DataSetStorageRollbacker(registratorContext, operationLog, action,
- incomingDataSetFile.getRealIncomingFile(), null, ex, errorType);
+ incomingDataSetFile, null, ex, errorType);
operationLog.info(rollbacker.getErrorMessageForLog());
rollbacker.doRollback(dssRegistrationLog);
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v1/DataSetStorageRollbacker.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v1/DataSetStorageRollbacker.java
index f2e340079caa763edabeb65202afc07a3c3d427d..b936c97431151d05d1338fe9c0479a1a1ebc1d09 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v1/DataSetStorageRollbacker.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v1/DataSetStorageRollbacker.java
@@ -31,6 +31,7 @@ import ch.systemsx.cisd.etlserver.DssRegistrationLogger;
import ch.systemsx.cisd.etlserver.FileRenamer;
import ch.systemsx.cisd.etlserver.IStorageProcessorTransactional.UnstoreDataAction;
import ch.systemsx.cisd.etlserver.TransferredDataSetHandler;
+import ch.systemsx.cisd.etlserver.registrator.DataSetFile;
import ch.systemsx.cisd.etlserver.registrator.v1.AbstractOmniscientTopLevelDataSetRegistrator.OmniscientTopLevelDataSetRegistratorState;
import ch.systemsx.cisd.etlserver.registrator.v1.IDataSetOnErrorActionDecision.ErrorType;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
@@ -50,7 +51,7 @@ public class DataSetStorageRollbacker
private final UnstoreDataAction unstoreAction;
- private final File incomingDataSetFile;
+ private final DataSetFile incomingDataSetFile;
private final String dataSetTypeCodeOrNull;
@@ -71,7 +72,7 @@ public class DataSetStorageRollbacker
* @param errorOrNull
*/
public DataSetStorageRollbacker(OmniscientTopLevelDataSetRegistratorState registratorContext,
- Logger operationLog, UnstoreDataAction unstoreAction, File incomingDataSetFile,
+ Logger operationLog, UnstoreDataAction unstoreAction, DataSetFile incomingDataSetFile,
String dataSetTypeCodeOrNull, Throwable errorOrNull)
{
this(registratorContext, operationLog, unstoreAction, incomingDataSetFile,
@@ -89,7 +90,7 @@ public class DataSetStorageRollbacker
* @param errorOrNull
*/
public DataSetStorageRollbacker(OmniscientTopLevelDataSetRegistratorState registratorContext,
- Logger operationLog, UnstoreDataAction unstoreAction, File incomingDataSetFile,
+ Logger operationLog, UnstoreDataAction unstoreAction, DataSetFile incomingDataSetFile,
String dataSetTypeCodeOrNull, Throwable errorOrNull, ErrorType errorTypeOrNull)
{
super();
@@ -123,6 +124,14 @@ public class DataSetStorageRollbacker
public File doRollback(DssRegistrationLogger dssRegistrationLog)
{
dssRegistrationLog.log(getErrorMessageForLog());
+
+ // delete pre-staging copy always - even if not deleting the real incoming directory
+ if (incomingDataSetFile.isLogicalFileSpecified())
+ {
+ FileUtilities.deleteRecursively(incomingDataSetFile.getLogicalIncomingFile()
+ .getParentFile());
+ }
+
if (unstoreAction == UnstoreDataAction.MOVE_TO_ERROR)
{
File newLocation = moveIncomingToError();
@@ -140,16 +149,16 @@ public class DataSetStorageRollbacker
{
dssRegistrationLog.log("File has been deleted.");
- FileUtilities.deleteRecursively(incomingDataSetFile,
+ FileUtilities.deleteRecursively(incomingDataSetFile.getRealIncomingFile(),
new Log4jSimpleLogger(operationLog));
return null;
}
StringBuilder untouchedMessage = new StringBuilder();
untouchedMessage.append("File has been left untouched ");
- untouchedMessage.append(incomingDataSetFile.getAbsolutePath());
+ untouchedMessage.append(incomingDataSetFile.getRealIncomingFile().getAbsolutePath());
dssRegistrationLog.log(untouchedMessage.toString());
- return incomingDataSetFile;
+ return incomingDataSetFile.getRealIncomingFile();
}
public File moveIncomingToError()
@@ -171,13 +180,14 @@ public class DataSetStorageRollbacker
TransferredDataSetHandler.ERROR_DATA_STRATEGY, registratorContext
.getStorageProcessor().getStoreRootDirectory(), registratorContext
.getFileOperations(), dataSetInfo, dataSetInfo.getDataSetType(),
- incomingDataSetFile);
+ incomingDataSetFile.getRealIncomingFile());
baseDirectoryHolder =
new BaseDirectoryHolder(TransferredDataSetHandler.ERROR_DATA_STRATEGY,
- baseDirectory, incomingDataSetFile);
+ baseDirectory, incomingDataSetFile.getRealIncomingFile());
// Move the incoming there
- FileRenamer.renameAndLog(incomingDataSetFile, baseDirectoryHolder.getTargetFile());
+ FileRenamer.renameAndLog(incomingDataSetFile.getRealIncomingFile(),
+ baseDirectoryHolder.getTargetFile());
return baseDirectoryHolder.getTargetFile();
}
@@ -187,7 +197,7 @@ public class DataSetStorageRollbacker
private void writeThrowable()
{
assert errorOrNull != null;
- final String fileName = incomingDataSetFile.getName() + ".exception";
+ final String fileName = incomingDataSetFile.getRealIncomingFile().getName() + ".exception";
final File file = new File(baseDirectoryHolder.getTargetFile().getParentFile(), fileName);
FileWriter writer = null;
try
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/AbstractOmniscientTopLevelDataSetRegistrator.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/AbstractOmniscientTopLevelDataSetRegistrator.java
index 798903b795f48f964b9a1ee6c1cdfd1d9eebd24a..b9a0bd24a9f7519236db2c39ca221afd11ec35a9 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/AbstractOmniscientTopLevelDataSetRegistrator.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/AbstractOmniscientTopLevelDataSetRegistrator.java
@@ -571,8 +571,7 @@ public abstract class AbstractOmniscientTopLevelDataSetRegistrator<T extends Dat
ErrorType.REGISTRATION_SCRIPT_ERROR, ex);
DataSetStorageRollbacker rollbacker =
new DataSetStorageRollbacker(getRegistratorState(), operationLog, action,
- incomingDataSetFile.getRealIncomingFile(), null, ex,
- ErrorType.REGISTRATION_SCRIPT_ERROR);
+ incomingDataSetFile, null, ex, ErrorType.REGISTRATION_SCRIPT_ERROR);
operationLog.info(rollbacker.getErrorMessageForLog());
service.getDssRegistrationLog().log("Processing failed : " + ex.toString());
@@ -612,8 +611,7 @@ public abstract class AbstractOmniscientTopLevelDataSetRegistrator<T extends Dat
ErrorType.INVALID_DATA_SET, null);
DataSetStorageRollbacker rollbacker =
new DataSetStorageRollbacker(getRegistratorState(), operationLog, action,
- incomingDataSetFile.getRealIncomingFile(), null, null,
- ErrorType.INVALID_DATA_SET);
+ incomingDataSetFile, null, null, ErrorType.INVALID_DATA_SET);
sb.append(rollbacker.getErrorMessageForLog());
String logMessage = sb.toString();
operationLog.info(logMessage);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/AbstractProgrammableTopLevelDataSetHandler.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/AbstractProgrammableTopLevelDataSetHandler.java
index 50bfc19a34a3940addcea891f8d2f2bd40204fc0..38bf43f4ef78d4d0645df31887ff0025b385ff49 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/AbstractProgrammableTopLevelDataSetHandler.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/AbstractProgrammableTopLevelDataSetHandler.java
@@ -479,8 +479,8 @@ public abstract class AbstractProgrammableTopLevelDataSetHandler<T extends DataS
state.getOnErrorActionDecision().computeUndoAction(
ErrorType.OPENBIS_REGISTRATION_FAILURE, null);
DataSetStorageRollbacker rollbacker =
- new DataSetStorageRollbacker(state, operationLog, action, recoveryState
- .getIncomingDataSetFile().getRealIncomingFile(), null, null,
+ new DataSetStorageRollbacker(state, operationLog, action,
+ recoveryState.getIncomingDataSetFile(), null, null,
ErrorType.OPENBIS_REGISTRATION_FAILURE);
operationLog.info(rollbacker.getErrorMessageForLog());
rollbacker.doRollback(logger);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/DataSetRegistrationService.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/DataSetRegistrationService.java
index 18d7fd7550500d4c711ffd33ab5172b5b70173e0..9f5c98f2d4cb2393c51bc03adbfd0ab3de589a8f 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/DataSetRegistrationService.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/DataSetRegistrationService.java
@@ -232,7 +232,7 @@ public class DataSetRegistrationService<T extends DataSetInformation> implements
{
DataSetStorageRollbacker rollbacker =
new DataSetStorageRollbacker(registratorContext, operationLog,
- UnstoreDataAction.MOVE_TO_ERROR, incomingDataSetFile.getRealIncomingFile(),
+ UnstoreDataAction.MOVE_TO_ERROR, incomingDataSetFile,
dataSetTypeCodeOrNull, null);
return rollbacker.doRollback(dssRegistrationLog);
}
@@ -249,7 +249,7 @@ public class DataSetRegistrationService<T extends DataSetInformation> implements
registratorContext.getOnErrorActionDecision().computeUndoAction(errorType, ex);
DataSetStorageRollbacker rollbacker =
new DataSetStorageRollbacker(registratorContext, operationLog, action,
- incomingDataSetFile.getRealIncomingFile(), null, ex, errorType);
+ incomingDataSetFile, null, ex, errorType);
operationLog.info(rollbacker.getErrorMessageForLog());
rollbacker.doRollback(dssRegistrationLog);
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/DataSetStorageRollbacker.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/DataSetStorageRollbacker.java
index e6f83c01f96f598b4b48ed1b683ac67fba6e2bf3..8a69ed42f611ce956824a11fd962fb422c0b1bfb 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/DataSetStorageRollbacker.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/v2/DataSetStorageRollbacker.java
@@ -31,6 +31,7 @@ import ch.systemsx.cisd.etlserver.DssRegistrationLogger;
import ch.systemsx.cisd.etlserver.FileRenamer;
import ch.systemsx.cisd.etlserver.IStorageProcessorTransactional.UnstoreDataAction;
import ch.systemsx.cisd.etlserver.TransferredDataSetHandler;
+import ch.systemsx.cisd.etlserver.registrator.DataSetFile;
import ch.systemsx.cisd.etlserver.registrator.v2.AbstractOmniscientTopLevelDataSetRegistrator.OmniscientTopLevelDataSetRegistratorState;
import ch.systemsx.cisd.etlserver.registrator.v2.IDataSetOnErrorActionDecision.ErrorType;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
@@ -50,7 +51,7 @@ public class DataSetStorageRollbacker
private final UnstoreDataAction unstoreAction;
- private final File incomingDataSetFile;
+ private final DataSetFile incomingDataSetFile;
private final String dataSetTypeCodeOrNull;
@@ -71,7 +72,7 @@ public class DataSetStorageRollbacker
* @param errorOrNull
*/
public DataSetStorageRollbacker(OmniscientTopLevelDataSetRegistratorState registratorContext,
- Logger operationLog, UnstoreDataAction unstoreAction, File incomingDataSetFile,
+ Logger operationLog, UnstoreDataAction unstoreAction, DataSetFile incomingDataSetFile,
String dataSetTypeCodeOrNull, Throwable errorOrNull)
{
this(registratorContext, operationLog, unstoreAction, incomingDataSetFile,
@@ -89,7 +90,7 @@ public class DataSetStorageRollbacker
* @param errorOrNull
*/
public DataSetStorageRollbacker(OmniscientTopLevelDataSetRegistratorState registratorContext,
- Logger operationLog, UnstoreDataAction unstoreAction, File incomingDataSetFile,
+ Logger operationLog, UnstoreDataAction unstoreAction, DataSetFile incomingDataSetFile,
String dataSetTypeCodeOrNull, Throwable errorOrNull, ErrorType errorTypeOrNull)
{
super();
@@ -123,6 +124,14 @@ public class DataSetStorageRollbacker
public File doRollback(DssRegistrationLogger dssRegistrationLog)
{
dssRegistrationLog.log(getErrorMessageForLog());
+
+ // delete pre-staging copy always - even if not deleting the real incoming directory
+ if (incomingDataSetFile.isLogicalFileSpecified())
+ {
+ FileUtilities.deleteRecursively(incomingDataSetFile.getLogicalIncomingFile()
+ .getParentFile());
+ }
+
if (unstoreAction == UnstoreDataAction.MOVE_TO_ERROR)
{
File newLocation = moveIncomingToError();
@@ -140,16 +149,16 @@ public class DataSetStorageRollbacker
{
dssRegistrationLog.log("File has been deleted.");
- FileUtilities.deleteRecursively(incomingDataSetFile,
+ FileUtilities.deleteRecursively(incomingDataSetFile.getRealIncomingFile(),
new Log4jSimpleLogger(operationLog));
return null;
}
StringBuilder untouchedMessage = new StringBuilder();
untouchedMessage.append("File has been left untouched ");
- untouchedMessage.append(incomingDataSetFile.getAbsolutePath());
+ untouchedMessage.append(incomingDataSetFile.getRealIncomingFile().getAbsolutePath());
dssRegistrationLog.log(untouchedMessage.toString());
- return incomingDataSetFile;
+ return incomingDataSetFile.getRealIncomingFile();
}
public File moveIncomingToError()
@@ -171,13 +180,14 @@ public class DataSetStorageRollbacker
TransferredDataSetHandler.ERROR_DATA_STRATEGY, registratorContext
.getStorageProcessor().getStoreRootDirectory(), registratorContext
.getFileOperations(), dataSetInfo, dataSetInfo.getDataSetType(),
- incomingDataSetFile);
+ incomingDataSetFile.getRealIncomingFile());
baseDirectoryHolder =
new BaseDirectoryHolder(TransferredDataSetHandler.ERROR_DATA_STRATEGY,
- baseDirectory, incomingDataSetFile);
+ baseDirectory, incomingDataSetFile.getRealIncomingFile());
// Move the incoming there
- FileRenamer.renameAndLog(incomingDataSetFile, baseDirectoryHolder.getTargetFile());
+ FileRenamer.renameAndLog(incomingDataSetFile.getRealIncomingFile(),
+ baseDirectoryHolder.getTargetFile());
return baseDirectoryHolder.getTargetFile();
}
@@ -187,7 +197,7 @@ public class DataSetStorageRollbacker
private void writeThrowable()
{
assert errorOrNull != null;
- final String fileName = incomingDataSetFile.getName() + ".exception";
+ final String fileName = incomingDataSetFile.getRealIncomingFile().getName() + ".exception";
final File file = new File(baseDirectoryHolder.getTargetFile().getParentFile(), fileName);
FileWriter writer = null;
try
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageRollbackerTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageRollbackerTest.java
index 20ee623273d9d79b7ca5b27fcea3e9d4e063402a..e2287c63fbadc14afc8e1abfc8baf44afcd47bb5 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageRollbackerTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageRollbackerTest.java
@@ -37,9 +37,9 @@ import ch.systemsx.cisd.etlserver.ThreadParameters;
import ch.systemsx.cisd.etlserver.TopLevelDataSetRegistratorGlobalState;
import ch.systemsx.cisd.etlserver.registrator.recovery.DataSetStorageRecoveryManager;
import ch.systemsx.cisd.etlserver.registrator.v1.AbstractOmniscientTopLevelDataSetRegistrator;
+import ch.systemsx.cisd.etlserver.registrator.v1.AbstractOmniscientTopLevelDataSetRegistrator.OmniscientTopLevelDataSetRegistratorState;
import ch.systemsx.cisd.etlserver.registrator.v1.DataSetRegistrationService;
import ch.systemsx.cisd.etlserver.registrator.v1.DataSetStorageRollbacker;
-import ch.systemsx.cisd.etlserver.registrator.v1.AbstractOmniscientTopLevelDataSetRegistrator.OmniscientTopLevelDataSetRegistratorState;
import ch.systemsx.cisd.etlserver.registrator.v1.IDataSetOnErrorActionDecision.ErrorType;
import ch.systemsx.cisd.etlserver.validation.IDataSetValidator;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
@@ -65,7 +65,7 @@ public class DataSetStorageRollbackerTest extends AbstractFileSystemTestCase
private IDataSetValidator dataSetValidator;
- private File incomingDataSetFile;
+ private DataSetFile incomingDataSetFile;
private TestDataSetRegistrator testRegistrator;
@@ -89,7 +89,8 @@ public class DataSetStorageRollbackerTest extends AbstractFileSystemTestCase
@Test
public void testLogging()
{
- incomingDataSetFile = createDirectory(workingDirectory, "data_set");
+ File realIncoming = createDirectory(workingDirectory, "data_set");
+ incomingDataSetFile = new DataSetFile(realIncoming);
OmniscientTopLevelDataSetRegistratorState globalState =
testRegistrator.getRegistratorState();
DataSetStorageRollbacker rollbacker =