From b65d5b29202cd4c28f216142fe8cf58d12b7e7c0 Mon Sep 17 00:00:00 2001
From: gpawel <gpawel>
Date: Thu, 21 Apr 2011 08:54:53 +0000
Subject: [PATCH] [LMS-2205] Rsync Archiver - allow to choose between simple
and paranoid checks
SVN: 21027
---
.../RemoteDataSetFileOperationsExecutor.java | 6 +-
.../server/plugins/demo/DemoArchiver.java | 10 ++-
.../AbstractArchiverProcessingPlugin.java | 84 ++++++++++++-------
.../DataSetFileOperationsManager.java | 30 +++++--
.../IDataSetFileOperationsManager.java | 13 +--
.../plugins/standard/RsyncArchiver.java | 11 ++-
.../cisd/yeastx/etl/MLArchiverTask.java | 9 +-
7 files changed, 112 insertions(+), 51 deletions(-)
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/RemoteDataSetFileOperationsExecutor.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/RemoteDataSetFileOperationsExecutor.java
index f337e229457..458458c8931 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/RemoteDataSetFileOperationsExecutor.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/RemoteDataSetFileOperationsExecutor.java
@@ -192,8 +192,7 @@ public final class RemoteDataSetFileOperationsExecutor implements IDataSetFileOp
{
return BooleanStatus.createFalse("Data set location '" + dataSet + "' doesn't exist");
}
- BooleanStatus existsStatus =
- executor.exists(destination.getPath(), timeoutInMillis);
+ BooleanStatus existsStatus = executor.exists(destination.getPath(), timeoutInMillis);
if (false == existsStatus.isSuccess())
{
return existsStatus;
@@ -203,8 +202,7 @@ public final class RemoteDataSetFileOperationsExecutor implements IDataSetFileOp
Map<String, Long> dataSetFileSizesByPaths =
FolderFileSizesReportGenerator.extractSizesByPaths(storeFiles, dataSet);
String cmd = createListFilesWithFileSizeCmd(destination.getPath(), gfindExecutable);
- ProcessResult result =
- executor.executeCommandRemotely(cmd, timeoutInMillis);
+ ProcessResult result = executor.executeCommandRemotely(cmd, timeoutInMillis);
if (result.isOK() == false)
{
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/demo/DemoArchiver.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/demo/DemoArchiver.java
index 696ee55bc60..146f3ea49b1 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/demo/DemoArchiver.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/demo/DemoArchiver.java
@@ -63,13 +63,20 @@ public class DemoArchiver extends AbstractArchiverProcessingPlugin
}
@Override
- public BooleanStatus isDataSetPresentInArchive(DatasetDescription dataset,
+ public BooleanStatus isDataSetSynchronizedWithArchive(DatasetDescription dataset,
ArchiverTaskContext context)
{
boolean present = archiveContents.contains(dataset.getDataSetCode());
return BooleanStatus.createFromBoolean(present);
}
+ @Override
+ protected BooleanStatus isDataSetPresentInArchive(DatasetDescription dataset)
+ {
+ boolean present = archiveContents.contains(dataset.getDataSetCode());
+ return BooleanStatus.createFromBoolean(present);
+ }
+
@Override
public DatasetProcessingStatuses doDeleteFromArchive(List<DatasetLocation> dataSets)
{
@@ -83,5 +90,4 @@ public class DemoArchiver extends AbstractArchiverProcessingPlugin
}
return statuses;
}
-
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java
index 470794d189c..0ef20cbd490 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java
@@ -74,17 +74,23 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
private static final long serialVersionUID = 1L;
- @Private public static final String SHARE_FINDER_KEY = "share-finder";
-
+ @Private
+ public static final String SHARE_FINDER_KEY = "share-finder";
+
+ private static final String SYNCHRONIZE_ARCHIVE = "synchronize-archive";
+
private final IStatusChecker archivePrerequisiteOrNull;
private final IStatusChecker unarchivePrerequisiteOrNull;
-
+
+ private final boolean synchronizeArchive;
+
private transient IShareIdManager shareIdManager;
-
+
private transient IEncapsulatedOpenBISService service;
-
- @Private transient IDataSetStatusUpdater statusUpdater;
+
+ @Private
+ transient IDataSetStatusUpdater statusUpdater;
public AbstractArchiverProcessingPlugin(Properties properties, File storeRoot,
IStatusChecker archivePrerequisiteOrNull, IStatusChecker unarchivePrerequisiteOrNull)
@@ -92,8 +98,13 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
super(properties, storeRoot);
this.archivePrerequisiteOrNull = archivePrerequisiteOrNull;
this.unarchivePrerequisiteOrNull = unarchivePrerequisiteOrNull;
+
+ properties.getProperty(SYNCHRONIZE_ARCHIVE, Boolean.TRUE.toString());
+ this.synchronizeArchive =
+ Boolean.parseBoolean(properties.getProperty(SYNCHRONIZE_ARCHIVE,
+ Boolean.TRUE.toString()));
}
-
+
/**
* NOTE: this method is not allowed to throw exception as this will leave data sets in the
* openBIS database with an inconsistent status.
@@ -103,11 +114,9 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
/**
* NOTE: this method is not allowed to throw exception as this will leave data sets in the
- * openBIS database with an inconsistent status.
- *
- * Implementations of this method should invoke
- * <code>context.getUnarchivingPreparation().prepareForUnarchiving()</code> for each
- * data set before doing the actual unarchiving.
+ * openBIS database with an inconsistent status. Implementations of this method should invoke
+ * <code>context.getUnarchivingPreparation().prepareForUnarchiving()</code> for each data set
+ * before doing the actual unarchiving.
*/
abstract protected DatasetProcessingStatuses doUnarchive(List<DatasetDescription> datasets,
ArchiverTaskContext context);
@@ -117,13 +126,19 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
* exist in the openBIS database.
*/
abstract protected DatasetProcessingStatuses doDeleteFromArchive(List<DatasetLocation> datasets);
-
+
+ /**
+ * @return <code>true</code> if the dataset is present and synchronized with the archive,
+ * <code>false</code> otherwise.
+ */
+ abstract protected BooleanStatus isDataSetSynchronizedWithArchive(DatasetDescription dataset,
+ ArchiverTaskContext context);
+
/**
* @return <code>true</code> if the dataset is present in the archive, <code>false</code>
* otherwise.
*/
- abstract protected BooleanStatus isDataSetPresentInArchive(DatasetDescription dataset,
- ArchiverTaskContext context);
+ abstract protected BooleanStatus isDataSetPresentInArchive(DatasetDescription dataset);
public ProcessingStatus archive(List<DatasetDescription> datasets,
final ArchiverTaskContext context, boolean removeFromDataStore)
@@ -194,16 +209,18 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
final ArchiverTaskContext context, boolean removeFromDataStore)
{
- GroupedDatasets groupedDatasets = groupByPresenceInArchive(datasets, context);
+ GroupedDatasets groupedDatasets =
+ groupByPresenceInArchive(datasets, context, synchronizeArchive);
+
List<DatasetDescription> notPresentInArchive = groupedDatasets.getNotPresentAsList();
DatasetProcessingStatuses statuses = new DatasetProcessingStatuses();
- if (notPresentInArchive.isEmpty() == false)
+ if (!notPresentInArchive.isEmpty())
{
// copy data sets in the archive
statuses = doArchive(notPresentInArchive, context);
// paranoid check to make sure everything really got archived
- groupedDatasets = groupByPresenceInArchive(datasets, context);
+ groupedDatasets = groupByPresenceInArchive(datasets, context, true);
}
if (removeFromDataStore)
@@ -211,7 +228,7 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
// only remove the when we are sure we have got a backup in the archive
removeFromDataStore(groupedDatasets.getPresentInArchive(), context);
}
-
+
// merge the archiver statuses with the paranoid check results
return mergeArchiveStatuses(statuses, groupedDatasets);
}
@@ -263,7 +280,7 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
IDataSetDeleter dataSetDeleter = ServiceProvider.getDataStoreService().getDataSetDeleter();
dataSetDeleter.scheduleDeletionOfDataSets(datasets);
}
-
+
public ProcessingStatus unarchive(List<DatasetDescription> datasets, ArchiverTaskContext context)
{
operationLog.info("Unarchiving of the following datasets has been requested: "
@@ -499,26 +516,30 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
}
}
- protected GroupedDatasets groupByPresenceInArchive(List<DatasetDescription> datasets,
- ArchiverTaskContext context)
+ private GroupedDatasets groupByPresenceInArchive(List<DatasetDescription> datasets,
+ ArchiverTaskContext context, boolean checkIfSynchronized)
{
List<DatasetDescription> present = new ArrayList<DatasetDescription>();
- Map<DatasetDescription, BooleanStatus> notPresent = new HashMap<DatasetDescription, BooleanStatus>();
-
- for (DatasetDescription dataset : datasets)
+ Map<DatasetDescription, BooleanStatus> notPresent =
+ new HashMap<DatasetDescription, BooleanStatus>();
+
+ for (DatasetDescription dataset : datasets)
{
- BooleanStatus presentStatus = isDataSetPresentInArchive(dataset, context);
+ BooleanStatus presentStatus =
+ checkIfSynchronized ? isDataSetSynchronizedWithArchive(dataset, context)
+ : isDataSetPresentInArchive(dataset);
if (presentStatus.isSuccess())
{
present.add(dataset);
- } else {
+ } else
+ {
notPresent.put(dataset, presentStatus);
}
}
-
+
return new GroupedDatasets(present, notPresent);
}
-
+
private IShareIdManager getShareIdManager()
{
if (shareIdManager == null)
@@ -527,7 +548,7 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
}
return shareIdManager;
}
-
+
private IShareFinder getShareFinder()
{
Properties props =
@@ -545,8 +566,7 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
return shareFinder;
}
-
-
+
private IEncapsulatedOpenBISService getService()
{
if (service == null)
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManager.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManager.java
index f09611dc0f9..99c6b9c2273 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManager.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManager.java
@@ -117,7 +117,7 @@ public class DataSetFileOperationsManager implements IDataSetFileOperationsManag
File sshExecutable = Copier.getExecutable(properties, SSH_EXEC);
File rsyncExecutable = Copier.getExecutable(properties, RSYNC_EXEC);
File gfindExecutable = Copier.getExecutable(properties, GFIND_EXEC);
-
+
IPathCopier copier =
pathCopierFactory.create(rsyncExecutable, sshExecutable, timeoutInMillis);
copier.check();
@@ -212,10 +212,10 @@ public class DataSetFileOperationsManager implements IDataSetFileOperationsManag
}
/**
- * Checks if specified dataset's data are present in the destination specified in constructor.
- * The path at the destination is defined by original location of the data set.
+ * Checks if specified dataset's data are present and synchronized in the destination specified
+ * in constructor. The path at the destination is defined by original location of the data set.
*/
- public BooleanStatus isPresentInDestination(File originalData, DatasetDescription dataset)
+ public BooleanStatus isSynchronizedWithDestination(File originalData, DatasetDescription dataset)
{
try
{
@@ -231,7 +231,28 @@ public class DataSetFileOperationsManager implements IDataSetFileOperationsManag
{
return BooleanStatus.createError(ex.getStatus().tryGetErrorMessage());
}
+ }
+ /**
+ * Checks if specified dataset's data are present in the destination specified in constructor.
+ * The path at the destination is defined by original location of the data set.
+ */
+ public BooleanStatus isPresentInDestination(DatasetDescription dataset)
+ {
+ try
+ {
+ File destinationFolder = new File(destination, dataset.getDataSetLocation());
+ BooleanStatus resultStatus = executor.exists(destinationFolder);
+ String message = resultStatus.tryGetMessage();
+ if (message != null) // if there is a message something went wrong
+ {
+ operationLog.error(message);
+ }
+ return resultStatus;
+ } catch (ExceptionWithStatus ex)
+ {
+ return BooleanStatus.createError(ex.getStatus().tryGetErrorMessage());
+ }
}
private void checkDestinationExists(File destinationFolder)
@@ -266,5 +287,4 @@ public class DataSetFileOperationsManager implements IDataSetFileOperationsManag
}
return destinationExists;
}
-
}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/IDataSetFileOperationsManager.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/IDataSetFileOperationsManager.java
index c67929fa905..7749d1be1c2 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/IDataSetFileOperationsManager.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/IDataSetFileOperationsManager.java
@@ -24,8 +24,6 @@ import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DatasetLocation;
import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
/**
- *
- *
* @author Franz-Josef Elmer
*/
public interface IDataSetFileOperationsManager
@@ -50,10 +48,15 @@ public interface IDataSetFileOperationsManager
public abstract Status deleteFromDestination(DatasetLocation dataset);
/**
- * Checks if specified dataset's data are present in the destination specified in constructor.
- * The path at the destination is defined by original location of the data set.
+ * Checks if specified dataset's data are present and synchronized in the destination specified
+ * in constructor. The path at the destination is defined by original location of the data set.
*/
- public abstract BooleanStatus isPresentInDestination(File originalData,
+ public abstract BooleanStatus isSynchronizedWithDestination(File originalData,
DatasetDescription dataset);
+ /**
+ * Checks if specified dataset's data are present in the destination specified in constructor.
+ * The path at the destination is defined by original location of the data set.
+ */
+ public abstract BooleanStatus isPresentInDestination(DatasetDescription dataset);
}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java
index 2b5aa222599..61829d55a67 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java
@@ -105,11 +105,17 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
}
@Override
- protected BooleanStatus isDataSetPresentInArchive(DatasetDescription dataset,
+ protected BooleanStatus isDataSetSynchronizedWithArchive(DatasetDescription dataset,
ArchiverTaskContext context)
{
File originalData = getDatasetDirectory(context, dataset);
- return fileOperationsManager.isPresentInDestination(originalData, dataset);
+ return fileOperationsManager.isSynchronizedWithDestination(originalData, dataset);
+ }
+
+ @Override
+ protected BooleanStatus isDataSetPresentInArchive(DatasetDescription dataset)
+ {
+ return fileOperationsManager.isPresentInDestination(dataset);
}
private Status doArchive(DatasetDescription dataset, File originalData)
@@ -131,4 +137,5 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
{
return context.getDirectoryProvider().getDataSetDirectory(dataset);
}
+
}
diff --git a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/MLArchiverTask.java b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/MLArchiverTask.java
index aef5e2f9d34..f86b63128a5 100644
--- a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/MLArchiverTask.java
+++ b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/MLArchiverTask.java
@@ -213,13 +213,20 @@ public class MLArchiverTask extends AbstractArchiverProcessingPlugin
}
@Override
- public BooleanStatus isDataSetPresentInArchive(DatasetDescription dataset,
+ public BooleanStatus isDataSetSynchronizedWithArchive(DatasetDescription dataset,
ArchiverTaskContext context)
{
// data is always present, since there is no archive
return BooleanStatus.createTrue();
}
+ @Override
+ protected BooleanStatus isDataSetPresentInArchive(DatasetDescription dataset)
+ {
+ // data is always present, since there is no archive
+ return BooleanStatus.createTrue();
+ }
+
@Override
protected void removeFromDataStore(List<DatasetDescription> datasets,
ArchiverTaskContext context)
--
GitLab