From 6814a12f7813d52815f5ad3aac46b133d58b8b31 Mon Sep 17 00:00:00 2001
From: jakubs <jakubs>
Date: Mon, 22 Sep 2014 11:30:25 +0000
Subject: [PATCH] SSDM-891 Store archiving status of datasets on the fly -
without waiting for the full operation to complete on all datasets
SVN: 32468
---
.../AbstractArchiverProcessingPlugin.java | 74 ++++++++++++++++++-
.../dss/generic/shared/ProcessingStatus.java | 9 ++-
2 files changed, 80 insertions(+), 3 deletions(-)
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java
index 477c889b91c..14ffe4d38e6 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java
@@ -22,6 +22,7 @@ import static ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetArchiving
import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Properties;
@@ -140,14 +141,29 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
operationLog.info("Archiving of the following datasets has been requested: "
+ CollectionUtils.abbreviate(datasets, 10));
- DatasetProcessingStatuses statuses = safeArchive(datasets, context, removeFromDataStore);
+ DatasetProcessingStatuses finalstatuses = new DatasetProcessingStatuses();
+
+ for (DatasetDescription singleDatset : datasets)
+ {
+ List<DatasetDescription> singleBatch = Collections.singletonList(singleDatset);
+
+ archiveSingleBatch(context, removeFromDataStore, finalstatuses, singleBatch);
+ }
+
+ return finalstatuses.getProcessingStatus();
+ }
+
+ private void archiveSingleBatch(final ArchiverTaskContext context, boolean removeFromDataStore, DatasetProcessingStatuses finalstatuses,
+ List<DatasetDescription> singleBatch)
+ {
+ DatasetProcessingStatuses statuses = safeArchive(singleBatch, context, removeFromDataStore);
DataSetArchivingStatus successStatus = (removeFromDataStore) ? ARCHIVED : AVAILABLE;
asyncUpdateStatuses(statuses.getSuccessfulDatasetCodes(), successStatus, true);
asyncUpdateStatuses(statuses.getFailedDatasetCodes(), AVAILABLE, false);
- return statuses.getProcessingStatus();
+ finalstatuses.addResults(statuses);
}
private void initializeDatasetSizesIfNeeded(List<DatasetDescription> datasets)
@@ -392,6 +408,25 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
this.processingStatus = new ProcessingStatus();
}
+ public void addResults(DatasetProcessingStatuses otherStatuses)
+ {
+ ProcessingStatus otherProcessingStatus = otherStatuses.getProcessingStatus();
+
+ for (String datasetCode : otherProcessingStatus.getDatasetsByStatus(Status.OK))
+ {
+ addResultQuietly(datasetCode, Status.OK);
+ }
+
+ for (Status error : otherProcessingStatus.getErrorStatuses())
+ {
+ for (String datasetCode : otherProcessingStatus.getDatasetsByStatus(error))
+ {
+ addResultQuietly(datasetCode, error);
+ }
+ }
+
+ }
+
public void addResult(Collection<DatasetDescription> datasets, Status status,
Operation operation)
{
@@ -421,6 +456,18 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
processingStatus.addDatasetStatus(datasetCode, status);
}
+ private void addResultQuietly(String datasetCode, Status status)
+ {
+ if (status.isError())
+ {
+ failedDatasetCodes.add(datasetCode);
+ } else
+ {
+ successfulDatasetCodes.add(datasetCode);
+ }
+ processingStatus.addDatasetStatus(datasetCode, status);
+ }
+
private String createLogMessage(String datasetCode, Status status, String operation)
{
return String.format("%s for dataset %s finished with the status: %s.", operation,
@@ -443,6 +490,29 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
}
}
+ protected final static DatasetProcessingStatuses createStatuesFailIfAnyFailed(DatasetProcessingStatuses statuses)
+ {
+ if (statuses.getFailedDatasetCodes().isEmpty())
+ return statuses;
+ DatasetProcessingStatuses result = new DatasetProcessingStatuses();
+
+ ProcessingStatus otherProcessingStatus = statuses.getProcessingStatus();
+
+ for (Status error : otherProcessingStatus.getErrorStatuses())
+ {
+ for (String datasetCode : otherProcessingStatus.getDatasetsByStatus(error))
+ {
+ result.addResultQuietly(datasetCode, error);
+ }
+ }
+
+ for (String datasetCode : otherProcessingStatus.getDatasetsByStatus(Status.OK))
+ {
+ result.addResultQuietly(datasetCode, Status.createError("Successful, but part of a failed batch."));
+ }
+ return result;
+ }
+
protected final static DatasetProcessingStatuses createStatuses(Status status,
List<DatasetDescription> datasets, Operation operation)
{
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/ProcessingStatus.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/ProcessingStatus.java
index cd55267f5e7..241c5eab7ae 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/ProcessingStatus.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/ProcessingStatus.java
@@ -17,6 +17,7 @@
package ch.systemsx.cisd.openbis.dss.generic.shared;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -56,7 +57,13 @@ public class ProcessingStatus
public List<String/* dataset code */> getDatasetsByStatus(Status status)
{
- return datasetByStatus.get(status);
+ if (datasetByStatus.containsKey(status))
+ {
+ return datasetByStatus.get(status);
+ } else
+ {
+ return Collections.emptyList();
+ }
}
public void addDatasetStatus(DatasetDescription dataset, Status status)
--
GitLab