From 2894282074f03814c6fc151542f261ebd816bb1a Mon Sep 17 00:00:00 2001
From: gakin <gakin>
Date: Tue, 16 May 2017 08:06:34 +0000
Subject: [PATCH] SSDM-5025 : Use parallelized execution prefs from harvester
config in data set and attachment synchronization
SVN: 38189
---
.../sync/harvester/config/SyncConfig.java | 5 +++++
.../synchronizer/EntitySynchronizer.java | 17 ++++++++++++-----
2 files changed, 17 insertions(+), 5 deletions(-)
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/config/SyncConfig.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/config/SyncConfig.java
index 6811f614b8e..d2972f767bd 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/config/SyncConfig.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/config/SyncConfig.java
@@ -328,4 +328,9 @@ public class SyncConfig
{
this.parallelizedExecutionPrefs = new ParallelizedExecutionPreferences(machineLoad, maxThreads, retriesOnFailure, stopOnFailure);
}
+
+ public ParallelizedExecutionPreferences getParallelizedExecutionPrefs()
+ {
+ return parallelizedExecutionPrefs;
+ }
}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
index 0754b1b2d5e..76c95d810df 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
@@ -64,6 +64,7 @@ import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.EntityRetrieve
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.SyncEntityKind;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.entitygraph.EntityGraph;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.entitygraph.Node;
+import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.config.ParallelizedExecutionPreferences;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.config.SyncConfig;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.ResourceListParserData.Connection;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.ResourceListParserData.IncomingDataSet;
@@ -444,10 +445,13 @@ public class EntitySynchronizer
{
AttachmentSynchronizationSummary synchronizationSummary = new AttachmentSynchronizationSummary();
+ ParallelizedExecutionPreferences preferences = config.getParallelizedExecutionPrefs();
+
ParallelizedExecutor.process(attachmentHoldersToProcess, new AttachmentSynchronizationTaskExecutor(synchronizationSummary,
service,
lastSyncTimestamp, config),
- 0.5, 10, "process attachments", 0, false);
+ preferences.getMachineLoad(), preferences.getMaxThreads(), "process attachments", preferences.getRetriesOnFail(),
+ preferences.isStopOnFailure());
return synchronizationSummary;
}
@@ -623,14 +627,17 @@ public class EntitySynchronizer
private DataSetSynchronizationSummary registerPhysicalDataSets(Map<String, IncomingDataSet> physicalDSMap) throws IOException
{
List<IncomingDataSet> dsList = new ArrayList<IncomingDataSet>(physicalDSMap.values());
- DataSetSynchronizationSummary dsRegistrationSummary = new DataSetSynchronizationSummary();
+ DataSetSynchronizationSummary dataSetSynchronizationSummary = new DataSetSynchronizationSummary();
// This parallelization is possible because each DS is registered without dependencies
// and the dependencies are established later on in the sync process.
- ParallelizedExecutor.process(dsList, new DataSetRegistrationTaskExecutor(dsRegistrationSummary),
- 0.5, 10, "register data sets", 0, false);
+ ParallelizedExecutionPreferences preferences = config.getParallelizedExecutionPrefs();
+
+ ParallelizedExecutor.process(dsList, new DataSetRegistrationTaskExecutor(dataSetSynchronizationSummary),
+ preferences.getMachineLoad(), preferences.getMaxThreads(), "register data sets", preferences.getRetriesOnFail(),
+ preferences.isStopOnFailure());
- return dsRegistrationSummary;
+ return dataSetSynchronizationSummary;
}
private void saveFailedEntitiesFile(List<String> notRegisteredDataSetCodes, List<String> notSyncedAttachmentsHolders) throws IOException
--
GitLab