From 1f8041100518fe58e5d1716821419eaa57f2a82a Mon Sep 17 00:00:00 2001
From: felmer <franz-josef.elmer@id.ethz.ch>
Date: Tue, 20 Nov 2018 11:20:21 +0100
Subject: [PATCH] SSDM-7430: Refactoring and performance improvements
---
.../harvester/HarvesterMaintenanceTask.java | 5 +
.../harvester/synchronizer/Connection.java | 41 ++
.../synchronizer/ConnectionType.java | 36 ++
.../synchronizer/EntitySynchronizer.java | 120 +++---
.../synchronizer/IncomingDataSet.java | 74 ++++
.../synchronizer/IncomingEntity.java | 95 +++++
.../synchronizer/IncomingExperiment.java | 34 ++
.../synchronizer/IncomingProject.java | 35 ++
.../synchronizer/IncomingSample.java | 35 ++
.../harvester/synchronizer/MasterData.java | 144 +++++++
.../synchronizer/MasterDataSynchronizer.java | 12 +-
.../MaterialWithLastModificationDate.java | 44 +++
.../synchronizer/ResourceListParser.java | 7 -
.../synchronizer/ResourceListParserData.java | 365 +-----------------
.../synchronizer/SynchronizationContext.java | 13 +
...AttachmentSynchronizationTaskExecutor.java | 43 ++-
.../DataSetRegistrationTaskExecutor.java | 2 +-
.../harvester/synchronizer/util/V3Utils.java | 11 +-
.../server/EncapsulatedOpenBISService.java | 7 +
19 files changed, 679 insertions(+), 444 deletions(-)
create mode 100644 datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/Connection.java
create mode 100644 datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ConnectionType.java
create mode 100644 datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/IncomingDataSet.java
create mode 100644 datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/IncomingEntity.java
create mode 100644 datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/IncomingExperiment.java
create mode 100644 datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/IncomingProject.java
create mode 100644 datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/IncomingSample.java
create mode 100644 datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MasterData.java
create mode 100644 datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MaterialWithLastModificationDate.java
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/HarvesterMaintenanceTask.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/HarvesterMaintenanceTask.java
index 3283983217a..6e46fbb15ac 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/HarvesterMaintenanceTask.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/HarvesterMaintenanceTask.java
@@ -45,6 +45,7 @@ import org.apache.log4j.DailyRollingFileAppender;
import org.apache.log4j.Logger;
import org.apache.log4j.PatternLayout;
+import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.SyncEntityKind;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.config.ConfigReader;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.config.SyncConfig;
@@ -93,6 +94,8 @@ public class HarvesterMaintenanceTask<T extends DataSetInformation> implements I
private IEncapsulatedOpenBISService service;
+ private IApplicationServerApi v3Api;
+
private DataSetProcessingContext context;
private File harvesterConfigFile;
@@ -119,6 +122,7 @@ public class HarvesterMaintenanceTask<T extends DataSetInformation> implements I
public void setUp(String pluginName, Properties properties)
{
service = ServiceProvider.getOpenBISService();
+ v3Api = ServiceProvider.getV3ApplicationService();
context = new DataSetProcessingContext(null, null, null, null, null, null);
dataStoreCode = getConfigProvider().getDataStoreCode();
storeRoot = new File(DssPropertyParametersUtil.loadServiceProperties().getProperty(PluginTaskInfoProvider.STOREROOT_DIR_KEY));
@@ -211,6 +215,7 @@ public class HarvesterMaintenanceTask<T extends DataSetInformation> implements I
SynchronizationContext syncContext = new SynchronizationContext();
syncContext.setService(service);
+ syncContext.setV3Api(v3Api);
syncContext.setDataStoreCode(dataStoreCode);
syncContext.setStoreRoot(storeRoot);
syncContext.setLastSyncTimestamp(cutOffTimestamp);
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/Connection.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/Connection.java
new file mode 100644
index 00000000000..5a8bd1b4238
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/Connection.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2018 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer;
+
+class Connection
+{
+ final String toPermId;
+
+ final String connType;
+
+ public String getType()
+ {
+ return connType;
+ }
+
+ Connection(String toPermId, String connType)
+ {
+ super();
+ this.toPermId = toPermId;
+ this.connType = connType;
+ }
+
+ public String getToPermId()
+ {
+ return toPermId;
+ }
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ConnectionType.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ConnectionType.java
new file mode 100644
index 00000000000..b96a6257f05
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ConnectionType.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2018 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer;
+
+enum ConnectionType
+{
+ SIMPLE_CONNECTION("Connection"),
+ PARENT_CHILD_RELATIONSHIP("Child"),
+ CONTAINER_COMPONENT_RELATIONSHIP("Component");
+
+ private final String type;
+
+ public String getType()
+ {
+ return type;
+ }
+
+ private ConnectionType(String type)
+ {
+ this.type = type;
+ }
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
index c52737eaa3d..9e8d3e6725c 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
@@ -32,6 +32,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.stream.Collectors;
import javax.xml.xpath.XPathExpressionException;
@@ -39,8 +40,6 @@ import org.apache.commons.codec.binary.Hex;
import org.apache.commons.collections4.map.MultiKeyMap;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.time.StopWatch;
-import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.w3c.dom.Document;
@@ -57,6 +56,8 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.material.id.MaterialPermId;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.project.delete.ProjectDeletionOptions;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.project.id.ProjectPermId;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.delete.SampleDeletionOptions;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.fetchoptions.SampleFetchOptions;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.id.ISampleId;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.id.SamplePermId;
import ch.ethz.sis.openbis.generic.dssapi.v3.IDataStoreServerApi;
import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile;
@@ -69,14 +70,6 @@ import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.entitygraph.IE
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.entitygraph.INode;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.config.ParallelizedExecutionPreferences;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.config.SyncConfig;
-import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.ResourceListParserData.Connection;
-import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.ResourceListParserData.IncomingDataSet;
-import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.ResourceListParserData.IncomingEntity;
-import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.ResourceListParserData.IncomingExperiment;
-import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.ResourceListParserData.IncomingProject;
-import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.ResourceListParserData.IncomingSample;
-import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.ResourceListParserData.MasterData;
-import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.ResourceListParserData.MaterialWithLastModificationDate;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.datasourceconnector.DataSourceConnector;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.parallelizedExecutor.AttachmentSynchronizationSummary;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.parallelizedExecutor.AttachmentSynchronizationTaskExecutor;
@@ -87,7 +80,6 @@ import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronize
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.util.DSPropertyUtils;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.util.Monitor;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.util.V3Utils;
-import ch.systemsx.cisd.cifex.shared.basic.UserFailureException;
import ch.systemsx.cisd.common.concurrent.ParallelizedExecutor;
import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.common.logging.Log4jSimpleLogger;
@@ -150,6 +142,8 @@ public class EntitySynchronizer
private final IEncapsulatedOpenBISService service;
+ private final IApplicationServerApi v3Api;
+
private final DataSetProcessingContext context;
private final Date lastSyncTimestamp;
@@ -170,17 +164,18 @@ public class EntitySynchronizer
public EntitySynchronizer(SynchronizationContext synContext)
{
- this.service = synContext.getService();
- this.dataStoreCode = synContext.getDataStoreCode();
- this.storeRoot = synContext.getStoreRoot();
- this.lastSyncTimestamp = synContext.getLastSyncTimestamp();
- this.lastIncSyncTimestamp = synContext.getLastIncSyncTimestamp();
- this.dataSetsCodesToRetry = synContext.getDataSetsCodesToRetry();
- this.attachmentHolderCodesToRetry = synContext.getAttachmentHolderCodesToRetry();
- this.blackListedDataSetCodes = synContext.getBlackListedDataSetCodes();
- this.context = synContext.getContext();
- this.config = synContext.getConfig();
- this.operationLog = synContext.getOperationLog();
+ service = synContext.getService();
+ v3Api = synContext.getV3Api();
+ dataStoreCode = synContext.getDataStoreCode();
+ storeRoot = synContext.getStoreRoot();
+ lastSyncTimestamp = synContext.getLastSyncTimestamp();
+ lastIncSyncTimestamp = synContext.getLastIncSyncTimestamp();
+ dataSetsCodesToRetry = synContext.getDataSetsCodesToRetry();
+ attachmentHolderCodesToRetry = synContext.getAttachmentHolderCodesToRetry();
+ blackListedDataSetCodes = synContext.getBlackListedDataSetCodes();
+ context = synContext.getContext();
+ config = synContext.getConfig();
+ operationLog = synContext.getOperationLog();
}
public Date synchronizeEntities() throws Exception
@@ -188,7 +183,7 @@ public class EntitySynchronizer
Document doc = getResourceList();
ResourceListParserData data = parseResourceList(doc);
- processDeletions(data);
+// processDeletions(data);
registerMasterData(data.getMasterData());
MultiKeyMap<String, String> newEntities = registerEntities(data);
List<String> notSyncedAttachmentsHolders = registerAttachments(data, newEntities);
@@ -249,7 +244,7 @@ public class EntitySynchronizer
operationLog.info("Processing attachments...");
List<IncomingEntity<?>> attachmentHoldersToProcess =
data.filterAttachmentHoldersByLastModificationDate(lastSyncTimestamp, attachmentHolderCodesToRetry);
-
+ monitor.log(attachmentHoldersToProcess.size() + " to process");
if (config.isVerbose())
{
verboseLogProcessAttachments(attachmentHoldersToProcess, newEntities);
@@ -258,7 +253,7 @@ public class EntitySynchronizer
List<String> notSyncedAttachmentsHolders = new ArrayList<String>();
if (config.isDryRun() == false)
{
- AttachmentSynchronizationSummary syncSummary = processAttachments(attachmentHoldersToProcess);
+ AttachmentSynchronizationSummary syncSummary = processAttachments(attachmentHoldersToProcess, monitor);
notSyncedAttachmentsHolders = syncSummary.notRegisteredAttachmentHolderCodes;
operationLog.info("Attachment synchronization summary:\n" + syncSummary.addedCount + " attachment(s) were added.\n"
+ syncSummary.updatedCount + " attachment(s) were updated.\n"
@@ -514,15 +509,20 @@ public class EntitySynchronizer
}
}
- private AttachmentSynchronizationSummary processAttachments(List<IncomingEntity<?>> attachmentHoldersToProcess)
+ private AttachmentSynchronizationSummary processAttachments(List<IncomingEntity<?>> attachmentHoldersToProcess,
+ Monitor monitor)
{
AttachmentSynchronizationSummary synchronizationSummary = new AttachmentSynchronizationSummary();
ParallelizedExecutionPreferences preferences = config.getParallelizedExecutionPrefs();
-
+ String asUrl = config.getDataSourceOpenbisURL();
+ String dssUrl = config.getDataSourceDSSURL();
+ monitor.log("AS URL: " + asUrl + ", DSS URL: " + dssUrl);
+ V3Utils v3Utils = V3Utils.create(asUrl, dssUrl);
+ monitor.log("Services for accessing data source established");
ParallelizedExecutor.process(attachmentHoldersToProcess, new AttachmentSynchronizationTaskExecutor(synchronizationSummary,
- service,
- lastSyncTimestamp, config),
+ service, v3Utils,
+ lastSyncTimestamp, config, monitor),
preferences.getMachineLoad(), preferences.getMaxThreads(), "process attachments", preferences.getRetriesOnFail(),
preferences.isStopOnFailure());
@@ -644,25 +644,25 @@ public class EntitySynchronizer
if (sampleIdentifier != null)
{
Sample sampleWithExperiment = service.tryGetSampleWithExperiment(sampleIdentifier);
- dsBatchUpdatesDTO.setSampleIdentifierOrNull(SampleIdentifierFactory.parse(sampleWithExperiment.getIdentifier()));
- dsBatchUpdatesDTO.getDetails().setSampleUpdateRequested(true);
+// dsBatchUpdatesDTO.setSampleIdentifierOrNull(SampleIdentifierFactory.parse(sampleWithExperiment.getIdentifier()));
+ dsBatchUpdatesDTO.setSampleIdentifierOrNull(sampleIdentifier);
} else
{
dsBatchUpdatesDTO.setSampleIdentifierOrNull(null);
- dsBatchUpdatesDTO.getDetails().setSampleUpdateRequested(true);
}
+ dsBatchUpdatesDTO.getDetails().setSampleUpdateRequested(true);
ExperimentIdentifier expIdentifier = dataSet.getExperimentIdentifierOrNull();
if (expIdentifier != null)
{
Experiment experiment = service.tryGetExperiment(expIdentifier);
- dsBatchUpdatesDTO.setExperimentIdentifierOrNull(ExperimentIdentifierFactory.parse(experiment.getIdentifier()));
- dsBatchUpdatesDTO.getDetails().setExperimentUpdateRequested(true);
+// dsBatchUpdatesDTO.setExperimentIdentifierOrNull(ExperimentIdentifierFactory.parse(experiment.getIdentifier()));
+ dsBatchUpdatesDTO.setExperimentIdentifierOrNull(expIdentifier);
} else
{
dsBatchUpdatesDTO.setExperimentIdentifierOrNull(null);
- dsBatchUpdatesDTO.getDetails().setExperimentUpdateRequested(true);
}
+ dsBatchUpdatesDTO.getDetails().setExperimentUpdateRequested(true);
builder.dataSetUpdate(dsBatchUpdatesDTO);
}
@@ -763,7 +763,7 @@ public class EntitySynchronizer
}
MasterDataSynchronizer masterDataSyncronizer =
new MasterDataSynchronizer(config.getHarvesterUser(), config.getHarvesterPass(), config.isDryRun(), config.isVerbose(), operationLog);
- masterDataSyncronizer.synchronizeMasterData(masterData);
+ masterDataSyncronizer.synchronizeMasterData(masterData, monitor);
monitor.log();
}
@@ -779,12 +779,12 @@ public class EntitySynchronizer
}
}
- private void processDeletions(ResourceListParserData data) throws NoSuchAlgorithmException, UnsupportedEncodingException
+ private void processDeletions(ResourceListParserData data) throws Exception
{
Monitor monitor = new Monitor("Delete entities", operationLog);
- String sessionToken = ServiceProvider.getOpenBISService().getSessionToken();
+ String sessionToken = service.getSessionToken();
IEntityRetriever entityRetriever =
- SkinnyEntityRetriever.createWithSessionToken(ServiceProvider.getV3ApplicationService(), sessionToken);
+ SkinnyEntityRetriever.createWithSessionToken(v3Api, sessionToken);
Set<String> incomingProjectPermIds = data.getProjectsToProcess().keySet();
Set<String> incomingExperimentPermIds = data.getExperimentsToProcess().keySet();
@@ -916,7 +916,6 @@ public class EntitySynchronizer
return;
}
- IApplicationServerApi v3Api = ServiceProvider.getV3ApplicationService();
// delete data sets
DataSetDeletionOptions dsDeletionOpts = new DataSetDeletionOptions();
String reasonDetail = " from data source : " + config.getDataSourceAlias();
@@ -1184,18 +1183,19 @@ public class EntitySynchronizer
prjUpdate.setSpaceCode(projectIdentifier.getSpaceCode());
return prjUpdate;
}
-
+
private void processSamples(ResourceListParserData data, AtomicEntityOperationDetailsBuilder builder, Monitor monitor)
{
// process samples
Map<String, IncomingSample> samplesToProcess = data.getSamplesToProcess();
+ Map<String, ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample> knownSamples = getKnownSamples(samplesToProcess.keySet());
Map<SampleIdentifier, NewSample> samplesToUpdate = new HashMap<SampleIdentifier, NewSample>();
Set<String> sampleWithUpdatedParents = new HashSet<String>();
int count = 0;
int n = samplesToProcess.size();
for (IncomingSample sample : samplesToProcess.values())
{
- if (++count % 100 == 0)
+ if (++count % 1000 == 0)
{
monitor.log(String.format("%7d/%d sample: %s", count, n, sample.getIdentifer()));
}
@@ -1203,16 +1203,9 @@ public class EntitySynchronizer
if (sample.getLastModificationDate().after(lastSyncTimestamp))
{
SampleIdentifier sampleIdentifier = SampleIdentifierFactory.parse(incomingSample);
- Sample sampleWithExperiment = null;
- try
- {
- sampleWithExperiment = service.tryGetSampleByPermId(incomingSample.getPermID());
- } catch (Exception e)
- {
- // doing nothing because when the sample with the perm is not found
- // an exception will be thrown. See the same issue for projects
- }
- if (sampleWithExperiment == null)
+ ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample knownSample = null;
+ knownSample = knownSamples.get(incomingSample.getPermID());
+ if (knownSample == null)
{
// ADD SAMPLE
builder.sample(incomingSample);
@@ -1220,10 +1213,10 @@ public class EntitySynchronizer
{
// defer creation of sample update objects until all samples have been gone through;
samplesToUpdate.put(sampleIdentifier, incomingSample);
- List<Sample> childSamples = getChildSamples(sampleWithExperiment);
- for (Sample child : childSamples)
+ ;
+ for (ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample child : knownSample.getChildren())
{
- String childSampleIdentifier = child.getIdentifier();// edgeNodePair.getNode().getIdentifier();
+ String childSampleIdentifier = child.getIdentifier().getIdentifier();// edgeNodePair.getNode().getIdentifier();
IncomingSample childSampleWithConns = findChildInSamplesToProcess(childSampleIdentifier, samplesToProcess);
if (childSampleWithConns == null)
{
@@ -1238,7 +1231,7 @@ public class EntitySynchronizer
}
}
}
- for (ResourceListParserData.Connection conn : sample.getConnections())
+ for (Connection conn : sample.getConnections())
{
if (conn.getType().equals("Component"))
{
@@ -1299,6 +1292,21 @@ public class EntitySynchronizer
}
}
+ private Map<String, ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample> getKnownSamples(Collection<String> samplePermIds)
+ {
+ String sessionToken = service.getSessionToken();
+ List<SamplePermId> sampleIds = samplePermIds.stream().map(SamplePermId::new).collect(Collectors.toList());
+ SampleFetchOptions fetchOptions = new SampleFetchOptions();
+ fetchOptions.withChildren();
+ Map<ISampleId, ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample> samples = v3Api.getSamples(sessionToken, sampleIds, fetchOptions);
+ HashMap<String, ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample> result = new HashMap<>();
+ for (ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample sample : samples.values())
+ {
+ result.put(sample.getPermId().getPermId(), sample);
+ }
+ return result;
+ }
+
/**
* Pads out the incoming property lists with remaining properties set to "" This way any properties that were re-set (value removed) in the data
* source will be carried over to the harvester
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/IncomingDataSet.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/IncomingDataSet.java
new file mode 100644
index 00000000000..419e309dcea
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/IncomingDataSet.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2018 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSetKind;
+import ch.systemsx.cisd.openbis.generic.shared.dto.NewContainerDataSet;
+import ch.systemsx.cisd.openbis.generic.shared.dto.NewExternalData;
+import ch.systemsx.cisd.openbis.generic.shared.dto.NewLinkDataSet;
+
+public class IncomingDataSet implements Serializable
+{
+ private static final long serialVersionUID = 1L;
+ private final NewExternalData dataSet;
+
+ final Date lastModificationDate;
+
+ public Date getLastModificationDate()
+ {
+ return lastModificationDate;
+ }
+
+ public DataSetKind getKind()
+ {
+ if (dataSet instanceof NewContainerDataSet)
+ return DataSetKind.CONTAINER;
+ else if (dataSet instanceof NewLinkDataSet)
+ return DataSetKind.LINK;
+ return DataSetKind.PHYSICAL;
+ }
+
+ public NewExternalData getDataSet()
+ {
+ return dataSet;
+ }
+
+ IncomingDataSet(NewExternalData dataSet, Date lastModDate)
+ {
+ super();
+ this.dataSet = dataSet;
+ this.lastModificationDate = lastModDate;
+ }
+
+ private List<Connection> connections = new ArrayList<Connection>();
+
+ public List<Connection> getConnections()
+ {
+ return connections;
+ }
+
+ public void setConnections(List<Connection> conns)
+ {
+ // TODO do this better
+ this.connections = conns;
+ }
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/IncomingEntity.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/IncomingEntity.java
new file mode 100644
index 00000000000..f4a5f93cdac
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/IncomingEntity.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2018 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.SyncEntityKind;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Identifier;
+
+public class IncomingEntity<T extends Identifier<T>>
+{
+ private final Identifier<T> entity;
+
+ private final SyncEntityKind entityKind;
+
+ private List<Connection> connections = new ArrayList<Connection>();
+
+ private boolean hasAttachments;
+
+ public List<Connection> getConnections()
+ {
+ return connections;
+ }
+
+ void addConnection(Connection conn)
+ {
+ this.connections.add(conn);
+ }
+
+ public SyncEntityKind getEntityKind()
+ {
+ return entityKind;
+ }
+
+ public void setConnections(List<Connection> conns)
+ {
+ // TODO do this better
+ this.connections = conns;
+ }
+
+ public boolean hasAttachments()
+ {
+ return hasAttachments;
+ }
+
+ public void setHasAttachments(boolean hasAttachments)
+ {
+ this.hasAttachments = hasAttachments;
+ }
+
+ public Identifier<T> getEntity()
+ {
+ return entity;
+ }
+
+ public String getIdentifer()
+ {
+ return getEntity().getIdentifier();
+ }
+
+ public String getPermID()
+ {
+ return getEntity().getPermID();
+ }
+
+ public Date getLastModificationDate()
+ {
+ return lastModificationDate;
+ }
+
+ private final Date lastModificationDate;
+
+ IncomingEntity(Identifier<T> entity, SyncEntityKind entityKind, Date lastModDate)
+ {
+ this.entity = entity;
+ this.entityKind = entityKind;
+ this.lastModificationDate = lastModDate;
+ }
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/IncomingExperiment.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/IncomingExperiment.java
new file mode 100644
index 00000000000..ee14cdcca40
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/IncomingExperiment.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2018 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer;
+
+import java.util.Date;
+
+import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.SyncEntityKind;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.NewExperiment;
+
+class IncomingExperiment extends IncomingEntity<NewExperiment>
+{
+ public NewExperiment getExperiment()
+ {
+ return (NewExperiment) getEntity();
+ }
+ IncomingExperiment(NewExperiment exp, Date lastModDate)
+ {
+ super(exp, SyncEntityKind.EXPERIMENT, lastModDate);
+ }
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/IncomingProject.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/IncomingProject.java
new file mode 100644
index 00000000000..83b8b32cf4f
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/IncomingProject.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2018 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer;
+
+import java.util.Date;
+
+import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.SyncEntityKind;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.NewProject;
+
+class IncomingProject extends IncomingEntity<NewProject>
+{
+ public NewProject getProject()
+ {
+ return (NewProject) getEntity();
+ }
+
+ IncomingProject(NewProject project, Date lastModDate)
+ {
+ super(project, SyncEntityKind.PROJECT, lastModDate);
+ }
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/IncomingSample.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/IncomingSample.java
new file mode 100644
index 00000000000..cfc67c8a4cd
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/IncomingSample.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2018 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer;
+
+import java.util.Date;
+
+import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.SyncEntityKind;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.NewSample;
+
+class IncomingSample extends IncomingEntity<NewSample>
+{
+ public NewSample getSample()
+ {
+ return (NewSample) getEntity();
+ }
+
+ IncomingSample(NewSample sample, Date lastModDate)
+ {
+ super(sample, SyncEntityKind.SAMPLE, lastModDate);
+ }
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MasterData.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MasterData.java
new file mode 100644
index 00000000000..fc7822dd127
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MasterData.java
@@ -0,0 +1,144 @@
+/*
+ * Copyright 2018 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.collections4.map.MultiKeyMap;
+
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetType;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ExperimentType;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.FileFormatType;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.MaterialType;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.NewETPTAssignment;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.NewVocabulary;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.PropertyType;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.SampleType;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Script;
+
+class MasterData
+{
+ private Map<String, FileFormatType> fileFormatTypesToProcess = new HashMap<String, FileFormatType>();
+
+ private Map<String, Script> validationPluginsToProcess = new HashMap<String, Script>();
+
+ private Map<String, NewVocabulary> vocabulariesToProcess = new HashMap<String, NewVocabulary>();
+
+ private Map<String, PropertyType> propertyTypesToProcess = new HashMap<String, PropertyType>();
+
+ private Map<String, SampleType> sampleTypesToProcess = new HashMap<String, SampleType>();
+
+ private Map<String, DataSetType> dataSetTypesToProcess = new HashMap<String, DataSetType>();
+
+ private Map<String, ExperimentType> experimentTypesToProcess = new HashMap<String, ExperimentType>();
+
+ private Map<String, MaterialType> materialTypesToProcess = new HashMap<String, MaterialType>();
+
+ private MultiKeyMap<String, List<NewETPTAssignment>> propertyAssignmentsToProcess = new MultiKeyMap<String, List<NewETPTAssignment>>();
+
+ public MultiKeyMap<String, List<NewETPTAssignment>> getPropertyAssignmentsToProcess()
+ {
+ return propertyAssignmentsToProcess;
+ }
+
+ public Map<String, Script> getValidationPluginsToProcess()
+ {
+ return validationPluginsToProcess;
+ }
+
+ public void setValidationPluginsToProcess(Map<String, Script> validationPluginsToProcess)
+ {
+ this.validationPluginsToProcess = validationPluginsToProcess;
+ }
+
+ public Map<String, PropertyType> getPropertyTypesToProcess()
+ {
+ return propertyTypesToProcess;
+ }
+
+ public Map<String, DataSetType> getDataSetTypesToProcess()
+ {
+ return dataSetTypesToProcess;
+ }
+
+ public Map<String, ExperimentType> getExperimentTypesToProcess()
+ {
+ return experimentTypesToProcess;
+ }
+
+ public Map<String, MaterialType> getMaterialTypesToProcess()
+ {
+ return materialTypesToProcess;
+ }
+
+ public Map<String, SampleType> getSampleTypesToProcess()
+ {
+ return sampleTypesToProcess;
+ }
+
+ public Map<String, NewVocabulary> getVocabulariesToProcess()
+ {
+ return vocabulariesToProcess;
+ }
+
+ public Map<String, FileFormatType> getFileFormatTypesToProcess()
+ {
+ return fileFormatTypesToProcess;
+ }
+
+ public void setFileFormatTypesToProcess(Map<String, FileFormatType> fileFormatTypesToProcess)
+ {
+ this.fileFormatTypesToProcess = fileFormatTypesToProcess;
+ }
+
+ public void setVocabulariesToProcess(Map<String, NewVocabulary> vocabulariesToProcess)
+ {
+ this.vocabulariesToProcess = vocabulariesToProcess;
+ }
+
+ public void setPropertyTypesToProcess(Map<String, PropertyType> propertyTypesToProcess)
+ {
+ this.propertyTypesToProcess = propertyTypesToProcess;
+ }
+
+ public void setSampleTypesToProcess(Map<String, SampleType> sampleTypesToProcess)
+ {
+ this.sampleTypesToProcess = sampleTypesToProcess;
+ }
+
+ public void setDataSetTypesToProcess(Map<String, DataSetType> dataSetTypesToProcess)
+ {
+ this.dataSetTypesToProcess = dataSetTypesToProcess;
+ }
+
+ public void setExperimentTypesToProcess(Map<String, ExperimentType> experimentTypesToProcess)
+ {
+ this.experimentTypesToProcess = experimentTypesToProcess;
+ }
+
+ public void setMaterialTypesToProcess(Map<String, MaterialType> materialTypesToProcess)
+ {
+ this.materialTypesToProcess = materialTypesToProcess;
+ }
+
+ public void setPropertyAssignmentsToProcess(MultiKeyMap<String, List<NewETPTAssignment>> propertyAssignmentsToProcess)
+ {
+ this.propertyAssignmentsToProcess = propertyAssignmentsToProcess;
+ }
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MasterDataSynchronizer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MasterDataSynchronizer.java
index 2338bcd77ce..136c2475e35 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MasterDataSynchronizer.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MasterDataSynchronizer.java
@@ -26,6 +26,7 @@ import org.apache.commons.collections4.map.MultiKeyMap;
import org.apache.log4j.Logger;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.ServiceFinderUtils;
+import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.util.Monitor;
import ch.systemsx.cisd.common.exceptions.UserFailureException;
import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
import ch.systemsx.cisd.openbis.generic.shared.ICommonServer;
@@ -73,18 +74,27 @@ public class MasterDataSynchronizer
vocabularyTermsToBeDeleted = new HashMap<TechId, List<VocabularyTerm>>();
}
- public void synchronizeMasterData(ResourceListParserData.MasterData masterData)
+ public void synchronizeMasterData(MasterData masterData, Monitor monitor)
{
MultiKeyMap<String, List<NewETPTAssignment>> propertyAssignmentsToProcess = masterData.getPropertyAssignmentsToProcess();
+ monitor.log("process file format types");
processFileFormatTypes(masterData.getFileFormatTypesToProcess());
+ monitor.log("process validation plugins");
processValidationPlugins(masterData.getValidationPluginsToProcess());
+ monitor.log("process vocabularies");
processVocabularies(masterData.getVocabulariesToProcess());
// materials are registered but their property assignments are deferred until after property types are processed
+ monitor.log("process material types");
processEntityTypes(masterData.getMaterialTypesToProcess(), propertyAssignmentsToProcess);
+ monitor.log("process property types");
processPropertyTypes(masterData.getPropertyTypesToProcess());
+ monitor.log("process sample types");
processEntityTypes(masterData.getSampleTypesToProcess(), propertyAssignmentsToProcess);
+ monitor.log("process data set types");
processEntityTypes(masterData.getDataSetTypesToProcess(), propertyAssignmentsToProcess);
+ monitor.log("process experiment types");
processEntityTypes(masterData.getExperimentTypesToProcess(), propertyAssignmentsToProcess);
+ monitor.log("process material type property assignments");
processDeferredMaterialTypePropertyAssignments(propertyAssignmentsToProcess);
synchronizerFacade.printSummary();
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MaterialWithLastModificationDate.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MaterialWithLastModificationDate.java
new file mode 100644
index 00000000000..0c783e092bf
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MaterialWithLastModificationDate.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2018 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer;
+
+import java.util.Date;
+
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.NewMaterialWithType;
+
+class MaterialWithLastModificationDate
+{
+ private final NewMaterialWithType material;
+
+ private final Date lastModificationDate;
+
+ public NewMaterialWithType getMaterial()
+ {
+ return material;
+ }
+
+ MaterialWithLastModificationDate(NewMaterialWithType material, Date lastModDate)
+ {
+ this.material = material;
+ this.lastModificationDate = lastModDate;
+ }
+
+ public Date getLastModificationDate()
+ {
+ return lastModificationDate;
+ }
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParser.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParser.java
index e55a13d8399..7a6c729b362 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParser.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParser.java
@@ -48,13 +48,6 @@ import org.w3c.dom.NodeList;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSetKind;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.SyncEntityKind;
-import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.ResourceListParserData.Connection;
-import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.ResourceListParserData.IncomingDataSet;
-import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.ResourceListParserData.IncomingExperiment;
-import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.ResourceListParserData.IncomingProject;
-import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.ResourceListParserData.IncomingSample;
-import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.ResourceListParserData.MasterData;
-import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.ResourceListParserData.MaterialWithLastModificationDate;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.translator.DefaultNameTranslator;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.translator.INameTranslator;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.util.Monitor;
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParserData.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParserData.java
index 87566ba6d45..7a523716fcd 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParserData.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParserData.java
@@ -15,7 +15,6 @@
*/
package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer;
-import java.io.Serializable;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
@@ -27,24 +26,7 @@ import java.util.Set;
import org.apache.commons.collections4.map.MultiKeyMap;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSetKind;
-import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.SyncEntityKind;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetType;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ExperimentType;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.FileFormatType;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Identifier;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.MaterialType;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.NewETPTAssignment;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.NewExperiment;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.NewMaterialWithType;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.NewProject;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.NewSample;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.NewVocabulary;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.PropertyType;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.SampleType;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Script;
-import ch.systemsx.cisd.openbis.generic.shared.dto.NewContainerDataSet;
-import ch.systemsx.cisd.openbis.generic.shared.dto.NewExternalData;
-import ch.systemsx.cisd.openbis.generic.shared.dto.NewLinkDataSet;
/**
* @author Ganime Betul Akin
@@ -59,13 +41,13 @@ public class ResourceListParserData
private MasterData masterData = new MasterData();
- private Map<String, IncomingProject> projectsToProcess = new HashMap<String, ResourceListParserData.IncomingProject>();
+ private Map<String, IncomingProject> projectsToProcess = new HashMap<String, IncomingProject>();
- private Map<String, IncomingExperiment> experimentsToProcess = new HashMap<String, ResourceListParserData.IncomingExperiment>();
+ private Map<String, IncomingExperiment> experimentsToProcess = new HashMap<String, IncomingExperiment>();
- private Map<String, IncomingSample> samplesToProcess = new HashMap<String, ResourceListParserData.IncomingSample>();
+ private Map<String, IncomingSample> samplesToProcess = new HashMap<String, IncomingSample>();
- private Map<String, IncomingDataSet> dataSetsToProcess = new HashMap<String, ResourceListParserData.IncomingDataSet>();
+ private Map<String, IncomingDataSet> dataSetsToProcess = new HashMap<String, IncomingDataSet>();
private MultiKeyMap<String, MaterialWithLastModificationDate> materialsToProcess = new MultiKeyMap<String, MaterialWithLastModificationDate>();
@@ -117,7 +99,7 @@ public class ResourceListParserData
public Map<String, IncomingDataSet> filterPhysicalDataSetsByLastModificationDate(Date lastSyncDate, Set<String> dataSetsCodesToRetry,
Set<String> blackListedDataSetCodes)
{
- Map<String, IncomingDataSet> dsMap = new HashMap<String, ResourceListParserData.IncomingDataSet>();
+ Map<String, IncomingDataSet> dsMap = new HashMap<String, IncomingDataSet>();
for (String permId : dataSetsToProcess.keySet())
{
IncomingDataSet ds = dataSetsToProcess.get(permId);
@@ -136,7 +118,7 @@ public class ResourceListParserData
public List<IncomingEntity<?>> filterAttachmentHoldersByLastModificationDate(Date lastSyncTimestamp, Set<String> attachmentHoldersToRetry)
{
- List<IncomingEntity<?>> attachmentHoldersToProcess = new ArrayList<ResourceListParserData.IncomingEntity<?>>();
+ List<IncomingEntity<?>> attachmentHoldersToProcess = new ArrayList<IncomingEntity<?>>();
// projects
for (IncomingProject incomingProject : projectsToProcess.values())
{
@@ -182,7 +164,7 @@ public class ResourceListParserData
public Map<String, IncomingDataSet> filterContainerDataSets()
{
- Map<String, IncomingDataSet> dsMap = new HashMap<String, ResourceListParserData.IncomingDataSet>();
+ Map<String, IncomingDataSet> dsMap = new HashMap<String, IncomingDataSet>();
for (String permId : dataSetsToProcess.keySet())
{
IncomingDataSet ds = dataSetsToProcess.get(permId);
@@ -193,337 +175,4 @@ public class ResourceListParserData
}
return dsMap;
}
-
- public static class IncomingEntity<T extends Identifier<T>>
- {
- private final Identifier<T> entity;
-
- private final SyncEntityKind entityKind;
-
- private List<Connection> connections = new ArrayList<Connection>();
-
- private boolean hasAttachments;
-
- public List<Connection> getConnections()
- {
- return connections;
- }
-
- void addConnection(Connection conn)
- {
- this.connections.add(conn);
- }
-
- public SyncEntityKind getEntityKind()
- {
- return entityKind;
- }
-
- public void setConnections(List<Connection> conns)
- {
- // TODO do this better
- this.connections = conns;
- }
-
- public boolean hasAttachments()
- {
- return hasAttachments;
- }
-
- public void setHasAttachments(boolean hasAttachments)
- {
- this.hasAttachments = hasAttachments;
- }
-
- public Identifier<T> getEntity()
- {
- return entity;
- }
-
- public String getIdentifer()
- {
- return getEntity().getIdentifier();
- }
-
- public String getPermID()
- {
- return getEntity().getPermID();
- }
-
- public Date getLastModificationDate()
- {
- return lastModificationDate;
- }
-
- private final Date lastModificationDate;
-
- IncomingEntity(Identifier<T> entity, SyncEntityKind entityKind, Date lastModDate)
- {
- this.entity = entity;
- this.entityKind = entityKind;
- this.lastModificationDate = lastModDate;
- }
- }
-
- static class IncomingProject extends IncomingEntity<NewProject>
- {
- public NewProject getProject()
- {
- return (NewProject) getEntity();
- }
-
- IncomingProject(NewProject project, Date lastModDate)
- {
- super(project, SyncEntityKind.PROJECT, lastModDate);
- }
- }
-
- static class IncomingExperiment extends IncomingEntity<NewExperiment>
- {
- public NewExperiment getExperiment()
- {
- return (NewExperiment) getEntity();
- }
- IncomingExperiment(NewExperiment exp, Date lastModDate)
- {
- super(exp, SyncEntityKind.EXPERIMENT, lastModDate);
- }
- }
-
- static class IncomingSample extends IncomingEntity<NewSample>
- {
- public NewSample getSample()
- {
- return (NewSample) getEntity();
- }
-
- IncomingSample(NewSample sample, Date lastModDate)
- {
- super(sample, SyncEntityKind.SAMPLE, lastModDate);
- }
- }
-
- public static class IncomingDataSet implements Serializable
- {
- private static final long serialVersionUID = 1L;
- private final NewExternalData dataSet;
-
- private final Date lastModificationDate;
-
- public Date getLastModificationDate()
- {
- return lastModificationDate;
- }
-
- public DataSetKind getKind()
- {
- if (dataSet instanceof NewContainerDataSet)
- return DataSetKind.CONTAINER;
- else if (dataSet instanceof NewLinkDataSet)
- return DataSetKind.LINK;
- return DataSetKind.PHYSICAL;
- }
-
- public NewExternalData getDataSet()
- {
- return dataSet;
- }
-
- IncomingDataSet(NewExternalData dataSet, Date lastModDate)
- {
- super();
- this.dataSet = dataSet;
- this.lastModificationDate = lastModDate;
- }
-
- private List<Connection> connections = new ArrayList<Connection>();
-
- public List<Connection> getConnections()
- {
- return connections;
- }
-
- public void setConnections(List<Connection> conns)
- {
- // TODO do this better
- this.connections = conns;
- }
- }
-
- static class MasterData
- {
- private Map<String, FileFormatType> fileFormatTypesToProcess = new HashMap<String, FileFormatType>();
-
- private Map<String, Script> validationPluginsToProcess = new HashMap<String, Script>();
-
- private Map<String, NewVocabulary> vocabulariesToProcess = new HashMap<String, NewVocabulary>();
-
- private Map<String, PropertyType> propertyTypesToProcess = new HashMap<String, PropertyType>();
-
- private Map<String, SampleType> sampleTypesToProcess = new HashMap<String, SampleType>();
-
- private Map<String, DataSetType> dataSetTypesToProcess = new HashMap<String, DataSetType>();
-
- private Map<String, ExperimentType> experimentTypesToProcess = new HashMap<String, ExperimentType>();
-
- private Map<String, MaterialType> materialTypesToProcess = new HashMap<String, MaterialType>();
-
- private MultiKeyMap<String, List<NewETPTAssignment>> propertyAssignmentsToProcess = new MultiKeyMap<String, List<NewETPTAssignment>>();
-
- public MultiKeyMap<String, List<NewETPTAssignment>> getPropertyAssignmentsToProcess()
- {
- return propertyAssignmentsToProcess;
- }
-
- public Map<String, Script> getValidationPluginsToProcess()
- {
- return validationPluginsToProcess;
- }
-
- public void setValidationPluginsToProcess(Map<String, Script> validationPluginsToProcess)
- {
- this.validationPluginsToProcess = validationPluginsToProcess;
- }
-
- public Map<String, PropertyType> getPropertyTypesToProcess()
- {
- return propertyTypesToProcess;
- }
-
- public Map<String, DataSetType> getDataSetTypesToProcess()
- {
- return dataSetTypesToProcess;
- }
-
- public Map<String, ExperimentType> getExperimentTypesToProcess()
- {
- return experimentTypesToProcess;
- }
-
- public Map<String, MaterialType> getMaterialTypesToProcess()
- {
- return materialTypesToProcess;
- }
-
- public Map<String, SampleType> getSampleTypesToProcess()
- {
- return sampleTypesToProcess;
- }
-
- public Map<String, NewVocabulary> getVocabulariesToProcess()
- {
- return vocabulariesToProcess;
- }
-
- public Map<String, FileFormatType> getFileFormatTypesToProcess()
- {
- return fileFormatTypesToProcess;
- }
-
- public void setFileFormatTypesToProcess(Map<String, FileFormatType> fileFormatTypesToProcess)
- {
- this.fileFormatTypesToProcess = fileFormatTypesToProcess;
- }
-
- public void setVocabulariesToProcess(Map<String, NewVocabulary> vocabulariesToProcess)
- {
- this.vocabulariesToProcess = vocabulariesToProcess;
- }
-
- public void setPropertyTypesToProcess(Map<String, PropertyType> propertyTypesToProcess)
- {
- this.propertyTypesToProcess = propertyTypesToProcess;
- }
-
- public void setSampleTypesToProcess(Map<String, SampleType> sampleTypesToProcess)
- {
- this.sampleTypesToProcess = sampleTypesToProcess;
- }
-
- public void setDataSetTypesToProcess(Map<String, DataSetType> dataSetTypesToProcess)
- {
- this.dataSetTypesToProcess = dataSetTypesToProcess;
- }
-
- public void setExperimentTypesToProcess(Map<String, ExperimentType> experimentTypesToProcess)
- {
- this.experimentTypesToProcess = experimentTypesToProcess;
- }
-
- public void setMaterialTypesToProcess(Map<String, MaterialType> materialTypesToProcess)
- {
- this.materialTypesToProcess = materialTypesToProcess;
- }
-
- public void setPropertyAssignmentsToProcess(MultiKeyMap<String, List<NewETPTAssignment>> propertyAssignmentsToProcess)
- {
- this.propertyAssignmentsToProcess = propertyAssignmentsToProcess;
- }
- }
-
- static class Connection
- {
- final String toPermId;
-
- final String connType;
-
- public String getType()
- {
- return connType;
- }
-
- Connection(String toPermId, String connType)
- {
- super();
- this.toPermId = toPermId;
- this.connType = connType;
- }
-
- public String getToPermId()
- {
- return toPermId;
- }
- }
-
- static enum ConnectionType
- {
- SIMPLE_CONNECTION("Connection"),
- PARENT_CHILD_RELATIONSHIP("Child"),
- CONTAINER_COMPONENT_RELATIONSHIP("Component");
-
- private final String type;
-
- public String getType()
- {
- return type;
- }
-
- private ConnectionType(String type)
- {
- this.type = type;
- }
- }
-
- static class MaterialWithLastModificationDate
- {
- private final NewMaterialWithType material;
-
- private final Date lastModificationDate;
-
- public NewMaterialWithType getMaterial()
- {
- return material;
- }
-
- MaterialWithLastModificationDate(NewMaterialWithType material, Date lastModDate)
- {
- this.material = material;
- this.lastModificationDate = lastModDate;
- }
-
- public Date getLastModificationDate()
- {
- return lastModificationDate;
- }
- }
}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/SynchronizationContext.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/SynchronizationContext.java
index fddecb4ba05..f93c059ee92 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/SynchronizationContext.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/SynchronizationContext.java
@@ -22,6 +22,7 @@ import java.util.Set;
import org.apache.log4j.Logger;
+import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.config.SyncConfig;
import ch.systemsx.cisd.openbis.dss.generic.shared.DataSetProcessingContext;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
@@ -33,6 +34,8 @@ public class SynchronizationContext
{
private IEncapsulatedOpenBISService service;
+ private IApplicationServerApi v3Api;
+
private String dataStoreCode;
private File storeRoot;
@@ -63,6 +66,16 @@ public class SynchronizationContext
this.service = service;
}
+ public IApplicationServerApi getV3Api()
+ {
+ return v3Api;
+ }
+
+ public void setV3Api(IApplicationServerApi v3Api)
+ {
+ this.v3Api = v3Api;
+ }
+
public String getDataStoreCode()
{
return dataStoreCode;
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/parallelizedExecutor/AttachmentSynchronizationTaskExecutor.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/parallelizedExecutor/AttachmentSynchronizationTaskExecutor.java
index 24735667cc3..1a521dd72e7 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/parallelizedExecutor/AttachmentSynchronizationTaskExecutor.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/parallelizedExecutor/AttachmentSynchronizationTaskExecutor.java
@@ -32,7 +32,8 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.id.SamplePermId;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.ServiceFinderUtils;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.SyncEntityKind;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.config.SyncConfig;
-import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.ResourceListParserData.IncomingEntity;
+import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.IncomingEntity;
+import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.util.Monitor;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.util.V3Utils;
import ch.systemsx.cisd.cifex.shared.basic.UserFailureException;
import ch.systemsx.cisd.common.concurrent.ITaskExecutor;
@@ -64,21 +65,28 @@ public final class AttachmentSynchronizationTaskExecutor implements ITaskExecuto
private AttachmentSynchronizationSummary syncSummary;
+ private V3Utils v3Utils;
+
+ private final Monitor monitor;
+
/**
* @param executionSummary will contain ids of attachment holders for which the attachment sync failed during parallel execution + counts of
* added, updated and deleted attachments
* @param service
* @param lastSyncTimestamp
* @param config
+ * @param monitor
*/
public AttachmentSynchronizationTaskExecutor(AttachmentSynchronizationSummary executionSummary,
- IEncapsulatedOpenBISService service,
- Date lastSyncTimestamp, SyncConfig config)
+ IEncapsulatedOpenBISService service, V3Utils v3Utils,
+ Date lastSyncTimestamp, SyncConfig config, Monitor monitor)
{
this.syncSummary = executionSummary;
this.service = service;
+ this.v3Utils = v3Utils;
this.lastSyncTimestamp = lastSyncTimestamp;
this.config = config;
+ this.monitor = monitor;
}
@Override
@@ -89,24 +97,25 @@ public final class AttachmentSynchronizationTaskExecutor implements ITaskExecuto
TechId techId = null;
ICommonServer commonServer = ServiceFinderUtils.getCommonServer(ServiceProvider.getConfigProvider().getOpenBisServerUrl());
String localSessionToken = ServiceFinderUtils.login(commonServer, config.getHarvesterUser(), config.getHarvesterPass());
+ monitor.log("Logged in: " + localSessionToken);
IAttachmentsOperationsHandler attachmentsOperationsHandler = null;
if (item.getEntityKind() == SyncEntityKind.EXPERIMENT)
{
Experiment experiment = service.tryGetExperiment(ExperimentIdentifierFactory.parse(item.getEntity().getIdentifier()));
techId = new TechId(experiment.getId());
- attachmentsOperationsHandler = new ExperimentAttachmentsOperationsHandler(config, commonServer, localSessionToken);
+ attachmentsOperationsHandler = new ExperimentAttachmentsOperationsHandler(config, commonServer, v3Utils, localSessionToken);
}
else if (item.getEntityKind() == SyncEntityKind.SAMPLE)
{
Sample sample = service.tryGetSampleByPermId(item.getEntity().getPermID());
techId = new TechId(sample.getId());
- attachmentsOperationsHandler = new SampleAttachmentsOperationsHandler(config, commonServer, localSessionToken);
+ attachmentsOperationsHandler = new SampleAttachmentsOperationsHandler(config, commonServer, v3Utils, localSessionToken);
}
else if (item.getEntityKind() == SyncEntityKind.PROJECT)
{
Project project = service.tryGetProject(ProjectIdentifierFactory.parse(item.getEntity().getIdentifier()));
techId = new TechId(project.getId());
- attachmentsOperationsHandler = new ProjectAttachmentsOperationsHandler(config, commonServer, localSessionToken);
+ attachmentsOperationsHandler = new ProjectAttachmentsOperationsHandler(config, commonServer, v3Utils, localSessionToken);
}
else
{
@@ -287,9 +296,12 @@ abstract class AbstractEntityAttachmentsOperationsHandler implements IAttachment
final SyncConfig config;
- AbstractEntityAttachmentsOperationsHandler(SyncConfig config, ICommonServer commonServer, String sessionToken)
+ protected V3Utils v3Utils;
+
+ AbstractEntityAttachmentsOperationsHandler(SyncConfig config, ICommonServer commonServer, V3Utils v3Utils, String sessionToken)
{
this.commonServer = commonServer;
+ this.v3Utils = v3Utils;
this.sessionToken = sessionToken;
this.config = config;
}
@@ -312,9 +324,9 @@ abstract class AbstractEntityAttachmentsOperationsHandler implements IAttachment
class ExperimentAttachmentsOperationsHandler extends AbstractEntityAttachmentsOperationsHandler
{
- ExperimentAttachmentsOperationsHandler(SyncConfig config, ICommonServer commonServer, String sessionToken)
+ ExperimentAttachmentsOperationsHandler(SyncConfig config, ICommonServer commonServer, V3Utils v3Utils, String sessionToken)
{
- super(config, commonServer, sessionToken);
+ super(config, commonServer, v3Utils, sessionToken);
}
@Override
@@ -356,9 +368,9 @@ class ExperimentAttachmentsOperationsHandler extends AbstractEntityAttachmentsOp
class SampleAttachmentsOperationsHandler extends AbstractEntityAttachmentsOperationsHandler
{
- SampleAttachmentsOperationsHandler(SyncConfig config, ICommonServer commonServer, String sessionToken)
+ SampleAttachmentsOperationsHandler(SyncConfig config, ICommonServer commonServer, V3Utils v3Utils, String sessionToken)
{
- super(config, commonServer, sessionToken);
+ super(config, commonServer, v3Utils, sessionToken);
}
@Override
@@ -400,17 +412,16 @@ class SampleAttachmentsOperationsHandler extends AbstractEntityAttachmentsOperat
class ProjectAttachmentsOperationsHandler extends AbstractEntityAttachmentsOperationsHandler
{
- ProjectAttachmentsOperationsHandler(SyncConfig config, ICommonServer commonServer, String sessionToken)
+ ProjectAttachmentsOperationsHandler(SyncConfig config, ICommonServer commonServer, V3Utils v3Utils, String sessionToken)
{
- super(config, commonServer, sessionToken);
+ super(config, commonServer, v3Utils, sessionToken);
}
@Override
public List<Attachment> listDataSourceAttachments(SyncConfig config, String permId)
{
- V3Utils dssFileUtils = V3Utils.create(config.getDataSourceOpenbisURL(), config.getDataSourceDSSURL());
- String sessionToken = dssFileUtils.login(config.getUser(), config.getPassword());
- return dssFileUtils.getProjectAttachments(sessionToken, new ProjectPermId(permId));
+ String sessionToken = v3Utils.login(config.getUser(), config.getPassword());
+ return v3Utils.getProjectAttachments(sessionToken, new ProjectPermId(permId));
}
@Override
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/parallelizedExecutor/DataSetRegistrationTaskExecutor.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/parallelizedExecutor/DataSetRegistrationTaskExecutor.java
index fdebb42db8c..7b610a96363 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/parallelizedExecutor/DataSetRegistrationTaskExecutor.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/parallelizedExecutor/DataSetRegistrationTaskExecutor.java
@@ -24,7 +24,7 @@ import java.util.Properties;
import org.apache.log4j.Logger;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.config.SyncConfig;
-import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.ResourceListParserData.IncomingDataSet;
+import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.IncomingDataSet;
import ch.systemsx.cisd.common.concurrent.ITaskExecutor;
import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.openbis.dss.generic.shared.DataSetProcessingContext;
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/util/V3Utils.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/util/V3Utils.java
index a8f5d4e012e..5b6c6178963 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/util/V3Utils.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/util/V3Utils.java
@@ -21,6 +21,8 @@ import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import org.apache.commons.lang.time.DateUtils;
+
import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.attachment.Attachment;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.search.SearchResult;
@@ -41,6 +43,7 @@ import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.id.IDataSetFileId;
import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.search.DataSetFileSearchCriteria;
import ch.systemsx.cisd.common.spring.HttpInvokerUtils;
import ch.systemsx.cisd.common.ssl.SslCertificateHelper;
+import ch.systemsx.cisd.openbis.dss.generic.server.EncapsulatedOpenBISService;
/**
*
@@ -61,11 +64,9 @@ public class V3Utils
private V3Utils (String asUrl, String dssUrl, int timeout)
{
- SslCertificateHelper.trustAnyCertificate(asUrl);
- SslCertificateHelper.trustAnyCertificate(dssUrl);
-
- this.dss = HttpInvokerUtils.createStreamSupportingServiceStub(IDataStoreServerApi.class, dssUrl + IDataStoreServerApi.SERVICE_URL, timeout);
- this.as = HttpInvokerUtils.createServiceStub(IApplicationServerApi.class, asUrl + IApplicationServerApi.SERVICE_URL, timeout);
+ String timeoutInMinutes = Long.toString(timeout / DateUtils.MILLIS_PER_MINUTE);
+ this.as = EncapsulatedOpenBISService.createOpenBisV3Service(asUrl, timeoutInMinutes);
+ this.dss = EncapsulatedOpenBISService.createDataStoreV3Service(dssUrl, timeoutInMinutes);
}
public SearchResult<DataSetFile> searchFiles(String sessionToken, DataSetFileSearchCriteria criteria, DataSetFileFetchOptions dsFileFetchOptions)
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/EncapsulatedOpenBISService.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/EncapsulatedOpenBISService.java
index 5dd99b7571a..842899c67db 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/EncapsulatedOpenBISService.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/EncapsulatedOpenBISService.java
@@ -28,6 +28,7 @@ import org.apache.log4j.Logger;
import org.springframework.beans.factory.FactoryBean;
import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi;
+import ch.ethz.sis.openbis.generic.dssapi.v3.IDataStoreServerApi;
import ch.systemsx.cisd.common.api.retry.RetryCaller;
import ch.systemsx.cisd.common.api.retry.config.RetryConfiguration;
import ch.systemsx.cisd.common.exceptions.UserFailureException;
@@ -35,6 +36,7 @@ import ch.systemsx.cisd.common.logging.Log4jSimpleLogger;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.openbis.common.api.client.ServiceFinder;
+import ch.systemsx.cisd.openbis.dss.generic.shared.DataStoreServerV3Factory;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedBasicOpenBISService;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
import ch.systemsx.cisd.openbis.dss.generic.shared.IShareIdManager;
@@ -202,6 +204,11 @@ public final class EncapsulatedOpenBISService implements IEncapsulatedOpenBISSer
{
return getGenericRetryingOpenBisV3Creator(openBISURL, timeout).callWithRetry();
}
+
+ public static IDataStoreServerApi createDataStoreV3Service(String dataStoreServerUrl, String timeout)
+ {
+ return new RetryingOpenBisCreator<>(dataStoreServerUrl, timeout, new DataStoreServerV3Factory(dataStoreServerUrl)).callWithRetry();
+ }
/**
* Creates a remote version of {@link IGeneralInformationService} for specified URL and time out (in minutes).
--
GitLab