From 82805cd993d945278652beb9259787bc3b636b5c Mon Sep 17 00:00:00 2001
From: gakin <gakin>
Date: Sun, 12 Mar 2017 16:39:14 +0000
Subject: [PATCH] SSDM-4394 : Read the attachment and sync - first pass
SVN: 37883
---
.../harvester/synchronizer/DSSFileUtils.java | 18 +++++
.../synchronizer/EntitySynchronizer.java | 72 ++++++++++++++++++-
.../synchronizer/ResourceListParserData.java | 1 -
3 files changed, 87 insertions(+), 4 deletions(-)
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/DSSFileUtils.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/DSSFileUtils.java
index 4ba6674df8e..b0404a15351 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/DSSFileUtils.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/DSSFileUtils.java
@@ -17,10 +17,16 @@
package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer;
import java.io.InputStream;
+import java.util.Arrays;
import java.util.List;
+import java.util.Map;
import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.attachment.Attachment;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.search.SearchResult;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.fetchoptions.ExperimentFetchOptions;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.id.IExperimentId;
import ch.ethz.sis.openbis.generic.dssapi.v3.IDataStoreServerApi;
import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile;
import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.download.DataSetFileDownloadOptions;
@@ -68,6 +74,18 @@ public class DSSFileUtils
return searchFiles(sessionToken, criteria, dsFileFetchOptions);
}
+ public List<Attachment> getExperimentAttachments(String sessionToken, IExperimentId experimentId)
+ {
+ ExperimentFetchOptions fetchOptions = new ExperimentFetchOptions();
+ fetchOptions.withAttachments().withContent();
+ Map<IExperimentId, Experiment> experiments = as.getExperiments(sessionToken, Arrays.asList(experimentId), fetchOptions);
+ if(experiments.size() == 1) {
+ Experiment experiment = experiments.get(experimentId);
+ return experiment.getAttachments();
+ }
+ return null;
+ }
+
public InputStream downloadFiles(String sessionToken, List<IDataSetFileId> fileIds, DataSetFileDownloadOptions options)
{
return dss.downloadFiles(sessionToken, fileIds, options);
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
index 43c5645c01f..376e45e4a1f 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
@@ -41,6 +41,7 @@ import org.apache.log4j.Logger;
import org.w3c.dom.Document;
import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.attachment.Attachment;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.search.SearchResult;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSetKind;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.delete.DataSetDeletionOptions;
@@ -59,6 +60,7 @@ import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile;
import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.fetchoptions.DataSetFileFetchOptions;
import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.search.DataSetFileSearchCriteria;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.EntityRetriever;
+import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.ServiceFinderUtils;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.config.SyncConfig;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.ResourceListParserData.Connection;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.ResourceListParserData.DataSetWithConnections;
@@ -86,11 +88,14 @@ import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
import ch.systemsx.cisd.openbis.dss.generic.shared.IShareIdManager;
import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.utils.SegmentedStoreUtils;
+import ch.systemsx.cisd.openbis.generic.shared.ICommonServer;
import ch.systemsx.cisd.openbis.generic.shared.basic.TechId;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AttachmentHolderKind;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Experiment;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.GenericEntityProperty;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IEntityProperty;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Identifier;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ListSampleCriteria;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Material;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.MaterialIdentifier;
@@ -117,6 +122,7 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.NewExternalData;
import ch.systemsx.cisd.openbis.generic.shared.dto.NewProperty;
import ch.systemsx.cisd.openbis.generic.shared.dto.ProjectUpdatesDTO;
import ch.systemsx.cisd.openbis.generic.shared.dto.SampleUpdatesDTO;
+import ch.systemsx.cisd.openbis.generic.shared.dto.SessionContextDTO;
import ch.systemsx.cisd.openbis.generic.shared.dto.builders.AtomicEntityOperationDetailsBuilder;
import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.ExperimentIdentifier;
import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.ExperimentIdentifierFactory;
@@ -149,6 +155,8 @@ public class EntitySynchronizer
private final Set<String> blackListedDataSetCodes;
+ private final List<Identifier<?>> entitiesWithAttachments = new ArrayList<Identifier<?>>();
+
private MasterDataSynchronizer masterDataSyncronizer;
public EntitySynchronizer(IEncapsulatedOpenBISService service, String dataStoreCode, File storeRoot, Date lastSyncTimestamp,
@@ -172,7 +180,7 @@ public class EntitySynchronizer
return syncronizeEntities(dataSourceConnector);
}
- public Date syncronizeEntities(IDataSourceConnector dataSourceConnector) throws Exception
+ private Date syncronizeEntities(IDataSourceConnector dataSourceConnector) throws Exception
{
// retrieve the document from the data source
operationLog.info("Retrieving the resource list..");
@@ -215,6 +223,9 @@ public class EntitySynchronizer
AtomicEntityOperationResult operationResult = service.performEntityOperations(builder.getDetails());
operationLog.info("entity operation result: " + operationResult);
+ operationLog.info("processing attachments...");
+ // processAttachments(entitiesWithAttachments);
+
// register physical data sets without any hierarchy
// Note that container/component and parent/child relationships are established post-reg.
// setParentDataSetsOnTheChildren(data);
@@ -239,6 +250,12 @@ public class EntitySynchronizer
return data.getResourceListTimestamp();
}
+ private void processAttachments(List<Identifier<?>> entities)
+ {
+ ParallelizedExecutor.process(entities, new AttachmentSynchronizationTaskExecutor(),
+ 0.5, 10, "process attachments", 0, false);
+ }
+
private void cleanup()
{
operationLog.info("Cleaning up unused master data");
@@ -673,6 +690,11 @@ public class EntitySynchronizer
ExperimentUpdatesDTO expUpdate = createExperimentUpdateDTOs(incomingExp, experiment);
builder.experimentUpdate(expUpdate);
}
+ // add to a list for processing the attachments later on.
+ if (exp.hasAttachments() == true)
+ {
+ entitiesWithAttachments.add(incomingExp);
+ }
}
handleExperimentConnections(data, exp, incomingExp);
}
@@ -770,6 +792,12 @@ public class EntitySynchronizer
// UPDATE PROJECT
builder.projectUpdate(createProjectUpdateDTO(incomingProject, project));
}
+
+ // add to a list for processing the attachments later on.
+ if (prj.hasAttachments() == true)
+ {
+ entitiesWithAttachments.add(incomingProject);
+ }
}
// handleProjectConnections(data, prj);
}
@@ -869,6 +897,11 @@ public class EntitySynchronizer
}
}
}
+ // add to a list for processing the attachments later on.
+ if (sample.hasAttachments() == true)
+ {
+ entitiesWithAttachments.add(incomingSample);
+ }
}
for (Connection conn : sample.getConnections())
{
@@ -1004,6 +1037,41 @@ public class EntitySynchronizer
return null;
}
+ private final class AttachmentSynchronizationTaskExecutor implements ITaskExecutor<Identifier<?>>
+ {
+ public AttachmentSynchronizationTaskExecutor()
+ {
+ }
+
+ @Override
+ public Status execute(Identifier<?> item)
+ {
+ DSSFileUtils dssFileUtils = DSSFileUtils.create(config.getDataSourceOpenbisURL(), config.getDataSourceDSSURL());
+ String sessionToken = dssFileUtils.login(config.getUser(), config.getPassword());
+ if(item instanceof NewExperiment) {
+
+ // TODO complete this by comparing versions, getting any previous versions using v3 api and syncing
+ List<Attachment> incomingAttachments = dssFileUtils.getExperimentAttachments(sessionToken, new ExperimentPermId(item.getPermID()));
+
+ Experiment experiment = service.tryGetExperiment(ExperimentIdentifierFactory.parse(item.getIdentifier()));
+ List<ch.systemsx.cisd.openbis.generic.shared.basic.dto.Attachment> listAttachments =
+ service.listAttachments(AttachmentHolderKind.EXPERIMENT, experiment.getId());
+ ICommonServer commonServer = ServiceFinderUtils.getCommonServer(ServiceProvider.getConfigProvider().getOpenBisServerUrl());
+ SessionContextDTO sessionDTO = commonServer.tryAuthenticate(config.getHarvesterUser(), config.getHarvesterPass());
+ for (Attachment attachment : incomingAttachments)
+ {
+ NewAttachment newAttachment = new NewAttachment();
+ newAttachment.setTitle(attachment.getTitle());
+ newAttachment.setDescription(attachment.getDescription());
+ newAttachment.setFilePath(attachment.getFileName());
+ newAttachment.setContent(attachment.getContent());
+ commonServer.addExperimentAttachment(sessionDTO.getSessionToken(), new TechId(experiment.getId()), newAttachment);
+ }
+ }
+ return null;
+ }
+ }
+
private final class DataSetRegistrationTaskExecutor implements ITaskExecutor<DataSetWithConnections>
{
private DataSetRegistrationSummary dsRegistrationSummary;
@@ -1016,8 +1084,6 @@ public class EntitySynchronizer
@Override
public Status execute(DataSetWithConnections dataSet)
{
- System.out.println("start " + dataSet.getDataSet().getCode());
-
Properties props = setProperties();
DataSetRegistrationIngestionService ingestionService =
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParserData.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParserData.java
index 3c263a0e249..66038030da2 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParserData.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParserData.java
@@ -127,7 +127,6 @@ public class ResourceListParserData
return dsMap;
}
-
public Map<String, DataSetWithConnections> filterContainerDataSets()
{
// List<NewDataSetWithConnections> dsList = new ArrayList<ResourceListParserData.NewDataSetWithConnections>();
--
GitLab