From 69392719a13a982ac4e6e14164e13ae9ad3ee917 Mon Sep 17 00:00:00 2001
From: pkupczyk <piotr.kupczyk@id.ethz.ch>
Date: Fri, 31 Aug 2018 20:12:56 +0200
Subject: [PATCH] SSDM-7081 : openBIS sync: Support project samples
---
.../plugins/sync/common/EntityRetriever.java | 25 ++++++++
.../plugins/sync/common/entitygraph/Node.java | 4 +-
.../synchronizer/MasterDataParser.java | 2 +-
.../synchronizer/ResourceListParser.java | 63 +++++++++++--------
.../openbis_test_openbis_sync_openbis1.sql | 8 ++-
.../generic/asapi/v3/dto/sample/Sample.java | 4 +-
6 files changed, 74 insertions(+), 32 deletions(-)
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/EntityRetriever.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/EntityRetriever.java
index 916481ae4e8..e06ea3716ef 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/EntityRetriever.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/EntityRetriever.java
@@ -133,6 +133,7 @@ public class EntityRetriever implements IEntityRetriever
Node<Project> prjNode = new Node<Project>(project);
graph.addNode(prjNode);
addExperiments(prjNode);
+ addSamplesForProject(prjNode);
}
// add space samples
@@ -164,8 +165,11 @@ public class EntityRetriever implements IEntityRetriever
private void addSpaceSamples(String spaceCode)
{
+ // Add samples that are connected with a space only (i.e. have project == null and experiment == null).
+
SampleSearchCriteria criteria = new SampleSearchCriteria();
criteria.withSpace().withCode().thatEquals(spaceCode);
+ criteria.withoutProject();
criteria.withoutExperiment();
criteria.withAndOperator();
@@ -195,6 +199,8 @@ public class EntityRetriever implements IEntityRetriever
{
for (Sample sample : expNode.getEntity().getSamples())
{
+ // Add samples that are connected with an experiment and optionally a project.
+
Node<Sample> sampleNode = new Node<Sample>(sample);
graph.addEdge(expNode, sampleNode, new Edge(CONNECTION));
@@ -203,6 +209,23 @@ public class EntityRetriever implements IEntityRetriever
}
}
+ private void addSamplesForProject(Node<Project> prjNode)
+ {
+ for (Sample sample : prjNode.getEntity().getSamples())
+ {
+ // Add samples that are connected with a project only (i.e. have experiment == null).
+
+ if (sample.getExperiment() == null)
+ {
+ Node<Sample> sampleNode = new Node<Sample>(sample);
+ graph.addEdge(prjNode, sampleNode, new Edge(CONNECTION));
+
+ addDataSetsForSample(sampleNode);
+ addChildAndComponentSamples(sampleNode);
+ }
+ }
+ }
+
private void addDataSetsForExperiment(Node<Experiment> expNode)
{
for (DataSet dataSet : expNode.getEntity().getDataSets())
@@ -339,6 +362,7 @@ public class EntityRetriever implements IEntityRetriever
fo.withSpace();
fo.withAttachments();
fo.withExperimentsUsing(createExperimentFetchOptions());
+ fo.withSamplesUsing(createSampleFetchOptions());
return fo;
}
@@ -361,6 +385,7 @@ public class EntityRetriever implements IEntityRetriever
fo.withDataSets();
fo.withType();
fo.withExperiment();
+ fo.withProject();
fo.withSpace();
fo.withAttachments();
fo.withChildrenUsing(fo);
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/entitygraph/Node.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/entitygraph/Node.java
index 0468aba9952..45e219ceb01 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/entitygraph/Node.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/entitygraph/Node.java
@@ -142,7 +142,9 @@ public class Node<T extends IModificationDateHolder & IModifierHolder & IRegistr
{
return null;
}
- return ((IProjectHolder) entity).getProject().getCode();
+
+ Project project = ((IProjectHolder) entity).getProject();
+ return project != null ? project.getCode() : null;
}
public Sample getSampleOrNull()
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MasterDataParser.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MasterDataParser.java
index 551fc823ebe..74cf84519b7 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MasterDataParser.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MasterDataParser.java
@@ -190,7 +190,7 @@ public class MasterDataParser
String entityKind = getAttribute(pluginElement, "entityKind").trim();
plugin.setScriptType(ScriptType.valueOf(getAttribute(pluginElement, "type")));
plugin.setPluginType(PluginType.JYTHON);
- plugin.setEntityKind(entityKind.equals("") ? null : new EntityKind[] { EntityKind.valueOf(entityKind) });
+ plugin.setEntityKind((entityKind.equals("") || entityKind.equals("All")) ? null : new EntityKind[] { EntityKind.valueOf(entityKind) });
plugin.setScript(pluginElement.getTextContent());
validationPlugins.put(plugin.getName(), plugin);
}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParser.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParser.java
index e1fa5a6b888..3fd1e797a8d 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParser.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParser.java
@@ -183,8 +183,7 @@ public class ResourceListParser
if (uri.endsWith("MASTER_DATA/MASTER_DATA/M"))
{
parseMasterData(doc, xpath, uri);
- }
- else if (uri.endsWith("/M"))
+ } else if (uri.endsWith("/M"))
{
list.add(uri);
}
@@ -218,20 +217,16 @@ public class ResourceListParser
if (SyncEntityKind.PROJECT.getLabel().equals(entityKind))
{
parseProjectMetaData(xpath, extractPermIdFromURI(uri), xdNode, lastModificationDate);
- }
- else if (SyncEntityKind.EXPERIMENT.getLabel().equals(entityKind))
+ } else if (SyncEntityKind.EXPERIMENT.getLabel().equals(entityKind))
{
parseExperimentMetaData(xpath, extractPermIdFromURI(uri), xdNode, lastModificationDate);
- }
- else if (SyncEntityKind.SAMPLE.getLabel().equals(entityKind))
+ } else if (SyncEntityKind.SAMPLE.getLabel().equals(entityKind))
{
parseSampleMetaData(xpath, extractPermIdFromURI(uri), xdNode, lastModificationDate);
- }
- else if (SyncEntityKind.DATA_SET.getLabel().equals(entityKind))
+ } else if (SyncEntityKind.DATA_SET.getLabel().equals(entityKind))
{
parseDataSetMetaData(xpath, extractDataSetCodeFromURI(uri), xdNode, lastModificationDate);
- }
- else if (SyncEntityKind.MATERIAL.getLabel().equals(entityKind))
+ } else if (SyncEntityKind.MATERIAL.getLabel().equals(entityKind))
{
parseMaterialMetaData(xpath, extractMaterialCodeFromURI(uri), xdNode, lastModificationDate);
}
@@ -287,13 +282,11 @@ public class ResourceListParser
{
ds = new NewContainerDataSet();
ds.setDataSetKind(ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetKind.CONTAINER);
- }
- else if (dsKind.equals(DataSetKind.PHYSICAL.toString()))
+ } else if (dsKind.equals(DataSetKind.PHYSICAL.toString()))
{
ds = new NewExternalData();
ds.setDataSetKind(ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetKind.PHYSICAL);
- }
- else
+ } else
{
throw new IllegalArgumentException(dsKind + " data sets are currently not supported");
}
@@ -318,13 +311,11 @@ public class ResourceListParser
if (nullAllowed == false)
{
throw new IllegalArgumentException(attrName + " cannot be empty in Resource List");
- }
- else
+ } else
{
return null;
}
- }
- else
+ } else
{
return val.trim();
}
@@ -392,14 +383,20 @@ public class ResourceListParser
return new ProjectIdentifier(createSpaceIdentifier(space), code);
}
- private SampleIdentifier createSampleIdentifier(String code, String space)
+ private SampleIdentifier createSampleIdentifier(String code, String project, String space)
{
if (space == null)
{
return new SampleIdentifier(nameTranslator.translate(code));
+ } else if (project == null)
+ {
+ SpaceIdentifier spaceIdentifier = createSpaceIdentifier(space);
+ return new SampleIdentifier(spaceIdentifier, code);
+ } else
+ {
+ ProjectIdentifier projectIdentifier = createProjectIdentifier(project, space);
+ return new SampleIdentifier(projectIdentifier, code);
}
- SpaceIdentifier spaceIdentifier = createSpaceIdentifier(space);
- return new SampleIdentifier(spaceIdentifier, code);
}
private SpaceIdentifier createSpaceIdentifier(String space)
@@ -499,7 +496,7 @@ public class ResourceListParser
val = nameTranslator.translate(val);
val = translateMaterialIdentifier(val);
}
-
+
}
propertyType.setCode(translatedCode);
property.setPropertyType(propertyType);
@@ -507,7 +504,8 @@ public class ResourceListParser
return property;
}
- private String translateMaterialIdentifier(String value) {
+ private String translateMaterialIdentifier(String value)
+ {
if (StringUtils.isBlank(value))
{
return null;
@@ -526,6 +524,7 @@ public class ResourceListParser
}
return new MaterialIdentifier(code, typeCode).toString();
}
+
private void parseExperimentMetaData(XPath xpath, String permId, Node xdNode, Date lastModificationDate)
{
String code = extractCode(xdNode);
@@ -547,14 +546,21 @@ public class ResourceListParser
String code = extractCode(xdNode);
String type = extractType(xdNode);
String experiment = extractAttribute(xdNode, "experiment", true);
+ String project = extractAttribute(xdNode, "project", true);
String space = extractSpace(xdNode, true);
SampleType sampleType = new SampleType();
sampleType.setCode(type);
-
- SampleIdentifier identifier = createSampleIdentifier(code, space);
+
+ if (false == StringUtils.isBlank(project) && StringUtils.isBlank(space))
+ {
+ throw new IllegalArgumentException("Sample with perm id '" + permId + "' has 'project' attribute specified but 'space' attribute empty");
+ }
+
+ SampleIdentifier identifier = createSampleIdentifier(code, project, space);
String expIdentifier = null;
ExperimentIdentifier experimentIdentifier = getExperimentIdentifier(experiment);
- if (experimentIdentifier != null) {
+ if (experimentIdentifier != null)
+ {
expIdentifier = experimentIdentifier.toString();
}
NewSample newSample = new NewSample(identifier.toString(), sampleType, null, null,
@@ -565,6 +571,10 @@ public class ResourceListParser
{
newSample.setDefaultSpaceIdentifier(null);
}
+ if (space != null && project != null)
+ {
+ newSample.setProjectIdentifier(createProjectIdentifier(project, space).toString());
+ }
IncomingSample incomingSample = data.new IncomingSample(newSample, lastModificationDate);
data.getSamplesToProcess().put(permId, incomingSample);
incomingSample.setHasAttachments(hasAttachments(xpath, xdNode));
@@ -618,4 +628,3 @@ public class ResourceListParser
// System.out.print(url_node.item(ji).getNodeName() + ":" + url_node.item(ji).getAttributes().getNamedItem("kind"));
// System.out.println();
// }
-
diff --git a/integration-tests/templates/test_openbis_sync/openbis_test_openbis_sync_openbis1.sql b/integration-tests/templates/test_openbis_sync/openbis_test_openbis_sync_openbis1.sql
index b6a61ef6635..f89b3cb5faa 100644
--- a/integration-tests/templates/test_openbis_sync/openbis_test_openbis_sync_openbis1.sql
+++ b/integration-tests/templates/test_openbis_sync/openbis_test_openbis_sync_openbis1.sql
@@ -3757,8 +3757,12 @@ COPY sample_types (id, code, description, is_listable, generated_from_depth, par
--
COPY samples_all (id, perm_id, code, expe_id, saty_id, registration_timestamp, modification_timestamp, pers_id_registerer, del_id, orig_del, space_id, samp_id_part_of, pers_id_modifier, code_unique_check, subcode_unique_check, version, proj_id) FROM stdin;
-1 20161010123907442-1 DEFAULT 1 1 2016-10-10 12:39:07.442524+02 2016-10-10 12:39:07.442524+02 1 \N \N 1 \N \N DEFAULT,-1,-1,1 \N 0 \N
-2 20161010125004966-2 PLATE 2 4 2016-10-10 12:50:06.971812+02 2016-10-10 12:50:06.971812+02 3 \N \N 2 \N 3 PLATE,-1,-1,2 \N 0 \N
+1 20161010123907442-1 DEFAULT 1 1 2016-10-10 12:39:07.442524+02 2016-10-10 12:39:07.442524+02 1 \N \N 1 \N \N DEFAULT,-1,1,1 \N 0 1
+2 20161010125004966-2 PLATE 2 4 2016-10-10 12:50:06.971812+02 2016-10-10 12:50:06.971812+02 3 \N \N 2 \N 3 PLATE,-1,2,2 \N 0 2
+4 20180831173535244-10 TEST_PROJECT_SAMPLE \N 1 2018-08-31 17:35:35.244317+02 2018-08-31 17:35:35.244317+02 2 \N \N 2 \N 2 TEST_PROJECT_SAMPLE,-1,2,2 \N 0 2
+5 20180831173606541-11 TEST_SPACE_SAMPLE \N 1 2018-08-31 17:36:06.541746+02 2018-08-31 17:36:06.541746+02 2 \N \N 2 \N 2 TEST_SPACE_SAMPLE,-1,-1,2 \N 0 \N
+6 20180831173625312-12 TEST_SHARED_SAMPLE \N 1 2018-08-31 17:36:25.31212+02 2018-08-31 17:36:25.31212+02 2 \N \N \N \N 2 TEST_SHARED_SAMPLE,-1,-1,-1 \N 0 \N
+7 20180831173705648-13 TEST_EXPERIMENT_SAMPLE 2 1 2018-08-31 17:37:05.64806+02 2018-08-31 17:37:05.64806+02 2 \N \N 2 \N 2 TEST_EXPERIMENT_SAMPLE,-1,2,2 \N 0 2
\.
diff --git a/openbis_api/source/java/ch/ethz/sis/openbis/generic/asapi/v3/dto/sample/Sample.java b/openbis_api/source/java/ch/ethz/sis/openbis/generic/asapi/v3/dto/sample/Sample.java
index 1bef10b2218..134a4500a4f 100644
--- a/openbis_api/source/java/ch/ethz/sis/openbis/generic/asapi/v3/dto/sample/Sample.java
+++ b/openbis_api/source/java/ch/ethz/sis/openbis/generic/asapi/v3/dto/sample/Sample.java
@@ -27,6 +27,7 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.interfaces.IModificationD
import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.interfaces.IModifierHolder;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.interfaces.IParentChildrenHolder;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.interfaces.IPermIdHolder;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.interfaces.IProjectHolder;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.interfaces.IPropertiesHolder;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.interfaces.IRegistrationDateHolder;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.interfaces.IRegistratorHolder;
@@ -60,7 +61,7 @@ import java.util.Set;
* Class automatically generated with DtoGenerator
*/
@JsonObject("as.dto.sample.Sample")
-public class Sample implements Serializable, IAttachmentsHolder, ICodeHolder, IDataSetsHolder, IEntityTypeHolder, IExperimentHolder, IIdentifierHolder, IMaterialPropertiesHolder, IModificationDateHolder, IModifierHolder, IParentChildrenHolder<Sample>, IPermIdHolder, IPropertiesHolder, IRegistrationDateHolder, IRegistratorHolder, ISpaceHolder, ITagsHolder
+public class Sample implements Serializable, IAttachmentsHolder, ICodeHolder, IDataSetsHolder, IEntityTypeHolder, IExperimentHolder, IIdentifierHolder, IMaterialPropertiesHolder, IModificationDateHolder, IModifierHolder, IParentChildrenHolder<Sample>, IPermIdHolder, IProjectHolder, IPropertiesHolder, IRegistrationDateHolder, IRegistratorHolder, ISpaceHolder, ITagsHolder
{
private static final long serialVersionUID = 1L;
@@ -236,6 +237,7 @@ public class Sample implements Serializable, IAttachmentsHolder, ICodeHolder, ID
// Method automatically generated with DtoGenerator
@JsonIgnore
+ @Override
public Project getProject()
{
if (getFetchOptions() != null && getFetchOptions().hasProject())
--
GitLab