From 1afe8c6f6d242d0cfe7285544af2228bd3efd7b3 Mon Sep 17 00:00:00 2001
From: gakin <gakin>
Date: Tue, 4 Apr 2017 13:02:50 +0000
Subject: [PATCH] SSDMSSDM-4584 NameTranslation for data coming from different
data sources, fix related to SSDM-5014, refactoring
SVN: 38013
---
.../synchronizer/EntitySynchronizer.java | 45 ++++++----
.../synchronizer/MasterDataParser.java | 46 +++++-----
.../synchronizer/MasterDataSynchronizer.java | 2 -
.../synchronizer/ResourceListParser.java | 88 +++++++++++++------
.../synchronizer/ResourceListParserData.java | 4 +-
.../translator/PrefixBasedNameTranslator.java | 24 ++++-
6 files changed, 135 insertions(+), 74 deletions(-)
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
index 54524d4f48d..d95d501e8af 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
@@ -35,6 +35,8 @@ import java.util.Properties;
import java.util.Set;
import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.collections.keyvalue.MultiKey;
+import org.apache.commons.collections.map.MultiKeyMap;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
@@ -74,6 +76,7 @@ import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronize
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.translator.PrefixBasedNameTranslator;
import ch.ethz.sis.openbis.generic.shared.entitygraph.EntityGraph;
import ch.ethz.sis.openbis.generic.shared.entitygraph.Node;
+import ch.systemsx.cisd.cifex.shared.basic.UserFailureException;
import ch.systemsx.cisd.common.concurrent.ITaskExecutor;
import ch.systemsx.cisd.common.concurrent.ParallelizedExecutor;
import ch.systemsx.cisd.common.exceptions.Status;
@@ -187,21 +190,31 @@ public class EntitySynchronizer
// Parse the resource list: This sends back all projects,
// experiments, samples and data sets contained in the XML together with their last modification date to be used for filtering
operationLog.info("parsing the resource list xml document");
- String dataSourcePrefix = config.getDataSourceAlias();
+ String dataSourceAlias = config.getDataSourceAlias();
INameTranslator nameTranslator = null;
- if (StringUtils.isBlank(dataSourcePrefix) == false)
+ if (config.isTranslateUsingDataSourceAlias() == true)
{
- nameTranslator = new PrefixBasedNameTranslator(dataSourcePrefix);
+ if (StringUtils.isBlank(dataSourceAlias) == true)
+ {
+ throw new UserFailureException("Please specify a data source alias in the config file to be used in name translations.");
+ }
+ else
+ {
+ nameTranslator = new PrefixBasedNameTranslator(dataSourceAlias);
+ }
}
- ResourceListParser parser = ResourceListParser.create(nameTranslator, dataStoreCode);
+ // even when do not translate other things, we always translate space name.
+ // TODO re-think
+ ResourceListParser parser = ResourceListParser.create(nameTranslator, new PrefixBasedNameTranslator(dataSourceAlias), dataStoreCode);
ResourceListParserData data = parser.parseResourceListDocument(doc);
processDeletions(data);
operationLog.info("registering master data");
- masterDataSyncronizer = new MasterDataSynchronizer(config.getHarvesterUser(), config.getHarvesterPass(), data.getMasterData());
+ masterDataSyncronizer =
+ new MasterDataSynchronizer(config.getHarvesterUser(), config.getHarvesterPass(), data.getMasterData());
registerMasterData(data);
AtomicEntityOperationDetailsBuilder builder = new AtomicEntityOperationDetailsBuilder();
@@ -233,7 +246,7 @@ public class EntitySynchronizer
data.filterPhysicalDataSetsByLastModificationDate(lastSyncTimestamp, dataSetsCodesToRetry);
operationLog.info("Registering data sets...");
DataSetRegistrationSummary dsRegistrationSummary = registerPhysicalDataSets(physicalDSMap);
- // backup the current not synced data set codes file, delete the original file
+ // backup the current not synched data set codes file, delete the original file
saveFailedEntitiesFile(dsRegistrationSummary.notRegisteredDataSetCodes, notSyncedAttachmentsHolders);
@@ -491,7 +504,7 @@ public class EntitySynchronizer
Set<String> incomingExperimentPermIds = data.getExperimentsToProcess().keySet();
Set<String> incomingSamplePermIds = data.getSamplesToProcess().keySet();
Set<String> incomingDataSetCodes = data.getDataSetsToProcess().keySet();
- Set<String> incomingMaterialCodes = data.getMaterialsToProcess().keySet();
+ MultiKeyMap<String, MaterialWithLastModificationDate> incomingMaterials = data.getMaterialsToProcess();
// find projects, experiments, samples and data sets to be deleted
List<ProjectPermId> projectPermIds = new ArrayList<ProjectPermId>();
@@ -588,7 +601,7 @@ public class EntitySynchronizer
for (ch.ethz.sis.openbis.generic.asapi.v3.dto.material.Material material : materials)
{
- if (incomingMaterialCodes.contains(material.getCode()) == false)
+ if (incomingMaterials.containsKey(material.getCode(), material.getType().getCode()) == false)
{
matPermIds.add(new MaterialPermId(material.getCode(), material.getType().getCode()));
}
@@ -613,6 +626,11 @@ public class EntitySynchronizer
expDeletionOpts.setReason("sync experiment deletions");
IDeletionId expDeletionId = v3Api.deleteExperiments(sessionToken, experimentPermIds, expDeletionOpts);
+ // confirm deletions: Deletions need be confirm in the right order because of dependencies(foreign key constraints)
+ v3Api.confirmDeletions(sessionToken, Collections.singletonList(dsDeletionId));
+ v3Api.confirmDeletions(sessionToken, Collections.singletonList(smpDeletionId));
+ v3Api.confirmDeletions(sessionToken, Collections.singletonList(expDeletionId));
+
// delete projects
ProjectDeletionOptions prjDeletionOpts = new ProjectDeletionOptions();
prjDeletionOpts.setReason("Sync projects");
@@ -629,10 +647,7 @@ public class EntitySynchronizer
{
operationLog.warn("One or more materials could not be deleted due to: " + e.getMessage());
}
-
- // confirm deletions
- ArrayList<IDeletionId> deletionIds = new ArrayList<IDeletionId>();
-
+
StringBuffer summary = new StringBuffer();
if (projectPermIds.size() > 0)
{
@@ -644,20 +659,16 @@ public class EntitySynchronizer
}
if (expDeletionId != null)
{
- deletionIds.add(expDeletionId);
summary.append(experimentPermIds.size() + " experiments,");
}
if (smpDeletionId != null)
{
- deletionIds.add(smpDeletionId);
summary.append(samplePermIds.size() + " samples,");
}
if (dsDeletionId != null)
{
- deletionIds.add(dsDeletionId);
summary.append(dsPermIds.size() + " data sets");
}
- v3Api.confirmDeletions(sessionToken, deletionIds); // Arrays.asList(expDeletionId, dsDeletionId, smpDeletionId)
if (summary.length() > 0)
{
@@ -747,7 +758,7 @@ public class EntitySynchronizer
private void processMaterials(ResourceListParserData data, AtomicEntityOperationDetailsBuilder builder)
{
// process materials
- Map<String, MaterialWithLastModificationDate> materialsToProcess = data.getMaterialsToProcess();
+ Map<MultiKey<String>, MaterialWithLastModificationDate> materialsToProcess = data.getMaterialsToProcess();
for (MaterialWithLastModificationDate newMaterialWithType : materialsToProcess.values())
{
NewMaterialWithType incomingMaterial = newMaterialWithType.getMaterial();
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MasterDataParser.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MasterDataParser.java
index a2a134250b5..12e042a1e4f 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MasterDataParser.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MasterDataParser.java
@@ -29,6 +29,7 @@ import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathExpressionException;
import org.apache.commons.collections.map.MultiKeyMap;
+import org.apache.commons.lang3.StringUtils;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
@@ -184,17 +185,14 @@ public class MasterDataParser
for (int i = 0; i < pluginNodes.getLength(); i++)
{
Element pluginElement = (Element) pluginNodes.item(i);
-
Script plugin = new Script();
- // String code = getAttribute(pluginElement, "code");
- plugin.setName(getAttribute(pluginElement, "name"));
+ plugin.setName(nameTranslator.translate(getAttribute(pluginElement, "name")));
plugin.setDescription(getAttribute(pluginElement, "description"));
String entityKind = getAttribute(pluginElement, "entityKind").trim();
plugin.setScriptType(ScriptType.valueOf(getAttribute(pluginElement, "type")));
plugin.setPluginType(PluginType.JYTHON);
plugin.setEntityKind(entityKind.equals("") ? null : new EntityKind[] { EntityKind.valueOf(entityKind) });
plugin.setScript(pluginElement.getTextContent());
-
validationPlugins.put(plugin.getName(), plugin);
}
}
@@ -245,8 +243,7 @@ public class MasterDataParser
for (int i = 0; i < vocabNodes.getLength(); i++)
{
Element vocabElement = (Element) vocabNodes.item(i);
- String code = getAttribute(vocabElement, "code");
-
+ String code = nameTranslator.translate(getAttribute(vocabElement, "code"));
NewVocabulary newVocabulary = new NewVocabulary();
newVocabulary.setCode(code);
newVocabulary.setDescription(getAttribute(vocabElement, "description"));
@@ -299,7 +296,7 @@ public class MasterDataParser
{
Element materialTypeElement = (Element) matTypeNodes.item(i);
MaterialType materialType = new MaterialType();
- materialType.setCode(getAttribute(materialTypeElement, "code"));
+ materialType.setCode(nameTranslator.translate(getAttribute(materialTypeElement, "code")));
materialType.setDescription(getAttribute(materialTypeElement, "description"));
String validationPluginName = getAttribute(materialTypeElement, "validationPlugin");
if (validationPluginName != null)
@@ -308,7 +305,7 @@ public class MasterDataParser
}
materialTypes.put(materialType.getCode(), materialType);
- parsePropertyAssignments(EntityKind.MATERIAL, materialType, materialTypeElement.getElementsByTagName("propertyAssignments"));
+ parsePropertyAssignments(EntityKind.MATERIAL, materialType, materialTypeElement);
}
}
@@ -325,9 +322,8 @@ public class MasterDataParser
for (int i = 0; i < expTypeNodes.getLength(); i++)
{
Element expTypeElement = (Element) expTypeNodes.item(i);
- String code = getAttribute(expTypeElement, "code");
ExperimentType expType = new ExperimentType();
- expType.setCode(code);
+ expType.setCode(nameTranslator.translate(getAttribute(expTypeElement, "code")));
expType.setDescription(getAttribute(expTypeElement, "description"));
String validationPluginName = getAttribute(expTypeElement, "validationPlugin");
if (validationPluginName != null)
@@ -336,7 +332,7 @@ public class MasterDataParser
}
experimentTypes.put(expType.getCode(), expType);
- parsePropertyAssignments(EntityKind.EXPERIMENT, expType, expTypeElement.getElementsByTagName("xmd:propertyAssignments"));
+ parsePropertyAssignments(EntityKind.EXPERIMENT, expType, expTypeElement);
}
}
@@ -354,7 +350,7 @@ public class MasterDataParser
{
Element sampleTypeElement = (Element) sampleTypeNodes.item(i);
SampleType sampleType = new SampleType();
- sampleType.setCode(getAttribute(sampleTypeElement, "code"));
+ sampleType.setCode(nameTranslator.translate(getAttribute(sampleTypeElement, "code")));
sampleType.setDescription(getAttribute(sampleTypeElement, "description"));
sampleType.setListable(Boolean.valueOf(getAttribute(sampleTypeElement, "listable")));
sampleType.setShowContainer(Boolean.valueOf(getAttribute(sampleTypeElement, "showContainer")));
@@ -370,7 +366,7 @@ public class MasterDataParser
}
sampleTypes.put(sampleType.getCode(), sampleType);
- parsePropertyAssignments(EntityKind.SAMPLE, sampleType, sampleTypeElement.getElementsByTagName("xmd:propertyAssignments"));
+ parsePropertyAssignments(EntityKind.SAMPLE, sampleType, sampleTypeElement);
}
}
@@ -387,9 +383,8 @@ public class MasterDataParser
for (int i = 0; i < dataSetTypeNodes.getLength(); i++)
{
Element dataSetTypeElement = (Element) dataSetTypeNodes.item(i);
- String code = getAttribute(dataSetTypeElement, "code");
DataSetType dataSetType = new DataSetType();
- dataSetType.setCode(code);
+ dataSetType.setCode(nameTranslator.translate(getAttribute(dataSetTypeElement, "code")));
dataSetType.setDescription(getAttribute(dataSetTypeElement, "description"));
dataSetType.setDataSetKind(DataSetKind.valueOf(getAttribute(dataSetTypeElement, "dataSetKind")));
String mainDataSetPattern = getAttribute(dataSetTypeElement, "mainDataSetPattern");
@@ -417,13 +412,14 @@ public class MasterDataParser
}
dataSetTypes.put(dataSetType.getCode(), dataSetType);
- parsePropertyAssignments(EntityKind.DATA_SET, dataSetType, dataSetTypeElement.getElementsByTagName("xmd:propertyAssignments"));
+ parsePropertyAssignments(EntityKind.DATA_SET, dataSetType, dataSetTypeElement);
}
}
- private void parsePropertyAssignments(EntityKind entityKind, EntityType entityType, NodeList propertyAssignmentsNode)
+ private void parsePropertyAssignments(EntityKind entityKind, EntityType entityType, Element entityTypeElement)
throws XPathExpressionException
{
+ NodeList propertyAssignmentsNode = entityTypeElement.getElementsByTagName("xmd:propertyAssignments");
if (propertyAssignmentsNode.getLength() == 0)
{
return;
@@ -438,6 +434,7 @@ public class MasterDataParser
{
Element propertyAssignmentElement = (Element) propertyAssignmentNodes.item(i);
String propertyTypeCode = getAttribute(propertyAssignmentElement, "propertyTypeCode");
+ propertyTypeCode = nameTranslator.translate(propertyTypeCode);
NewETPTAssignment assignment = new NewETPTAssignment();
assignment.setPropertyTypeCode(propertyTypeCode);
assignment.setEntityKind(entityType.getEntityKind());
@@ -465,33 +462,34 @@ public class MasterDataParser
for (int i = 0; i < propertyTypeNodes.getLength(); i++)
{
Element propertyTypeElement = (Element) propertyTypeNodes.item(i);
- PropertyType newPropertyType = new PropertyType();
- String code = getAttribute(propertyTypeElement, "code");
+ PropertyType newPropertyType = new PropertyType();
+ Boolean isInternalNameSpace = Boolean.valueOf(getAttribute(propertyTypeElement, "internalNamespace"));
+ String code = nameTranslator.translate(getAttribute(propertyTypeElement, "code"));
+ newPropertyType.setCode(CodeConverter.tryToBusinessLayer(code, isInternalNameSpace));
newPropertyType.setLabel(getAttribute(propertyTypeElement, "label"));
DataTypeCode dataTypeCode = DataTypeCode.valueOf(getAttribute(propertyTypeElement, "dataType"));
newPropertyType.setDataType(new DataType(dataTypeCode));
newPropertyType.setDescription(getAttribute(propertyTypeElement, "description"));
+ newPropertyType.setInternalNamespace(isInternalNameSpace);
newPropertyType.setManagedInternally(Boolean.valueOf(getAttribute(propertyTypeElement, "managedInternally")));
- newPropertyType.setInternalNamespace(Boolean.valueOf(getAttribute(propertyTypeElement, "internalNamespace")));
- newPropertyType.setCode(CodeConverter.tryToBusinessLayer(code, newPropertyType.isInternalNamespace()));
propertyTypes.put(newPropertyType.getCode(), newPropertyType);
if (dataTypeCode.equals(DataTypeCode.CONTROLLEDVOCABULARY))
{
- String vocabularyCode = getAttribute(propertyTypeElement, "vocabulary");
+ String vocabularyCode = nameTranslator.translate((getAttribute(propertyTypeElement, "vocabulary")));
newPropertyType.setVocabulary(vocabularies.get(vocabularyCode));
}
else if (dataTypeCode.equals(DataTypeCode.MATERIAL))
{
String materialCode = getAttribute(propertyTypeElement, "material");
- if (materialCode.trim().length() < 1)
+ if (StringUtils.isBlank(materialCode))
{
newPropertyType.setMaterialType(null); // material of any type
}
else
{
- newPropertyType.setMaterialType(materialTypes.get(materialCode));
+ newPropertyType.setMaterialType(materialTypes.get(nameTranslator.translate(materialCode)));
}
}
}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MasterDataSynchronizer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MasterDataSynchronizer.java
index a417361f20a..3cae25520e4 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MasterDataSynchronizer.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MasterDataSynchronizer.java
@@ -180,8 +180,6 @@ public class MasterDataSynchronizer
existingVocabulary.setURLTemplate(newVocabulary.getURLTemplate());
existingVocabulary.setChosenFromList(newVocabulary.isChosenFromList());
commonServer.updateVocabulary(sessionToken, existingVocabulary);
- // vocabulary.setTerms(newVocabulary.getTerms());
-
processVocabularyTerms(sessionToken, commonServer, newVocabulary, existingVocabulary);
}
else
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParser.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParser.java
index 4e4756b2ed9..317deec3302 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParser.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParser.java
@@ -15,6 +15,9 @@
*/
package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer;
+import static ch.systemsx.cisd.openbis.generic.shared.basic.dto.MaterialIdentifier.TYPE_SEPARATOR_PREFIX;
+import static ch.systemsx.cisd.openbis.generic.shared.basic.dto.MaterialIdentifier.TYPE_SEPARATOR_SUFFIX;
+
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
@@ -51,9 +54,12 @@ import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronize
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.ResourceListParserData.MaterialWithLastModificationDate;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.translator.DefaultNameTranslator;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.translator.INameTranslator;
+import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.translator.PrefixBasedNameTranslator;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetType;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataTypeCode;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.EntityProperty;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IEntityProperty;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.MaterialIdentifier;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.NewAttachment;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.NewExperiment;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.NewMaterialWithType;
@@ -80,6 +86,8 @@ public class ResourceListParser
private final INameTranslator nameTranslator;
+ private final INameTranslator spaceNameTranslator;
+
private final String dataStoreCode;
public INameTranslator getNameTranslator()
@@ -87,25 +95,26 @@ public class ResourceListParser
return nameTranslator;
}
- private ResourceListParser(INameTranslator nameTranslator, String dataStoreCode)
+ private ResourceListParser(INameTranslator nameTranslator, PrefixBasedNameTranslator spaceNameTranslator, String dataStoreCode)
{
this.data = new ResourceListParserData();
this.nameTranslator = nameTranslator;
this.dataStoreCode = dataStoreCode;
+ this.spaceNameTranslator = spaceNameTranslator;
}
- public static ResourceListParser create(INameTranslator nameTranslator, String dataStoreCode)
+ public static ResourceListParser create(INameTranslator nameTranslator, PrefixBasedNameTranslator spaceNameTranslator, String dataStoreCode)
{
if (nameTranslator == null)
{
- return create(dataStoreCode);
+ return create(spaceNameTranslator, dataStoreCode);
}
- return new ResourceListParser(nameTranslator, dataStoreCode);
+ return new ResourceListParser(nameTranslator, spaceNameTranslator, dataStoreCode);
}
- public static ResourceListParser create(String dataStoreCode)
+ public static ResourceListParser create(PrefixBasedNameTranslator spaceNameTranslator, String dataStoreCode)
{
- return create(new DefaultNameTranslator(), dataStoreCode);
+ return create(new DefaultNameTranslator(), spaceNameTranslator, dataStoreCode);
}
public ResourceListParserData parseResourceListDocument(Document doc) throws XPathExpressionException
@@ -227,7 +236,7 @@ public class ResourceListParser
{
parseDataSetMetaData(xpath, extractDataSetCodeFromURI(uri), xdNode, lastModificationDate);
}
- else if (EntityKind.MATERIAL.equals(entityKind))
+ else if (EntityKind.MATERIAL.getLabel().equals(entityKind))
{
parseMaterialMetaData(xpath, extractMaterialCodeFromURI(uri), xdNode, lastModificationDate);
}
@@ -379,6 +388,11 @@ public class ResourceListParser
incomingProject.setHasAttachments(hasAttachments(xpath, xdNode));
}
+ private ExperimentIdentifier createExperimentIdentifier(String spaceId, String prjCode, String expCode)
+ {
+ return new ExperimentIdentifier(createProjectIdentifier(prjCode, spaceId), expCode);
+ }
+
private ProjectIdentifier createProjectIdentifier(String code, String space)
{
return new ProjectIdentifier(createSpaceIdentifier(space), code);
@@ -401,12 +415,12 @@ public class ResourceListParser
private void parseMaterialMetaData(XPath xpath, String permId, Node xdNode, Date lastModificationDate)
{
- String code = extractCode(xdNode);
+ String code = nameTranslator.translate(extractCode(xdNode));
String type = extractType(xdNode);
NewMaterialWithType newMaterial = new NewMaterialWithType(code, type);
MaterialWithLastModificationDate materialWithLastModDate =
data.new MaterialWithLastModificationDate(newMaterial, lastModificationDate);
- data.getMaterialsToProcess().put(code, materialWithLastModDate);
+ data.getMaterialsToProcess().put(code, type, materialWithLastModDate);
newMaterial.setProperties(parseProperties(xpath, xdNode));
}
@@ -442,22 +456,11 @@ public class ResourceListParser
private List<NewProperty> parseDataSetProperties(XPath xpath, Node xdNode)
{
-
+ EntityProperty[] entityProperties = parseProperties(xpath, xdNode);
List<NewProperty> dsProperties = new ArrayList<NewProperty>();
- Element docElement = (Element) xdNode;
- NodeList propsNode = docElement.getElementsByTagName("x:properties");
- if (propsNode.getLength() == 1)
+ for (EntityProperty entityProperty : entityProperties)
{
- Element propsElement = (Element) propsNode.item(0);
- NodeList propertyNodes = propsElement.getElementsByTagName("x:property");
- for (int i = 0; i < propertyNodes.getLength(); i++)
- {
- Element propertyElement = (Element) propertyNodes.item(i);
- // TODO proper error handling needed below in case the XML is not correct and item 0 does not exist
- String code = propertyElement.getElementsByTagName("x:code").item(0).getTextContent();
- String val = propertyElement.getElementsByTagName("x:value").item(0).getTextContent();
- dsProperties.add(new NewProperty(code, val));
- }
+ dsProperties.add(new NewProperty(entityProperty.getPropertyType().getCode(), entityProperty.getValue()));
}
return dsProperties;
}
@@ -489,19 +492,50 @@ public class ResourceListParser
{
EntityProperty property = new EntityProperty();
PropertyType propertyType = new PropertyType();
- propertyType.setCode(code);
+ String translatedCode = nameTranslator.translate(code);
+ PropertyType pt = data.getMasterData().getPropertyTypesToProcess().get(translatedCode);
+ if (pt.getDataType().getCode().equals(DataTypeCode.MATERIAL))
+ {
+ if (val != null)
+ {
+ val = nameTranslator.translate(val);
+ val = translateMaterialIdentifier(val);
+ }
+
+ }
+ propertyType.setCode(translatedCode);
property.setPropertyType(propertyType);
property.setValue(val);
return property;
}
+ private String translateMaterialIdentifier(String value) {
+ if (StringUtils.isBlank(value))
+ {
+ return null;
+ }
+ int typePrefix = value.indexOf(TYPE_SEPARATOR_PREFIX);
+ if (typePrefix == -1)
+ {
+ return null;
+ }
+ String code = value.substring(0, typePrefix);
+ String typeCode = nameTranslator.translate(value.substring(typePrefix + TYPE_SEPARATOR_PREFIX.length()));
+ // we allow to omit the closing brace
+ if (typeCode.endsWith(TYPE_SEPARATOR_SUFFIX))
+ {
+ typeCode = typeCode.substring(0, typeCode.length() - TYPE_SEPARATOR_SUFFIX.length());
+ }
+ return new MaterialIdentifier(code, typeCode).toString();
+ }
private void parseExperimentMetaData(XPath xpath, String permId, Node xdNode, Date lastModificationDate)
{
String code = extractCode(xdNode);
String type = extractType(xdNode);
String project = extractAttribute(xdNode, "project");
String space = extractSpace(xdNode, false);
- NewExperiment newExp = new NewExperiment("/" + nameTranslator.translate(space) + "/" + project + "/" + code, type);
+ ExperimentIdentifier experimentIdentifier = createExperimentIdentifier(space, project, code);
+ NewExperiment newExp = new NewExperiment(experimentIdentifier.toString(), type);
newExp.setPermID(permId);
IncomingExperiment incomingExperiment = data.new IncomingExperiment(newExp, lastModificationDate);
data.getExperimentsToProcess().put(permId, incomingExperiment);
@@ -537,7 +571,7 @@ public class ResourceListParser
private String extractType(Node xdNode)
{
- return extractAttribute(xdNode, "type");
+ return nameTranslator.translate(extractAttribute(xdNode, "type"));
}
private String extractSpace(Node xdNode, boolean nullAllowed)
@@ -545,7 +579,7 @@ public class ResourceListParser
String space = extractAttribute(xdNode, "space", nullAllowed);
if (space != null)
{
- data.getHarvesterSpaceList().add(nameTranslator.translate(space));
+ data.getHarvesterSpaceList().add(spaceNameTranslator.translate(space));
}
return space;
}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParserData.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParserData.java
index 036c18f0ab7..52a05b820ab 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParserData.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParserData.java
@@ -66,7 +66,7 @@ public class ResourceListParserData
private Map<String, IncomingDataSet> dataSetsToProcess = new HashMap<String, ResourceListParserData.IncomingDataSet>();
- private Map<String, MaterialWithLastModificationDate> materialsToProcess = new HashMap<String, MaterialWithLastModificationDate>();
+ private MultiKeyMap<String, MaterialWithLastModificationDate> materialsToProcess = new MultiKeyMap<String, MaterialWithLastModificationDate>();
public MasterData getMasterData()
{
@@ -108,7 +108,7 @@ public class ResourceListParserData
return dataSetsToProcess;
}
- public Map<String, MaterialWithLastModificationDate> getMaterialsToProcess()
+ public MultiKeyMap<String, MaterialWithLastModificationDate> getMaterialsToProcess()
{
return materialsToProcess;
}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/translator/PrefixBasedNameTranslator.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/translator/PrefixBasedNameTranslator.java
index bfc2ccc5646..6b879ce25de 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/translator/PrefixBasedNameTranslator.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/translator/PrefixBasedNameTranslator.java
@@ -16,6 +16,8 @@
package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.translator;
+import static ch.systemsx.cisd.openbis.generic.shared.basic.BasicConstant.INTERNAL_NAMESPACE_PREFIX;
+
import org.apache.commons.lang3.StringUtils;
/**
@@ -32,11 +34,29 @@ public class PrefixBasedNameTranslator implements INameTranslator
this.prefix = prefix;
}
+ private String translateInternal(String name)
+ {
+ return prefix + "_" + name;
+ }
+
+ /**
+ * INTERNAL_NAMESPACE_PREFIX is checked because of the following cases following cases: 1. While parsing master data, for property types with dataType = CONTROLLEDVOCABULARY or
+ * MATERIAL in which case the vocabulary or material attribute might start with $ (INTERNAL_NAMESPACE_PREFIX) 2. While parsing master data, for
+ * property assignments where property type code might start with $ (INTERNAL_NAMESPACE_PREFIX) the prop. assignment element in the incoming xml
+ * will start with $ if propertyTypeCode is internalNamespace. 3. While parsing meta data, the property code will will start with $ if
+ * propertyTypeCode is internalNamespac
+ */
@Override
public String translate(String name)
{
assert StringUtils.isBlank(name) == false : "Prefix translation can only be done for non-null values";
- return prefix + "_" + name;
+ if (name.startsWith(INTERNAL_NAMESPACE_PREFIX))
+ {
+ return INTERNAL_NAMESPACE_PREFIX + translateInternal(name.substring(INTERNAL_NAMESPACE_PREFIX.length()));
+ }
+ else
+ {
+ return translateInternal(name);
+ }
}
-
}
--
GitLab