From e4a6fdffabf415e52bfc96c32c88fc18a2ff8bd4 Mon Sep 17 00:00:00 2001
From: gakin <gakin>
Date: Tue, 4 Oct 2016 14:42:51 +0000
Subject: [PATCH] SSDM-4197 : Implement basic black list, make data source
space configuration optional, create space through performentityoperations if
it does not exist, implement basic name translation strategy
SVN: 37144
---
.../harvester/DataSetRegistrationTask.java | 2 +-
.../harvester/config/ConfigReader.java | 4 +-
.../plugins/harvester/config/SyncConfig.java | 8 ++
.../config/SynchronizationConfigReader.java | 17 +++-
.../synchronizer/DataSourceConnector.java | 23 +++--
.../synchronizer/EntitySynchronizer.java | 30 +++++-
.../synchronizer/ResourceListParser.java | 98 +++++++++++++------
.../synchronizer/ResourceListParserData.java | 9 ++
.../translator/CustomNameTranslator.java | 53 ++++++++++
.../translator/DefaultNameTranslator.java | 32 ++++++
.../translator/INameTranslator.java | 27 +++++
.../translator/PrefixBasedNameTranslator.java | 39 ++++++++
.../generic/server/EntityRetriever.java | 12 ++-
.../AtomicEntityOperationDetailsBuilder.java | 7 ++
14 files changed, 310 insertions(+), 51 deletions(-)
create mode 100644 datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/translator/CustomNameTranslator.java
create mode 100644 datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/translator/DefaultNameTranslator.java
create mode 100644 datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/translator/INameTranslator.java
create mode 100644 datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/translator/PrefixBasedNameTranslator.java
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/DataSetRegistrationTask.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/DataSetRegistrationTask.java
index 9dd76658bbc..9566365fc77 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/DataSetRegistrationTask.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/DataSetRegistrationTask.java
@@ -134,7 +134,7 @@ public class DataSetRegistrationTask<T extends DataSetInformation> implements IM
try
{
operationLog
- .info("Start synchronization from data source: " + config.getDataSourceOpenbisURL() + " spaces:" + spaceMappings.keySet());
+ .info("Start synchronization from data source: " + config.getDataSourceOpenbisURL() + " for user " + config.getUser());
String fileName = config.getLastSyncTimestampFileName();
lastSyncTimestampFile = new File(fileName);
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/config/ConfigReader.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/config/ConfigReader.java
index db07d22280e..84be30be2d8 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/config/ConfigReader.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/config/ConfigReader.java
@@ -36,6 +36,8 @@ import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
public class ConfigReader
{
+ private static final String IGNORE_LINE_CHAR = "#";
+
private Pattern sectionRegex = Pattern.compile("\\s*\\[([^]]*)\\]\\s*");
private Pattern keyValueRegex = Pattern.compile("\\s*([^=]*)=(.*)");
@@ -110,7 +112,7 @@ public class ConfigReader
else if (section != null)
{
m = keyValueRegex.matcher(line);
- if (m.matches())
+ if (m.matches() && line.startsWith(IGNORE_LINE_CHAR) == false)
{
String key = m.group(1).trim();
String value = m.group(2).trim();
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/config/SyncConfig.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/config/SyncConfig.java
index 29329b4297c..4674ffb679e 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/config/SyncConfig.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/config/SyncConfig.java
@@ -89,6 +89,10 @@ public class SyncConfig
public void setDataSourceSpaces(String dataSourceSpaces)
{
+ if (dataSourceSpaces == null)
+ {
+ return;
+ }
for (String token : dataSourceSpaces.split(SEPARATOR))
{
this.dataSourceSpaces.add(token.trim());
@@ -102,6 +106,10 @@ public class SyncConfig
public void setHarvesterSpaces(String harvesterSpaces)
{
+ if (harvesterSpaces == null)
+ {
+ return;
+ }
for (String token : harvesterSpaces.split(SEPARATOR))
{
this.harvesterSpaces.add(token.trim());
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/config/SynchronizationConfigReader.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/config/SynchronizationConfigReader.java
index 10e75638646..85960387f21 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/config/SynchronizationConfigReader.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/config/SynchronizationConfigReader.java
@@ -92,9 +92,20 @@ public class SynchronizationConfigReader
String pass = reader.getString(section, DATA_SOURCE_AUTH_PASS_PROPERTY_NAME, null, true);
config.setAuthCredentials(realm, user, pass);
- config.setDataSourceSpaces(reader.getString(section, DATA_SOURCE_SPACES_PROPERTY_NAME, null, true));
- config.setHarvesterSpaces(reader.getString(section, HARVESTER_SPACES_PROPERTY_NAME, null, true));
- createDataSourceToHarvesterSpaceMappings(config);
+ String dsSpaces = reader.getString(section, DATA_SOURCE_SPACES_PROPERTY_NAME, null, false);
+ if (dsSpaces != null)
+ {
+ config.setDataSourceSpaces(dsSpaces);
+ }
+ String hrvSpaces = reader.getString(section, HARVESTER_SPACES_PROPERTY_NAME, null, false);
+ if (hrvSpaces != null)
+ {
+ config.setHarvesterSpaces(hrvSpaces);
+ }
+ if (dsSpaces != null && hrvSpaces != null)
+ {
+ createDataSourceToHarvesterSpaceMappings(config);
+ }
config.setDataSourcePrefix(reader.getString(section, DATA_SOURCE_PREFIX_PROPERTY_NAME, null, false));
config.setHarvesterTempDir(reader.getString(section, HARVESTER_TEMP_DIR_PROPERTY_NAME, "targets/store", false));
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/DataSourceConnector.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/DataSourceConnector.java
index f891867b643..09bf7312f8b 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/DataSourceConnector.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/DataSourceConnector.java
@@ -58,14 +58,14 @@ public class DataSourceConnector
this.authCredentials = authCredentials;
}
- public Document getResourceListAsXMLDoc(List<String> spaces) throws ParserConfigurationException, SAXException, IOException,
+ public Document getResourceListAsXMLDoc(List<String> spaceBlackList) throws ParserConfigurationException, SAXException, IOException,
XPathExpressionException,
URISyntaxException,
InterruptedException, TimeoutException, ExecutionException
{
HttpClient client = JettyHttpClientFactory.getHttpClient();
addAuthenticationCredentials(client);
- Request requestEntity = createNewHttpRequest(client, spaces);
+ Request requestEntity = createNewHttpRequest(client, spaceBlackList);
ContentResponse contentResponse = getResponse(requestEntity);
return parseResponse(contentResponse);
}
@@ -89,23 +89,26 @@ public class DataSourceConnector
if (statusCode != HttpStatus.Code.OK.getCode())
{
- throw new IOException("Status Code was " + statusCode + " instead of " + HttpStatus.Code.OK.getCode());
+ throw new IOException("Resource List could not be retrieved: " + contentResponse.getContentAsString());
}
return contentResponse;
}
- private Request createNewHttpRequest(HttpClient client, List<String> spaces)
+ private Request createNewHttpRequest(HttpClient client, List<String> spaceBlackList)
{
- String spacesParam = "";
- for (String dataSourceSpace : spaces)
+ StringBuffer sb = new StringBuffer();
+ for (String dataSourceSpace : spaceBlackList)
{
- spacesParam += dataSourceSpace + ",";
+ sb.append(dataSourceSpace + ",");
}
- if (spacesParam.isEmpty() == false)
+ String req = dataSourceUrl + "?verb=resourcelist.xml";
+ if (sb.length() != 0)
{
- spacesParam = spacesParam.substring(0, spacesParam.length() - 1);
+ String str = sb.toString();
+ str = str.substring(0, str.length() - 1);
+ req += "&black_list=" + str;
}
- Request requestEntity = client.newRequest(dataSourceUrl + "?verb=resourcelist.xml&spaces=" + spacesParam).method("GET");
+ Request requestEntity = client.newRequest(req).method("GET");
return requestEntity;
}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/EntitySynchronizer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/EntitySynchronizer.java
index a8a4e1a5c71..0bc54137ace 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/EntitySynchronizer.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/EntitySynchronizer.java
@@ -40,6 +40,7 @@ import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPathExpressionException;
import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.lang.ArrayUtils;
import org.apache.log4j.Logger;
import org.w3c.dom.Document;
import org.xml.sax.SAXException;
@@ -70,6 +71,8 @@ import ch.ethz.sis.openbis.generic.server.dss.plugins.harvester.synchronizer.Res
import ch.ethz.sis.openbis.generic.server.dss.plugins.harvester.synchronizer.ResourceListParserData.MaterialWithLastModificationDate;
import ch.ethz.sis.openbis.generic.server.dss.plugins.harvester.synchronizer.ResourceListParserData.ProjectWithConnections;
import ch.ethz.sis.openbis.generic.server.dss.plugins.harvester.synchronizer.ResourceListParserData.SampleWithConnections;
+import ch.ethz.sis.openbis.generic.server.dss.plugins.harvester.synchronizer.translator.INameTranslator;
+import ch.ethz.sis.openbis.generic.server.dss.plugins.harvester.synchronizer.translator.PrefixBasedNameTranslator;
import ch.ethz.sis.openbis.generic.shared.entitygraph.EntityGraph;
import ch.ethz.sis.openbis.generic.shared.entitygraph.Node;
import ch.systemsx.cisd.common.concurrent.ITaskExecutor;
@@ -100,9 +103,11 @@ import ch.systemsx.cisd.openbis.generic.shared.basic.dto.NewExperiment;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.NewMaterialWithType;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.NewProject;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.NewSample;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.NewSpace;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.PhysicalDataSet;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Project;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Sample;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Space;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.builders.PropertyBuilder;
import ch.systemsx.cisd.openbis.generic.shared.dto.AtomicEntityOperationResult;
import ch.systemsx.cisd.openbis.generic.shared.dto.DataSetBatchUpdatesDTO;
@@ -120,6 +125,7 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.ProjectIdentifier;
import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.ProjectIdentifierFactory;
import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifier;
import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifierFactory;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SpaceIdentifier;
/**
*
*
@@ -159,17 +165,33 @@ public class EntitySynchronizer
// retrieve the document from the data source
operationLog.info("Retrieving the resource list..");
DataSourceConnector connector = new DataSourceConnector(config.getDataSourceURI(), config.getAuthenticationCredentials());
- Document doc = connector.getResourceListAsXMLDoc(new ArrayList<String>(config.getSpaceMappings().keySet()));
+ Document doc = connector.getResourceListAsXMLDoc(Arrays.asList(ArrayUtils.EMPTY_STRING_ARRAY));
// Parse the resource list: This sends back all projects,
// experiments, samples and data sets contained in the XML together with their last modification date to be used for filtering
operationLog.info("parsing the resource list xml document");
- ResourceListParser parser = ResourceListParser.create(config.getSpaceMappings()); // , lastSyncTimestamp
+ String dataSourcePrefix = config.getDataSourcePrefix();
+ INameTranslator nameTranslator = null;
+ if (dataSourcePrefix != null && dataSourcePrefix.trim().equals("") == false)
+ {
+ nameTranslator = new PrefixBasedNameTranslator(dataSourcePrefix);
+ }
+ ResourceListParser parser = ResourceListParser.create(nameTranslator); // , lastSyncTimestamp
ResourceListParserData data = parser.parseResourceListDocument(doc);
processDeletions(data);
AtomicEntityOperationDetailsBuilder builder = new AtomicEntityOperationDetailsBuilder();
+
+ for (String spaceCode : data.getHarvesterSpaceList())
+ {
+ Space space = service.tryGetSpace(new SpaceIdentifier(spaceCode));
+ if (space == null)
+ {
+ builder.space(new NewSpace(spaceCode, "Synchronized from: " + config.getDataSourceURI(), null));
+ }
+ }
+
processMetaData(data, builder);
operationLog.info("Registering meta data...");
@@ -334,8 +356,6 @@ public class EntitySynchronizer
// service.commit();
}
-
-
private void processDeletions(ResourceListParserData data) throws NoSuchAlgorithmException, UnsupportedEncodingException
{
operationLog.info("Processing deletions");
@@ -358,7 +378,7 @@ public class EntitySynchronizer
Set<PhysicalDataSet> physicalDataSetsDelete = new HashSet<PhysicalDataSet>();
// first find out the entities to be deleted
- for (String harvesterSpaceId : config.getSpaceMappings().values())
+ for (String harvesterSpaceId : data.getHarvesterSpaceList())
{
EntityGraph<Node<?>> harvesterEntityGraph = entityRetriever.getEntityGraph(harvesterSpaceId);
List<Node<?>> entities = harvesterEntityGraph.getNodes();
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/ResourceListParser.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/ResourceListParser.java
index 332e90a5e3b..a474f978c20 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/ResourceListParser.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/ResourceListParser.java
@@ -20,7 +20,6 @@ import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
-import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
@@ -48,6 +47,8 @@ import ch.ethz.sis.openbis.generic.server.dss.plugins.harvester.synchronizer.Res
import ch.ethz.sis.openbis.generic.server.dss.plugins.harvester.synchronizer.ResourceListParserData.MaterialWithLastModificationDate;
import ch.ethz.sis.openbis.generic.server.dss.plugins.harvester.synchronizer.ResourceListParserData.ProjectWithConnections;
import ch.ethz.sis.openbis.generic.server.dss.plugins.harvester.synchronizer.ResourceListParserData.SampleWithConnections;
+import ch.ethz.sis.openbis.generic.server.dss.plugins.harvester.synchronizer.translator.DefaultNameTranslator;
+import ch.ethz.sis.openbis.generic.server.dss.plugins.harvester.synchronizer.translator.INameTranslator;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetType;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.EntityProperty;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IEntityProperty;
@@ -75,20 +76,37 @@ public class ResourceListParser
{
private final ResourceListParserData data;
+ private INameTranslator nameTranslator;
- private final HashMap<String, String> spaceMappings;
+ public void setNameTranslator(INameTranslator nameTranslator)
+ {
+ this.nameTranslator = nameTranslator;
+ }
+
+ public INameTranslator getNameTranslator()
+ {
+ return nameTranslator;
+ }
- private ResourceListParser(HashMap<String, String> spaceMappings)
+ private ResourceListParser(INameTranslator nameTranslator)
{
// TODO do the returning of parser data better
this.data = new ResourceListParserData();
- this.spaceMappings = spaceMappings;
+ this.nameTranslator = nameTranslator;
}
- public static ResourceListParser create(HashMap<String, String> spaceMappings)
+ public static ResourceListParser create(INameTranslator nameTranslator)
{
- ResourceListParser parser = new ResourceListParser(spaceMappings);
- return parser;
+ if (nameTranslator == null)
+ {
+ return create();
+ }
+ return new ResourceListParser(nameTranslator);
+ }
+
+ public static ResourceListParser create()
+ {
+ return create(new DefaultNameTranslator());
}
public ResourceListParserData parseResourceListDocument(Document doc) throws XPathExpressionException
@@ -256,11 +274,11 @@ public class ResourceListParser
private void parseDataSetMetaData(XPath xpath, String permId, Node xdNode, Date lastModificationDate)
{
- String code = xdNode.getAttributes().getNamedItem("code").getTextContent();
- String sample = xdNode.getAttributes().getNamedItem("sample").getTextContent();
- String experiment = xdNode.getAttributes().getNamedItem("experiment").getTextContent();
- String type = xdNode.getAttributes().getNamedItem("type").getTextContent();
- String dsKind = xdNode.getAttributes().getNamedItem("dsKind").getTextContent();
+ String code = extractCode(xdNode);
+ String sample = extractAttribute(xdNode, "sample");
+ String experiment = extractAttribute(xdNode, "experimente");
+ String type = extractType(xdNode);
+ String dsKind = extractAttribute(xdNode, "dsKind");
NewExternalData ds = new NewExternalData();
if (dsKind.equals(DataSetKind.CONTAINER.toString()))
{
@@ -297,12 +315,22 @@ public class ResourceListParser
ds.setDataSetProperties(parseDataSetProperties(xpath, xdNode));
}
+ private String extractAttribute(Node xdNode, String attrName)
+ {
+ return xdNode.getAttributes().getNamedItem(attrName).getTextContent();
+ }
+
+ private String extractCode(Node xdNode)
+ {
+ return extractAttribute(xdNode, "code");
+ }
+
private SampleIdentifier getSampleIdentifier(String sampleIdentifierStr)
{
SampleIdentifier sampleIdentifier = SampleIdentifierFactory.parse(sampleIdentifierStr);
SpaceIdentifier spaceLevel = sampleIdentifier.getSpaceLevel();
String originalSpaceCode = spaceLevel.getSpaceCode();
- return new SampleIdentifier(new SpaceIdentifier(spaceMappings.get(originalSpaceCode)), sampleIdentifier.getSampleCode());
+ return new SampleIdentifier(new SpaceIdentifier(nameTranslator.translate(originalSpaceCode)), sampleIdentifier.getSampleCode());
}
private ExperimentIdentifier getExperimentIdentifier(String experiment)
@@ -311,17 +339,17 @@ public class ResourceListParser
String originalSpaceCode = experimentIdentifier.getSpaceCode();
String projectCode = experimentIdentifier.getProjectCode();
String expCode = experimentIdentifier.getExperimentCode();
- return new ExperimentIdentifier(new ProjectIdentifier(spaceMappings.get(originalSpaceCode), projectCode), expCode);
+ return new ExperimentIdentifier(new ProjectIdentifier(nameTranslator.translate(originalSpaceCode), projectCode), expCode);
}
private void parseProjectMetaData(XPath xpath, String permId, Node xdNode, Date lastModificationDate)
{
- String code = xdNode.getAttributes().getNamedItem("code").getTextContent();
+ String code = extractCode(xdNode);
String desc = xdNode.getAttributes().getNamedItem("desc").getTextContent();
- String space = xdNode.getAttributes().getNamedItem("space").getTextContent();
+ String space = extractSpace(xdNode);
// TODO is there a better way to create project identifier below?
- NewProject newProject = new NewProject("/" + spaceMappings.get(space) + "/" + code, desc);
+ NewProject newProject = new NewProject("/" + nameTranslator.translate(space) + "/" + code, desc);
newProject.setPermID(permId);
ProjectWithConnections newPrjWithConns =
data.new ProjectWithConnections(newProject, lastModificationDate);
@@ -331,8 +359,8 @@ public class ResourceListParser
private void parseMaterialMetaData(XPath xpath, String permId, Node xdNode, Date lastModificationDate)
{
- String code = xdNode.getAttributes().getNamedItem("code").getTextContent();
- String type = xdNode.getAttributes().getNamedItem("type").getTextContent();
+ String code = extractCode(xdNode);
+ String type = extractType(xdNode);
NewMaterialWithType newMaterial = new NewMaterialWithType(code, type);
MaterialWithLastModificationDate materialWithLastModDate =
data.new MaterialWithLastModificationDate(newMaterial, lastModificationDate);
@@ -416,11 +444,11 @@ public class ResourceListParser
private void parseExperimentMetaData(XPath xpath, String permId, Node xdNode, Date lastModificationDate)
{
- String code = xdNode.getAttributes().getNamedItem("code").getTextContent();
- String type = xdNode.getAttributes().getNamedItem("type").getTextContent();
- String project = xdNode.getAttributes().getNamedItem("project").getTextContent();
- String space = xdNode.getAttributes().getNamedItem("space").getTextContent();
- NewExperiment newExp = new NewExperiment("/" + spaceMappings.get(space) + "/" + project + "/" + code, type);
+ String code = extractCode(xdNode);
+ String type = extractType(xdNode);
+ String project = extractAttribute(xdNode, "project");
+ String space = extractSpace(xdNode);
+ NewExperiment newExp = new NewExperiment("/" + nameTranslator.translate(space) + "/" + project + "/" + code, type);
newExp.setPermID(permId);
ExperimentWithConnections newExpWithConns = data.new ExperimentWithConnections(newExp, lastModificationDate);
data.getExperimentsToProcess().put(permId, newExpWithConns);
@@ -430,14 +458,14 @@ public class ResourceListParser
private void parseSampleMetaData(XPath xpath, String permId, Node xdNode, Date lastModificationDate)
{
- String code = xdNode.getAttributes().getNamedItem("code").getTextContent();
- String type = xdNode.getAttributes().getNamedItem("type").getTextContent();
- String experiment = xdNode.getAttributes().getNamedItem("experiment").getTextContent();
- String space = xdNode.getAttributes().getNamedItem("space").getTextContent();
+ String code = extractCode(xdNode);
+ String type = extractType(xdNode);
+ String experiment = extractAttribute(xdNode, "experiment");
+ String space = extractSpace(xdNode);
SampleType sampleType = new SampleType();
sampleType.setCode(type);
- NewSample newSample = new NewSample("/" + spaceMappings.get(space) + "/" + code, sampleType, null, null,
+ NewSample newSample = new NewSample("/" + nameTranslator.translate(space) + "/" + code, sampleType, null, null,
experiment.trim().equals("") ? null : experiment, null, null, new IEntityProperty[0],
new ArrayList<NewAttachment>());
newSample.setPermID(permId);
@@ -447,6 +475,18 @@ public class ResourceListParser
newSample.setProperties(parseProperties(xpath, xdNode));
}
+ private String extractType(Node xdNode)
+ {
+ return extractAttribute(xdNode, "type");
+ }
+
+ private String extractSpace(Node xdNode)
+ {
+ String space = extractAttribute(xdNode, "space");
+ data.getHarvesterSpaceList().add(nameTranslator.translate(space));
+ return space;
+ }
+
private String extractPermIdFromURI(String uri) throws XPathExpressionException
{
Pattern pattern = Pattern.compile("([0-9\\-]{17,})");
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/ResourceListParserData.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/ResourceListParserData.java
index 59f5e3f8905..ff3c764d5ab 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/ResourceListParserData.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/ResourceListParserData.java
@@ -19,8 +19,10 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSetKind;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.NewExperiment;
@@ -36,6 +38,8 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.NewLinkDataSet;
*/
public class ResourceListParserData
{
+ private Set<String> harvesterSpaceList = new HashSet<>();
+
private Map<String, ProjectWithConnections> projectsToProcess = new HashMap<String, ResourceListParserData.ProjectWithConnections>();
private Map<String, ExperimentWithConnections> experimentsToProcess = new HashMap<String, ResourceListParserData.ExperimentWithConnections>();
@@ -46,6 +50,11 @@ public class ResourceListParserData
private Map<String, MaterialWithLastModificationDate> materialsToProcess = new HashMap<String, MaterialWithLastModificationDate>();
+ public Set<String> getHarvesterSpaceList()
+ {
+ return harvesterSpaceList;
+ }
+
public Map<String, ProjectWithConnections> getProjectsToProcess()
{
return projectsToProcess;
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/translator/CustomNameTranslator.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/translator/CustomNameTranslator.java
new file mode 100644
index 00000000000..3377f6c5ba1
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/translator/CustomNameTranslator.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2016 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.harvester.synchronizer.translator;
+
+import java.util.HashMap;
+
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+
+/**
+ *
+ *
+ * @author Ganime Betul Akin
+ */
+public class CustomNameTranslator implements INameTranslator
+{
+
+ private final HashMap<String, String> spaceMappings;
+
+ public CustomNameTranslator(HashMap<String, String> spaceMappings)
+ {
+ this.spaceMappings = spaceMappings;
+ }
+
+ @Override
+ public String translate(String name)
+ {
+ if (spaceMappings == null)
+ {
+ throw new ConfigurationFailureException("Space mappings cannot be null");
+ }
+ String newName = spaceMappings.get(name);
+ if (newName == null)
+ {
+ throw new ConfigurationFailureException("No corresponding mapping found for '" + name + "'");
+ }
+ return newName;
+ }
+
+}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/translator/DefaultNameTranslator.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/translator/DefaultNameTranslator.java
new file mode 100644
index 00000000000..1203ab2a2c2
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/translator/DefaultNameTranslator.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2016 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.harvester.synchronizer.translator;
+
+/**
+ *
+ *
+ * @author Ganime Betul Akin
+ */
+public class DefaultNameTranslator implements INameTranslator
+{
+
+ @Override
+ public String translate(String name)
+ {
+ return name;
+ }
+}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/translator/INameTranslator.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/translator/INameTranslator.java
new file mode 100644
index 00000000000..ff31068bba6
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/translator/INameTranslator.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2016 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.harvester.synchronizer.translator;
+/**
+ *
+ *
+ * @author Ganime Betul Akin
+ */
+public interface INameTranslator
+{
+ String translate(String name);
+
+}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/translator/PrefixBasedNameTranslator.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/translator/PrefixBasedNameTranslator.java
new file mode 100644
index 00000000000..c46c0f34e64
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/translator/PrefixBasedNameTranslator.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2016 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.harvester.synchronizer.translator;
+
+/**
+ *
+ *
+ * @author Ganime Betul Akin
+ */
+public class PrefixBasedNameTranslator implements INameTranslator
+{
+ private final String prefix;
+
+ public PrefixBasedNameTranslator(String prefix)
+ {
+ this.prefix = prefix;
+ }
+
+ @Override
+ public String translate(String name)
+ {
+ return prefix + "-" + name;
+ }
+
+}
diff --git a/openbis/source/java/ch/ethz/sis/openbis/generic/server/EntityRetriever.java b/openbis/source/java/ch/ethz/sis/openbis/generic/server/EntityRetriever.java
index 9828c75db87..78253169960 100644
--- a/openbis/source/java/ch/ethz/sis/openbis/generic/server/EntityRetriever.java
+++ b/openbis/source/java/ch/ethz/sis/openbis/generic/server/EntityRetriever.java
@@ -78,6 +78,7 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.Space;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.fetchoptions.SpaceFetchOptions;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.id.ISpaceId;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.id.SpacePermId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.search.SpaceSearchCriteria;
import ch.ethz.sis.openbis.generic.shared.entitygraph.Edge;
import ch.ethz.sis.openbis.generic.shared.entitygraph.EntityGraph;
import ch.ethz.sis.openbis.generic.shared.entitygraph.Node;
@@ -164,8 +165,7 @@ public class EntityRetriever
public static EntityRetriever createWithSessionToken(IApplicationServerApi v3Api, String sessionToken)
{
- EntityRetriever er = new EntityRetriever(v3Api, sessionToken);
- return er;
+ return new EntityRetriever(v3Api, sessionToken);
}
public EntityGraph<Node<?>> getEntityGraph(String spaceId)
@@ -186,6 +186,14 @@ public class EntityRetriever
return map.get(spacePermId) != null;
}
+ /**
+ * Returns spaces the logged in user is allowed to see
+ */
+ public List<Space> getSpaces()
+ {
+ return v3Api.searchSpaces(sessionToken, new SpaceSearchCriteria(), new SpaceFetchOptions()).getObjects();
+ }
+
public void buildEntityGraph(String spaceId)
{
// TODO add experiment datasets
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/dto/builders/AtomicEntityOperationDetailsBuilder.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/dto/builders/AtomicEntityOperationDetailsBuilder.java
index 2c4da7d92f7..ee6eb8c083b 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/dto/builders/AtomicEntityOperationDetailsBuilder.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/dto/builders/AtomicEntityOperationDetailsBuilder.java
@@ -185,4 +185,11 @@ public class AtomicEntityOperationDetailsBuilder
list.add(newMaterial.getMaterial());
return this;
}
+
+ public AtomicEntityOperationDetailsBuilder space(NewSpace newSpace)
+ {
+ spaceRegistrations.add(newSpace);
+ return this;
+ }
+
}
--
GitLab