From 7b63f644b0f4e098dd57ca159d6faa62f992e7a1 Mon Sep 17 00:00:00 2001
From: felmer <franz-josef.elmer@id.ethz.ch>
Date: Tue, 5 Feb 2019 13:54:40 +0100
Subject: [PATCH] SSDM-7857: Starting a complete rewrite of data source servlet
---
.../datasource/AbstractEntityDeliverer.java | 282 ++++++++++++++++++
.../sync/datasource/ConnectionsBuilder.java | 90 ++++++
.../sync/datasource/DataSetDeliverer.java | 179 +++++++++++
.../datasource/DataSourceRequestHandler.java | 200 +++++++++++++
.../sync/datasource/DataSourceUtils.java | 38 +++
.../sync/datasource/DeliveryContext.java | 88 ++++++
.../sync/datasource/ExperimentDeliverer.java | 104 +++++++
.../sync/datasource/MasterDataDeliverer.java | 46 +++
.../sync/datasource/MaterialDeliverer.java | 100 +++++++
.../sync/datasource/ProjectDeliverer.java | 99 ++++++
.../sync/datasource/SampleDeliverer.java | 109 +++++++
11 files changed, 1335 insertions(+)
create mode 100644 datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/AbstractEntityDeliverer.java
create mode 100644 datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/ConnectionsBuilder.java
create mode 100644 datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DataSetDeliverer.java
create mode 100644 datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DataSourceRequestHandler.java
create mode 100644 datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DataSourceUtils.java
create mode 100644 datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DeliveryContext.java
create mode 100644 datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/ExperimentDeliverer.java
create mode 100644 datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/MasterDataDeliverer.java
create mode 100644 datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/MaterialDeliverer.java
create mode 100644 datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/ProjectDeliverer.java
create mode 100644 datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/SampleDeliverer.java
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/AbstractEntityDeliverer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/AbstractEntityDeliverer.java
new file mode 100644
index 00000000000..6acd9428bfd
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/AbstractEntityDeliverer.java
@@ -0,0 +1,282 @@
+/*
+ * Copyright 2019 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.datasource;
+
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.function.Function;
+
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamWriter;
+
+import org.apache.log4j.Logger;
+
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.attachment.Attachment;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.interfaces.ICodeHolder;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.interfaces.IModificationDateHolder;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.interfaces.IModifierHolder;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.interfaces.IRegistrationDateHolder;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.interfaces.IRegistratorHolder;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.project.Project;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.Space;
+import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.openbis.generic.server.batch.BatchOperationExecutor;
+import ch.systemsx.cisd.openbis.generic.server.batch.IBatchOperation;
+
+/**
+ * @author Franz-Josef Elmer
+ */
+abstract class AbstractEntityDeliverer<T>
+{
+ private static final int CHUNK_SIZE = 1000;
+
+ private static interface IConsumer<T>
+ {
+ public void consume(T t) throws XMLStreamException;
+ }
+
+ protected final Logger operationLog;
+
+ protected final DeliveryContext context;
+
+ private final String entityKind;
+
+ AbstractEntityDeliverer(DeliveryContext context, String entityKind)
+ {
+ this.context = context;
+ this.entityKind = entityKind;
+ operationLog = LogFactory.getLogger(LogCategory.OPERATION, getClass());
+ }
+
+ void deliverEntities(XMLStreamWriter writer, String sessionToken, Set<String> spaces, Date requestTimestamp) throws XMLStreamException
+ {
+ List<T> allEntities = getAllEntities(sessionToken);
+ executeInBatches(allEntities, entities -> deliverEntities(writer, sessionToken, spaces, entities));
+ }
+
+ protected List<T> getAllEntities(String sessionToken)
+ {
+ return Collections.emptyList();
+ }
+
+ protected void deliverEntities(XMLStreamWriter writer, String sessionToken, Set<String> spaces, List<T> entities)
+ throws XMLStreamException
+ {
+
+ }
+
+ protected void addProperties(XMLStreamWriter writer, Map<String, String> properties) throws XMLStreamException
+ {
+ if (properties.isEmpty() == false)
+ {
+ writer.writeStartElement("x:properties");
+ Set<Entry<String, String>> entrySet = properties.entrySet();
+ for (Entry<String, String> entry : entrySet)
+ {
+ writer.writeStartElement("x:property");
+ writer.writeStartElement("x:code");
+ writer.writeCharacters(entry.getKey());
+ writer.writeEndElement();
+ writer.writeStartElement("x:value");
+ writer.writeCharacters(entry.getValue());
+ writer.writeEndElement();
+ writer.writeEndElement();
+ }
+ writer.writeEndElement();
+ }
+ }
+
+ protected void addAttachments(XMLStreamWriter writer, List<Attachment> attachments) throws XMLStreamException
+ {
+ if (attachments.isEmpty())
+ {
+ return;
+ }
+ startBinaryDataElement(writer);
+ for (Attachment attachment : attachments)
+ {
+ writer.writeStartElement("x:attachment");
+ addAttribute(writer, "description", attachment.getDescription());
+ addAttribute(writer, "fileName", attachment.getFileName());
+ addAttribute(writer, "latestVersion", attachment.getVersion(), v -> Integer.toString(v));
+ addAttribute(writer, "permLink", attachment.getPermlink());
+ addAttribute(writer, "title", attachment.getTitle());
+ writer.writeEndElement();
+ }
+ writer.writeEndElement();
+
+ }
+
+ protected void addSpace(XMLStreamWriter writer, Space space) throws XMLStreamException
+ {
+ addAttribute(writer, "space", space, s -> s.getCode());
+ }
+
+ protected void addProject(XMLStreamWriter writer, Project project) throws XMLStreamException
+ {
+ addAttribute(writer, "project", project, p -> p.getCode());
+ }
+
+ protected void addSample(XMLStreamWriter writer, Sample sample) throws XMLStreamException
+ {
+ addAttribute(writer, "sample", sample, s -> s.getIdentifier().getIdentifier());
+ }
+
+ protected void addExperiment(XMLStreamWriter writer, Experiment experiment) throws XMLStreamException
+ {
+ addAttribute(writer, "experiment", experiment, e -> e.getIdentifier().getIdentifier());
+ }
+
+ protected void addKind(XMLStreamWriter writer, Object kind) throws XMLStreamException
+ {
+ addAttribute(writer, "kind", kind, k -> k.toString());
+ }
+
+ protected void addType(XMLStreamWriter writer, ICodeHolder type) throws XMLStreamException
+ {
+ addAttribute(writer, "type", type, t -> t.getCode());
+ }
+
+ protected void addModifier(XMLStreamWriter writer, IModifierHolder dataSet) throws XMLStreamException
+ {
+ addAttribute(writer, "modifier", dataSet.getModifier(), m -> m.getUserId());
+ }
+
+ protected void addRegistrator(XMLStreamWriter writer, IRegistratorHolder dataSet) throws XMLStreamException
+ {
+ addAttribute(writer, "registrator", dataSet.getRegistrator(), r -> r.getUserId());
+ }
+
+ protected void addRegistrationDate(XMLStreamWriter writer, IRegistrationDateHolder dateHolder) throws XMLStreamException
+ {
+ addAttribute(writer, "registration-timestamp", dateHolder.getRegistrationDate(), h -> DataSourceUtils.convertToW3CDate(h));
+ }
+
+ protected void addAttribute(XMLStreamWriter writer, String attributeName, String value) throws XMLStreamException
+ {
+ addAttribute(writer, attributeName, value, v -> v);
+ }
+
+ protected <O> void addAttribute(XMLStreamWriter writer, String attributeName, O object, Function<O, String> mapper) throws XMLStreamException
+ {
+ if (object != null)
+ {
+ writer.writeAttribute(attributeName, mapper.apply(object));
+ }
+ }
+
+ protected void addLink(XMLStreamWriter writer, String code, String entityKind) throws XMLStreamException
+ {
+ addLink(writer, "?viewMode=SIMPLE&anonymous=true#entity=" + entityKind + "&permId=" + code);
+ }
+
+ protected void addLink(XMLStreamWriter writer, String urlPart2) throws XMLStreamException
+ {
+ writer.writeStartElement("rs:ln");
+ writer.writeAttribute("href", context.getServerUrl() + urlPart2);
+ writer.writeAttribute("rel", "describes");
+ writer.writeEndElement();
+ }
+
+ protected void addLocation(XMLStreamWriter writer, String code, String entityKind) throws XMLStreamException
+ {
+ writer.writeStartElement("loc");
+ writer.writeCharacters(context.getServerUrl() + "/" + entityKind + "/" + code + "/M");
+ writer.writeEndElement();
+ }
+
+ protected void addLastModificationDate(XMLStreamWriter writer, IModificationDateHolder dateHolder) throws XMLStreamException
+ {
+ addLastModificationDate(writer, dateHolder.getModificationDate());
+ }
+
+ protected void addLastModificationDate(XMLStreamWriter writer, Date modificationDate) throws XMLStreamException
+ {
+ writer.writeStartElement("lastmod");
+ writer.writeCharacters(DataSourceUtils.convertToW3CDate(modificationDate));
+ writer.writeEndElement();
+ }
+
+ protected void startUrlElement(XMLStreamWriter writer, String entityKind, String permId, Date modificationDate) throws XMLStreamException
+ {
+ startUrlElement(writer);
+ addLocation(writer, permId, entityKind);
+ addLastModificationDate(writer, modificationDate);
+ addLink(writer, permId, entityKind);
+ }
+
+ protected void startUrlElement(XMLStreamWriter writer) throws XMLStreamException
+ {
+ writer.writeStartElement("url");
+ }
+
+ protected void startXdElement(XMLStreamWriter writer) throws XMLStreamException
+ {
+ writer.writeStartElement("x:xd");
+ }
+
+ protected void startBinaryDataElement(XMLStreamWriter writer) throws XMLStreamException
+ {
+ writer.writeStartElement("x:binaryData");
+ }
+
+ private void executeInBatches(List<T> allEntities, IConsumer<List<T>> action)
+ {
+ operationLog.info(allEntities.size() + " " + entityKind + "s in total.");
+ BatchOperationExecutor.executeInBatches(new IBatchOperation<T>()
+ {
+ @Override
+ public void execute(List<T> entities)
+ {
+ try
+ {
+ action.consume(entities);
+ } catch (XMLStreamException e)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(e);
+ }
+ }
+
+ @Override
+ public List<T> getAllEntities()
+ {
+ return allEntities;
+ }
+
+ @Override
+ public String getEntityName()
+ {
+ return entityKind;
+ }
+
+ @Override
+ public String getOperationName()
+ {
+ return "deliver";
+ }
+ }, CHUNK_SIZE);
+ }
+
+}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/ConnectionsBuilder.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/ConnectionsBuilder.java
new file mode 100644
index 00000000000..7c29c736d39
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/ConnectionsBuilder.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2019 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.datasource;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamWriter;
+
+import org.apache.commons.lang3.StringUtils;
+
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.interfaces.IPermIdHolder;
+
+class ConnectionsBuilder
+{
+ private enum ConnectionType
+ {
+ CONNECTION, CHILD, COMPONENT
+ }
+
+ private static final class Connection
+ {
+ private String permId;
+
+ private ConnectionsBuilder.ConnectionType type;
+
+ Connection(String permId, ConnectionsBuilder.ConnectionType type)
+ {
+ this.permId = permId;
+ this.type = type;
+ }
+ }
+
+ private List<ConnectionsBuilder.Connection> connections = new ArrayList<>();
+
+ void addChildren(List<? extends IPermIdHolder> entities)
+ {
+ addConnections(entities, ConnectionType.CHILD);
+ }
+
+ void addComponents(List<? extends IPermIdHolder> entities)
+ {
+ addConnections(entities, ConnectionType.COMPONENT);
+ }
+
+ void addConnections(List<? extends IPermIdHolder> entities)
+ {
+ addConnections(entities, ConnectionType.CONNECTION);
+ }
+
+ void writeTo(XMLStreamWriter writer) throws XMLStreamException
+ {
+ if (connections.isEmpty())
+ {
+ return;
+ }
+ writer.writeStartElement("x:connections");
+ for (Connection connection : connections)
+ {
+ writer.writeStartElement("x:connection");
+ writer.writeAttribute("to", connection.permId);
+ writer.writeAttribute("type", StringUtils.capitalize(connection.type.toString().toLowerCase()));
+ writer.writeEndElement();
+ }
+ writer.writeEndElement();
+ }
+
+ private void addConnections(List<? extends IPermIdHolder> entities, ConnectionsBuilder.ConnectionType child)
+ {
+ for (IPermIdHolder entity : entities)
+ {
+ connections.add(new Connection(entity.getPermId().toString(), child));
+ }
+ }
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DataSetDeliverer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DataSetDeliverer.java
new file mode 100644
index 00000000000..53892dee278
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DataSetDeliverer.java
@@ -0,0 +1,179 @@
+/*
+ * Copyright 2019 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.datasource;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamWriter;
+
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.ContentCopy;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.LinkedData;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.PhysicalData;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.fetchoptions.DataSetFetchOptions;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.search.DataSetSearchCriteria;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.entitytype.EntityKind;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment;
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContentNode;
+
+/**
+ * @author Franz-Josef Elmer
+ */
+public class DataSetDeliverer extends AbstractEntityDeliverer<DataSet>
+{
+
+ DataSetDeliverer(DeliveryContext context)
+ {
+ super(context, "data set");
+ }
+
+ @Override
+ protected List<DataSet> getAllEntities(String sessionToken)
+ {
+ DataSetSearchCriteria searchCriteria = new DataSetSearchCriteria();
+ DataSetFetchOptions fetchOptions = new DataSetFetchOptions();
+ fetchOptions.sortBy().code().asc();
+ return context.getV3api().searchDataSets(sessionToken, searchCriteria, fetchOptions).getObjects();
+
+ }
+
+ @Override
+ protected void deliverEntities(XMLStreamWriter writer, String sessionToken, Set<String> spaces, List<DataSet> dataSets) throws XMLStreamException
+ {
+ List<DataSetPermId> permIds = dataSets.stream().map(DataSet::getPermId).collect(Collectors.toList());
+ Collection<DataSet> fullDataSets = context.getV3api().getDataSets(sessionToken, permIds, createDataSetFetchOptions()).values();
+ int count = 0;
+ for (DataSet dataSet : fullDataSets)
+ {
+ if (accept(dataSet, spaces))
+ {
+ String code = dataSet.getCode();
+ startUrlElement(writer, "DATA_SET", code, dataSet.getModificationDate());
+ startXdElement(writer);
+ writer.writeAttribute("code", code);
+ writer.writeAttribute("dsKind", dataSet.getKind().toString());
+ addExperiment(writer, dataSet.getExperiment());
+ addKind(writer, EntityKind.DATA_SET);
+ addModifier(writer, dataSet);
+ addRegistrationDate(writer, dataSet);
+ addRegistrator(writer, dataSet);
+ addSample(writer, dataSet.getSample());
+ addType(writer, dataSet.getType());
+ addProperties(writer, dataSet.getProperties());
+ addPhysicalData(writer, dataSet, code);
+ addLinkedData(writer, dataSet, code);
+ ConnectionsBuilder connectionsBuilder = new ConnectionsBuilder();
+ connectionsBuilder.addChildren(dataSet.getChildren());
+ connectionsBuilder.addComponents(dataSet.getComponents());
+ connectionsBuilder.writeTo(writer);
+ writer.writeEndElement();
+ writer.writeEndElement();
+ count++;
+ }
+ }
+ operationLog.info(count + " of " + dataSets.size() + " data sets have been delivered.");
+ }
+
+ private void addPhysicalData(XMLStreamWriter writer, DataSet dataSet, String code) throws XMLStreamException
+ {
+ PhysicalData physicalData = dataSet.getPhysicalData();
+ if (physicalData != null)
+ {
+ startBinaryDataElement(writer);
+ addFileNodes(writer, code, context.getContentProvider().asContent(code).getRootNode());
+ writer.writeEndElement();
+ }
+ }
+
+ private void addLinkedData(XMLStreamWriter writer, DataSet dataSet, String code) throws XMLStreamException
+ {
+ LinkedData linkedData = dataSet.getLinkedData();
+ if (linkedData != null)
+ {
+ startBinaryDataElement(writer);
+ List<ContentCopy> contentCopies = linkedData.getContentCopies();
+ for (ContentCopy contentCopy : contentCopies)
+ {
+ addAttribute(writer, "externalCode", contentCopy.getExternalCode());
+ addAttribute(writer, "externalDMS", contentCopy.getExternalDms(), edms -> edms.getCode());
+ addAttribute(writer, "gitCommitHash", contentCopy.getGitCommitHash());
+ addAttribute(writer, "gitRepositoryId", contentCopy.getGitRepositoryId());
+ addAttribute(writer, "id", contentCopy.getId(), id -> id.getPermId());
+ addAttribute(writer, "path", contentCopy.getPath());
+ }
+ addFileNodes(writer, code, context.getContentProvider().asContent(code).getRootNode());
+ writer.writeEndElement();
+ }
+ }
+
+ private void addFileNodes(XMLStreamWriter writer, String dataSetCode, IHierarchicalContentNode node) throws XMLStreamException
+ {
+ if (node.isDirectory())
+ {
+ List<IHierarchicalContentNode> childNodes = node.getChildNodes();
+ for (IHierarchicalContentNode childNode : childNodes)
+ {
+ addFileNodes(writer, dataSetCode, childNode);
+ }
+ } else
+ {
+ writer.writeStartElement("x:fileNode");
+ addAttribute(writer, "checksum", node.getChecksum());
+ if (node.isChecksumCRC32Precalculated())
+ {
+ writer.writeAttribute("crc32checksum", Integer.toString(node.getChecksumCRC32()));
+ }
+ writer.writeAttribute("length", Long.toString(node.getFileLength()));
+ writer.writeAttribute("path", context.getDownloadUrl() + "/datastore_server/" + dataSetCode + "/"
+ + node.getRelativePath() + "?");
+ writer.writeEndElement();
+ }
+ }
+
+ private boolean accept(DataSet dataSet, Set<String> spaces)
+ {
+ Experiment experiment = dataSet.getExperiment();
+ if (experiment != null)
+ {
+ return spaces.contains(experiment.getProject().getSpace().getCode());
+ }
+ return spaces.contains(dataSet.getSample().getSpace().getCode());
+ }
+
+ private DataSetFetchOptions createDataSetFetchOptions()
+ {
+ DataSetFetchOptions fo = new DataSetFetchOptions();
+ fo.withRegistrator();
+ fo.withModifier();
+ fo.withType();
+ fo.withSample().withSpace();
+ fo.withExperiment().withProject().withSpace();
+ fo.withProperties();
+ fo.withChildren();
+ fo.withComponents();
+ fo.withPhysicalData();
+ fo.withLinkedData().withExternalDms();
+ fo.sortBy().code();
+ return fo;
+ }
+
+}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DataSourceRequestHandler.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DataSourceRequestHandler.java
new file mode 100644
index 00000000000..9ffb2d846ac
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DataSourceRequestHandler.java
@@ -0,0 +1,200 @@
+/*
+ * Copyright 2019 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.datasource;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.TreeSet;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.xml.stream.XMLOutputFactory;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamWriter;
+
+import org.apache.log4j.Logger;
+
+import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.Space;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.fetchoptions.SpaceFetchOptions;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.search.SpaceSearchCriteria;
+import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.properties.PropertyUtils;
+import ch.systemsx.cisd.openbis.dss.generic.server.oaipmh.IRequestHandler;
+import ch.systemsx.cisd.openbis.dss.generic.shared.DataSourceQueryService;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
+import ch.systemsx.cisd.openbis.generic.shared.dto.SessionContextDTO;
+
+/**
+ * @author Franz-Josef Elmer
+ */
+public class DataSourceRequestHandler implements IRequestHandler
+{
+ private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION, DataSourceRequestHandler.class);
+
+ private File tempDir;
+
+ private String serverUrl;
+
+ private String downloadUrl;
+
+ private String servletPath;
+
+ private DataSourceQueryService queryService;
+
+ private IApplicationServerApi v3api;
+
+ private IHierarchicalContentProvider contentProvider;
+
+ private AbstractEntityDeliverer<DataSet> dataSetDeliverer;
+
+ private ExperimentDeliverer experimentDeliverer;
+
+ private MaterialDeliverer materialDeliverer;
+
+ private SampleDeliverer sampleDeliverer;
+
+ private ProjectDeliverer projectDeliverer;
+
+ private MasterDataDeliverer masterDataDeliverer;
+
+ @Override
+ public void init(Properties properties)
+ {
+ DeliveryContext deliveryContext = new DeliveryContext();
+ servletPath = new File(PropertyUtils.getMandatoryProperty(properties, "path")).getParent();
+ deliveryContext.setServletPath(servletPath);
+ tempDir = new File(PropertyUtils.getMandatoryProperty(properties, "temp-dir"));
+ serverUrl = PropertyUtils.getMandatoryProperty(properties, "server-url");
+ deliveryContext.setServerUrl(serverUrl);
+ downloadUrl = PropertyUtils.getMandatoryProperty(properties, "download-url");
+ deliveryContext.setDownloadUrl(downloadUrl);
+ queryService = new DataSourceQueryService();
+ v3api = ServiceProvider.getV3ApplicationService();
+ deliveryContext.setV3api(v3api);
+ contentProvider = ServiceProvider.getHierarchicalContentProvider();
+ deliveryContext.setContentProvider(contentProvider);
+ dataSetDeliverer = new DataSetDeliverer(deliveryContext);
+ experimentDeliverer = new ExperimentDeliverer(deliveryContext);
+ materialDeliverer = new MaterialDeliverer(deliveryContext);
+ sampleDeliverer = new SampleDeliverer(deliveryContext);
+ projectDeliverer = new ProjectDeliverer(deliveryContext);
+ masterDataDeliverer = new MasterDataDeliverer(deliveryContext);
+ }
+
+ @Override
+ public void handle(SessionContextDTO session, HttpServletRequest request, HttpServletResponse response)
+ {
+ try
+ {
+ Map<String, Set<String>> parameterMap = getParameterMap(request);
+ XMLOutputFactory xmlOutputFactory = XMLOutputFactory.newInstance();
+ XMLStreamWriter writer = xmlOutputFactory.createXMLStreamWriter(response.getWriter());
+ writer.writeStartDocument();
+ writer.writeStartElement("urlset");
+ writer.writeAttribute("xmlns", "http://www.sitemaps.org/schemas/sitemap/0.9");
+ writer.writeAttribute("xmlns:rs", "http://www.openarchives.org/rs/terms/");
+ writer.writeAttribute("xmlns:xmd", "https://sis.id.ethz.ch/software/#openbis/xmdterms/");
+ writer.writeAttribute("xmlns:x", "https://sis.id.ethz.ch/software/#openbis/xdterms/");
+ writer.writeAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance");
+ writer.writeAttribute("xsi:schemaLocation",
+ "https://sis.id.ethz.ch/software/#openbis/xdterms/ ./xml/xdterms.xsd https://sis.id.ethz.ch/software/#openbis/xmdterms/");
+ String sessionToken = session.getSessionToken();
+ Set<String> verbs = parameterMap.get("verb");
+ if (verbs.contains("resourcelist.xml"))
+ {
+ deliverResourceList(parameterMap, sessionToken, writer);
+ }
+ writer.writeEndElement();
+ writer.writeEndDocument();
+ } catch (Exception e)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(e);
+ }
+ }
+
+ private void deliverResourceList(Map<String, Set<String>> parameterMap, String sessionToken, XMLStreamWriter writer) throws XMLStreamException
+ {
+
+ writer.writeStartElement("rs:ln");
+ writer.writeAttribute("href", downloadUrl + servletPath + "/?verb=capabilitylist.xml");
+ writer.writeAttribute("rel", "up");
+ writer.writeEndElement();
+ writer.writeStartElement("rs:md");
+ Date requestTimestamp = getRequestTimestamp();
+ writer.writeAttribute("at", DataSourceUtils.convertToW3CDate(requestTimestamp));
+ writer.writeAttribute("capability", "resourceList");
+ writer.writeEndElement();
+ Set<String> ignoredSpaces = parameterMap.get("black_list");
+ if (ignoredSpaces == null)
+ {
+ ignoredSpaces = Collections.emptySet();
+ }
+ Set<String> requestedSpaces = new TreeSet<>();
+ List<Space> spaces = v3api.searchSpaces(sessionToken, new SpaceSearchCriteria(), new SpaceFetchOptions()).getObjects();
+ for (Space space : spaces)
+ {
+ if (ignoredSpaces.contains(space.getCode()) == false)
+ {
+ requestedSpaces.add(space.getCode());
+ }
+ }
+ dataSetDeliverer.deliverEntities(writer, sessionToken, requestedSpaces, requestTimestamp);
+ experimentDeliverer.deliverEntities(writer, sessionToken, requestedSpaces, requestTimestamp);
+ masterDataDeliverer.deliverEntities(writer, sessionToken, requestedSpaces, requestTimestamp);;
+ materialDeliverer.deliverEntities(writer, sessionToken, requestedSpaces, requestTimestamp);
+ projectDeliverer.deliverEntities(writer, sessionToken, requestedSpaces, requestTimestamp);
+ sampleDeliverer.deliverEntities(writer, sessionToken, requestedSpaces, requestTimestamp);
+ }
+
+ protected Date getRequestTimestamp()
+ {
+ Date requestTimestamp = new Date();
+ String query = "select xact_start FROM pg_stat_activity WHERE xact_start IS NOT NULL ORDER BY xact_start ASC LIMIT 1";
+ for (Map<String, Object> map : queryService.select("openbis-db", query))
+ {
+ requestTimestamp = (Date) map.get("xact_start");
+ }
+ return requestTimestamp;
+ }
+
+ private Map<String, Set<String>> getParameterMap(HttpServletRequest request)
+ {
+ Enumeration<String> enumeration = request.getParameterNames();
+ Map<String, Set<String>> parameterMap = new HashMap<>();
+ while (enumeration.hasMoreElements())
+ {
+ String parameter = enumeration.nextElement();
+ parameterMap.put(parameter, new HashSet<>(Arrays.asList(request.getParameterValues(parameter))));
+ }
+ return parameterMap;
+ }
+
+}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DataSourceUtils.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DataSourceUtils.java
new file mode 100644
index 00000000000..6390ce5dbbb
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DataSourceUtils.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2019 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.datasource;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Locale;
+import java.util.TimeZone;
+
+/**
+ * @author Franz-Josef Elmer
+ *
+ */
+class DataSourceUtils
+{
+
+ static String convertToW3CDate(Date date)
+ {
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US);
+ format.setTimeZone(TimeZone.getTimeZone("GMT"));
+ return format.format(date) + "Z";
+ }
+
+}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DeliveryContext.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DeliveryContext.java
new file mode 100644
index 00000000000..8bbea0b506c
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DeliveryContext.java
@@ -0,0 +1,88 @@
+/*
+ * Copyright 2019 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.datasource;
+
+import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
+
+/**
+ * @author Franz-Josef Elmer
+ *
+ */
+class DeliveryContext
+{
+ private String serverUrl;
+
+ private String downloadUrl;
+
+ private String servletPath;
+
+ private IApplicationServerApi v3api;
+
+ private IHierarchicalContentProvider contentProvider;
+
+ public String getServerUrl()
+ {
+ return serverUrl;
+ }
+
+ public void setServerUrl(String serverUrl)
+ {
+ this.serverUrl = serverUrl;
+ }
+
+ public String getDownloadUrl()
+ {
+ return downloadUrl;
+ }
+
+ public void setDownloadUrl(String downloadUrl)
+ {
+ this.downloadUrl = downloadUrl;
+ }
+
+ public String getServletPath()
+ {
+ return servletPath;
+ }
+
+ public void setServletPath(String servletPath)
+ {
+ this.servletPath = servletPath;
+ }
+
+ public IApplicationServerApi getV3api()
+ {
+ return v3api;
+ }
+
+ public void setV3api(IApplicationServerApi v3api)
+ {
+ this.v3api = v3api;
+ }
+
+ public IHierarchicalContentProvider getContentProvider()
+ {
+ return contentProvider;
+ }
+
+ public void setContentProvider(IHierarchicalContentProvider contentProvider)
+ {
+ this.contentProvider = contentProvider;
+ }
+
+}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/ExperimentDeliverer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/ExperimentDeliverer.java
new file mode 100644
index 00000000000..e66a461c557
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/ExperimentDeliverer.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2019 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.datasource;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamWriter;
+
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.entitytype.EntityKind;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.fetchoptions.ExperimentFetchOptions;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.id.ExperimentPermId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.search.ExperimentSearchCriteria;
+
+/**
+ * @author Franz-Josef Elmer
+ *
+ */
+public class ExperimentDeliverer extends AbstractEntityDeliverer<Experiment>
+{
+
+ ExperimentDeliverer(DeliveryContext context)
+ {
+ super(context, "experiment");
+ }
+
+ @Override
+ protected List<Experiment> getAllEntities(String sessionToken)
+ {
+ ExperimentSearchCriteria searchCriteria = new ExperimentSearchCriteria();
+ ExperimentFetchOptions fetchOptions = new ExperimentFetchOptions();
+ fetchOptions.sortBy().permId();
+ return context.getV3api().searchExperiments(sessionToken, searchCriteria, fetchOptions).getObjects();
+ }
+
+ @Override
+ protected void deliverEntities(XMLStreamWriter writer, String sessionToken, Set<String> spaces, List<Experiment> experiments)
+ throws XMLStreamException
+ {
+ List<ExperimentPermId> permIds = experiments.stream().map(Experiment::getPermId).collect(Collectors.toList());
+ Collection<Experiment> fullExperiments = context.getV3api().getExperiments(sessionToken, permIds, createFullFetchOptions()).values();
+ int count = 0;
+ for (Experiment experiment : fullExperiments)
+ {
+ if (spaces.contains(experiment.getProject().getSpace().getCode()))
+ {
+ String permId = experiment.getPermId().getPermId();
+ startUrlElement(writer, "EXPERIMENT", permId, experiment.getModificationDate());
+ startXdElement(writer);
+ writer.writeAttribute("code", experiment.getCode());
+ addKind(writer, EntityKind.EXPERIMENT);
+ addModifier(writer, experiment);
+ addProject(writer, experiment.getProject());
+ addRegistrationDate(writer, experiment);
+ addRegistrator(writer, experiment);
+ addSpace(writer, experiment.getProject().getSpace());
+ addType(writer, experiment.getType());
+ addProperties(writer, experiment.getProperties());
+ ConnectionsBuilder connectionsBuilder = new ConnectionsBuilder();
+ connectionsBuilder.addConnections(experiment.getSamples());
+ connectionsBuilder.addConnections(experiment.getDataSets());
+ connectionsBuilder.writeTo(writer);
+ addAttachments(writer, experiment.getAttachments());
+ writer.writeEndElement();
+ writer.writeEndElement();
+ count++;
+ }
+ }
+ operationLog.info(count + " of " + experiments.size() + " experiments have been delivered.");
+ }
+
+ private ExperimentFetchOptions createFullFetchOptions()
+ {
+ ExperimentFetchOptions fo = new ExperimentFetchOptions();
+ fo.withRegistrator();
+ fo.withModifier();
+ fo.withProperties();
+ fo.withProject().withSpace();
+ fo.withType();
+ fo.withAttachments();
+ fo.withSamples();
+ fo.withDataSets();
+ return fo;
+ }
+
+}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/MasterDataDeliverer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/MasterDataDeliverer.java
new file mode 100644
index 00000000000..e604e0b057e
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/MasterDataDeliverer.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2019 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.datasource;
+
+import java.util.Date;
+import java.util.Set;
+
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamWriter;
+
+/**
+ * @author Franz-Josef Elmer
+ *
+ */
+public class MasterDataDeliverer extends AbstractEntityDeliverer<Object>
+{
+
+ MasterDataDeliverer(DeliveryContext context)
+ {
+ super(context, "master data");
+ }
+
+ @Override
+ void deliverEntities(XMLStreamWriter writer, String sessionToken, Set<String> spaces, Date requestTimestamp) throws XMLStreamException
+ {
+ startUrlElement(writer);
+ addLocation(writer, "MASTER_DATA", "MASTER_DATA");
+ addLastModificationDate(writer, requestTimestamp);
+ writer.writeEndElement();
+ }
+
+}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/MaterialDeliverer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/MaterialDeliverer.java
new file mode 100644
index 00000000000..7b72d5c38d7
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/MaterialDeliverer.java
@@ -0,0 +1,100 @@
+/*
+ * Copyright 2019 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.datasource;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamWriter;
+
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.material.Material;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.material.fetchoptions.MaterialFetchOptions;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.material.id.MaterialPermId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.material.search.MaterialSearchCriteria;
+
+/**
+ * @author Franz-Josef Elmer
+ */
+public class MaterialDeliverer extends AbstractEntityDeliverer<Material>
+{
+
+ MaterialDeliverer(DeliveryContext context)
+ {
+ super(context, "material");
+ }
+
+ @Override
+ protected List<Material> getAllEntities(String sessionToken)
+ {
+
+ MaterialSearchCriteria searchCriteria = new MaterialSearchCriteria();
+ MaterialFetchOptions fetchOptions = new MaterialFetchOptions();
+ fetchOptions.sortBy().permId();
+ return context.getV3api().searchMaterials(sessionToken, searchCriteria, fetchOptions).getObjects();
+ }
+
+ @Override
+ protected void deliverEntities(XMLStreamWriter writer, String sessionToken, Set<String> spaces, List<Material> materials)
+ throws XMLStreamException
+ {
+ List<MaterialPermId> permIds = materials.stream().map(Material::getPermId).collect(Collectors.toList());
+ Collection<Material> fullMaterials = context.getV3api().getMaterials(sessionToken, permIds, createFullFetchOptions()).values();
+ for (Material material : fullMaterials)
+ {
+ startUrlElement(writer);
+ String type = material.getType().getCode();
+ String code = material.getCode();
+ addLocation(writer, type + "/" + code, "MATERIAL");
+ addLastModificationDate(writer, material.getModificationDate());
+ addLink(writer, "#action=VIEW&entity=MATERIAL&code=" + code + "&type=" + type);
+ startXdElement(writer);
+ writer.writeAttribute("code", code);
+ writer.writeAttribute("dsKind", "MATERIAL");
+ addRegistrationDate(writer, material);
+ addRegistrator(writer, material);
+ addType(writer, material.getType());
+ HashMap<String, String> allProperties = new HashMap<>(material.getProperties());
+ Map<String, Material> materialProperties = material.getMaterialProperties();
+ Set<Entry<String, Material>> entrySet = materialProperties.entrySet();
+ for (Entry<String, Material> entity : entrySet)
+ {
+ allProperties.put(entity.getKey(), entity.getValue().getPermId().toString());
+ }
+ addProperties(writer, allProperties);
+ writer.writeEndElement();
+ writer.writeEndElement();
+ }
+ operationLog.info(materials.size() + " of " + materials.size() + " materials have been delivered.");
+ }
+
+ private MaterialFetchOptions createFullFetchOptions()
+ {
+ MaterialFetchOptions fetchOptions = new MaterialFetchOptions();
+ fetchOptions.withMaterialProperties();
+ fetchOptions.withRegistrator();
+ fetchOptions.withType();
+ fetchOptions.withProperties();
+ return fetchOptions;
+ }
+
+}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/ProjectDeliverer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/ProjectDeliverer.java
new file mode 100644
index 00000000000..a1d65ea994c
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/ProjectDeliverer.java
@@ -0,0 +1,99 @@
+/*
+ * Copyright 2019 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.datasource;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamWriter;
+
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.project.Project;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.project.fetchoptions.ProjectFetchOptions;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.project.id.ProjectPermId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.project.search.ProjectSearchCriteria;
+
+/**
+ * @author Franz-Josef Elmer
+ *
+ */
+public class ProjectDeliverer extends AbstractEntityDeliverer<Project>
+{
+
+ ProjectDeliverer(DeliveryContext context)
+ {
+ super(context, "project");
+ }
+
+ @Override
+ protected List<Project> getAllEntities(String sessionToken)
+ {
+ ProjectSearchCriteria searchCriteria = new ProjectSearchCriteria();
+ ProjectFetchOptions fetchOptions = new ProjectFetchOptions();
+ fetchOptions.sortBy().permId();
+ return context.getV3api().searchProjects(sessionToken, searchCriteria, fetchOptions).getObjects();
+ }
+
+ @Override
+ protected void deliverEntities(XMLStreamWriter writer, String sessionToken, Set<String> spaces, List<Project> projects) throws XMLStreamException
+ {
+ List<ProjectPermId> permIds = projects.stream().map(Project::getPermId).collect(Collectors.toList());
+ Collection<Project> fullProjects = context.getV3api().getProjects(sessionToken, permIds, createFullFetchOptions()).values();
+ int count = 0;
+ for (Project project : fullProjects)
+ {
+ if (spaces.contains(project.getSpace().getCode()))
+ {
+ String permId = project.getPermId().getPermId();
+ startUrlElement(writer, "PROJECT", permId, project.getModificationDate());
+ startXdElement(writer);
+ writer.writeAttribute("code", project.getCode());
+ addAttribute(writer, "desc", project.getDescription());
+ addKind(writer, "PROJECT");
+ addModifier(writer, project);
+ addRegistrationDate(writer, project);
+ addRegistrator(writer, project);
+ addSpace(writer, project.getSpace());
+ ConnectionsBuilder connectionsBuilder = new ConnectionsBuilder();
+ connectionsBuilder.addConnections(project.getExperiments());
+ connectionsBuilder.addConnections(project.getSamples().stream()
+ .filter(s -> s.getExperiment() == null).collect(Collectors.toList()));
+ connectionsBuilder.writeTo(writer);
+ addAttachments(writer, project.getAttachments());
+ writer.writeEndElement();
+ writer.writeEndElement();
+ count++;
+ }
+ }
+ operationLog.info(count + " of " + projects.size() + " projects have been delivered.");
+ }
+
+ private ProjectFetchOptions createFullFetchOptions()
+ {
+ ProjectFetchOptions fetchOptions = new ProjectFetchOptions();
+ fetchOptions.withAttachments();
+ fetchOptions.withRegistrator();
+ fetchOptions.withModifier();
+ fetchOptions.withSpace();
+ fetchOptions.withExperiments();
+ fetchOptions.withSamples().withExperiment();
+ return fetchOptions;
+ }
+
+}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/SampleDeliverer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/SampleDeliverer.java
new file mode 100644
index 00000000000..9ea3706e2b8
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/SampleDeliverer.java
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2019 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.datasource;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamWriter;
+
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.entitytype.EntityKind;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.fetchoptions.SampleFetchOptions;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.id.SamplePermId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.search.SampleSearchCriteria;
+
+/**
+ * @author Franz-Josef Elmer
+ *
+ */
+public class SampleDeliverer extends AbstractEntityDeliverer<Sample>
+{
+
+ SampleDeliverer(DeliveryContext context)
+ {
+ super(context, "sample");
+ }
+
+ @Override
+ protected List<Sample> getAllEntities(String sessionToken)
+ {
+ SampleSearchCriteria searchCriteria = new SampleSearchCriteria();
+ SampleFetchOptions fetchOptions = new SampleFetchOptions();
+ fetchOptions.sortBy().permId();
+ return context.getV3api().searchSamples(sessionToken, searchCriteria, fetchOptions).getObjects();
+ }
+
+ @Override
+ protected void deliverEntities(XMLStreamWriter writer, String sessionToken, Set<String> spaces, List<Sample> samples) throws XMLStreamException
+ {
+ List<SamplePermId> permIds = samples.stream().map(Sample::getPermId).collect(Collectors.toList());
+ Collection<Sample> fullSamples = context.getV3api().getSamples(sessionToken, permIds, createFullFetchOptions()).values();
+ int count = 0;
+ for (Sample sample : fullSamples)
+ {
+ if (sample.getSpace() == null || spaces.contains(sample.getSpace().getCode()))
+ {
+ String permId = sample.getPermId().getPermId();
+ startUrlElement(writer, "SAMPLE", permId, sample.getModificationDate());
+ startXdElement(writer);
+ writer.writeAttribute("code", sample.getCode());
+ addExperiment(writer, sample.getExperiment());
+ addKind(writer, EntityKind.SAMPLE);
+ addModifier(writer, sample);
+ addProject(writer, sample.getProject());
+ addRegistrationDate(writer, sample);
+ addRegistrator(writer, sample);
+ addSpace(writer, sample.getSpace());
+ addType(writer, sample.getType());
+ addProperties(writer, sample.getProperties());
+ ConnectionsBuilder connectionsBuilder = new ConnectionsBuilder();
+ connectionsBuilder.addConnections(sample.getDataSets());
+ connectionsBuilder.addChildren(sample.getChildren());
+ connectionsBuilder.addComponents(sample.getComponents());
+ connectionsBuilder.writeTo(writer);
+ addAttachments(writer, sample.getAttachments());
+ writer.writeEndElement();
+ writer.writeEndElement();
+ count++;
+ }
+ }
+ operationLog.info(count + " of " + samples.size() + " samples have been delivered.");
+ }
+
+ private SampleFetchOptions createFullFetchOptions()
+ {
+ SampleFetchOptions fo = new SampleFetchOptions();
+ fo.withRegistrator();
+ fo.withModifier();
+ fo.withProperties();
+ fo.withDataSets();
+ fo.withType();
+ fo.withExperiment();
+ fo.withProject();
+ fo.withSpace();
+ fo.withAttachments();
+ fo.withChildren();
+ fo.withComponents();
+ fo.withDataSets();
+ return fo;
+ }
+
+}
--
GitLab