From d5c5175eb615aab6f0edeeda8c358958a95589c1 Mon Sep 17 00:00:00 2001
From: felmer <franz-josef.elmer@id.ethz.ch>
Date: Tue, 12 Mar 2019 12:37:54 +0100
Subject: [PATCH] SSDM-7493: data source provides files of file service if
mentioned in some entity properties
---
.../datasource/AbstractEntityDeliverer.java | 16 +++-
.../sync/datasource/DataSetDeliverer.java | 3 +-
.../datasource/DataSourceRequestHandler.java | 12 +++
.../sync/datasource/DeliveryContext.java | 14 ++++
.../datasource/DeliveryExecutionContext.java | 12 +++
.../sync/datasource/ExperimentDeliverer.java | 3 +-
.../sync/datasource/FileDeliverer.java | 76 +++++++++++++++++++
.../sync/datasource/MaterialDeliverer.java | 3 +-
.../sync/datasource/SampleDeliverer.java | 3 +-
9 files changed, 136 insertions(+), 6 deletions(-)
create mode 100644 datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/FileDeliverer.java
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/AbstractEntityDeliverer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/AbstractEntityDeliverer.java
index 75e9f98eb09..dc1a18eb2b8 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/AbstractEntityDeliverer.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/AbstractEntityDeliverer.java
@@ -23,6 +23,8 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.function.Function;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamWriter;
@@ -40,6 +42,7 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.project.Project;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.Space;
+import ch.ethz.sis.openbis.generic.server.FileServiceServlet;
import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
@@ -53,6 +56,9 @@ import ch.systemsx.cisd.openbis.generic.server.batch.IBatchOperation;
abstract class AbstractEntityDeliverer<T> implements IDeliverer
{
private static final int CHUNK_SIZE = 1000;
+
+ private static final Pattern FILE_SERVICE_PATTERN = Pattern.compile("openbis/" + FileServiceServlet.FILE_SERVICE_PATH
+ + "/([^\"']*)");
private static interface IConsumer<T>
{
@@ -95,7 +101,7 @@ abstract class AbstractEntityDeliverer<T> implements IDeliverer
return context.getV3api();
}
- protected void addProperties(XMLStreamWriter writer, Map<String, String> properties) throws XMLStreamException
+ protected void addProperties(XMLStreamWriter writer, Map<String, String> properties, Set<String> fileServicePaths) throws XMLStreamException
{
if (properties.isEmpty() == false)
{
@@ -108,7 +114,13 @@ abstract class AbstractEntityDeliverer<T> implements IDeliverer
writer.writeCharacters(entry.getKey());
writer.writeEndElement();
writer.writeStartElement("x:value");
- writer.writeCharacters(entry.getValue());
+ String value = entry.getValue();
+ Matcher matcher = FILE_SERVICE_PATTERN.matcher(value);
+ while (matcher.find())
+ {
+ fileServicePaths.add(matcher.group(1));
+ }
+ writer.writeCharacters(value);
writer.writeEndElement();
writer.writeEndElement();
}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DataSetDeliverer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DataSetDeliverer.java
index 47c5c04ce20..a350c0dc6c6 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DataSetDeliverer.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DataSetDeliverer.java
@@ -52,6 +52,7 @@ public class DataSetDeliverer extends AbstractEntityWithPermIdDeliverer
XMLStreamWriter writer = context.getWriter();
String sessionToken = context.getSessionToken();
Set<String> spaces = context.getSpaces();
+ Set<String> fileServicePaths = context.getFileServicePaths();
IApplicationServerApi v3api = getV3Api();
List<DataSetPermId> permIds = dataSets.stream().map(DataSetPermId::new).collect(Collectors.toList());
Collection<DataSet> fullDataSets = v3api.getDataSets(sessionToken, permIds, createDataSetFetchOptions()).values();
@@ -77,7 +78,7 @@ public class DataSetDeliverer extends AbstractEntityWithPermIdDeliverer
addRegistrator(writer, dataSet);
addSample(writer, dataSet.getSample());
addType(writer, dataSet.getType());
- addProperties(writer, dataSet.getProperties());
+ addProperties(writer, dataSet.getProperties(), fileServicePaths);
addPhysicalData(writer, dataSet, code);
addLinkedData(writer, dataSet, code);
ConnectionsBuilder connectionsBuilder = new ConnectionsBuilder();
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DataSourceRequestHandler.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DataSourceRequestHandler.java
index f48f43849d6..8356ebb5e93 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DataSourceRequestHandler.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DataSourceRequestHandler.java
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
+import java.util.TreeSet;
import java.util.stream.Collectors;
import javax.servlet.http.HttpServletRequest;
@@ -76,6 +77,7 @@ public class DataSourceRequestHandler implements IRequestHandler
executionContext.setSessionToken(sessionToken);
executionContext.setSpaces(requestedSpaces);
executionContext.setWriter(context.getWriter());
+ executionContext.setFileServicePaths(context.getFileServicePaths());
deliverer.deliverEntities(executionContext);
}
};
@@ -168,6 +170,8 @@ public class DataSourceRequestHandler implements IRequestHandler
deliveryContext.setServletPath(new File(PropertyUtils.getMandatoryProperty(properties, "path")).getParent());
deliveryContext.setServerUrl(PropertyUtils.getMandatoryProperty(properties, "server-url"));
deliveryContext.setDownloadUrl(PropertyUtils.getMandatoryProperty(properties, "download-url"));
+ String fileServiceRepositoryPath = PropertyUtils.getMandatoryProperty(properties, "file-service-repository-path");
+ deliveryContext.setFileServiceRepository(new File(fileServiceRepositoryPath));
deliveryContext.setV3api(ServiceProvider.getV3ApplicationService());
deliveryContext.setContentProvider(ServiceProvider.getHierarchicalContentProvider());
deliveryContext.setOpenBisDataSourceName(properties.getProperty("openbis-data-source-name", "openbis-db"));
@@ -179,6 +183,7 @@ public class DataSourceRequestHandler implements IRequestHandler
deliverers.addDeliverer(new ExperimentDeliverer(deliveryContext));
deliverers.addDeliverer(new SampleDeliverer(deliveryContext));
deliverers.addDeliverer(new DataSetDeliverer(deliveryContext));
+ deliverers.addDeliverer(new FileDeliverer(deliveryContext));
deliverer = deliverers;
}
@@ -277,6 +282,8 @@ public class DataSourceRequestHandler implements IRequestHandler
private String sessionToken;
private Date requestTimestamp;
+
+ private Set<String> fileServicePaths = new TreeSet<>();
public XMLStreamWriter getWriter()
{
@@ -347,5 +354,10 @@ public class DataSourceRequestHandler implements IRequestHandler
{
this.requestTimestamp = requestTimestamp;
}
+
+ public Set<String> getFileServicePaths()
+ {
+ return fileServicePaths;
+ }
}
}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DeliveryContext.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DeliveryContext.java
index 36c45511719..9f57249767d 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DeliveryContext.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DeliveryContext.java
@@ -16,6 +16,8 @@
package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.datasource;
+import java.io.File;
+
import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi;
import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
@@ -36,6 +38,8 @@ class DeliveryContext
private String openBisDataSourceName;
+ private File fileServiceRepository;
+
public String getServerUrl()
{
return serverUrl;
@@ -96,4 +100,14 @@ class DeliveryContext
this.openBisDataSourceName = openBisDataSourceName;
}
+ public File getFileServiceRepository()
+ {
+ return fileServiceRepository;
+ }
+
+ public void setFileServiceRepository(File fileServiceRepository)
+ {
+ this.fileServiceRepository = fileServiceRepository;
+ }
+
}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DeliveryExecutionContext.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DeliveryExecutionContext.java
index 8f40605510b..5f3b0530edb 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DeliveryExecutionContext.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DeliveryExecutionContext.java
@@ -38,6 +38,8 @@ class DeliveryExecutionContext
private Date requestTimestamp;
+ private Set<String> fileServicePaths;
+
public XMLStreamWriter getWriter()
{
return writer;
@@ -87,4 +89,14 @@ class DeliveryExecutionContext
{
this.requestTimestamp = requestTimestamp;
}
+
+ public Set<String> getFileServicePaths()
+ {
+ return fileServicePaths;
+ }
+
+ public void setFileServicePaths(Set<String> fileServicePaths)
+ {
+ this.fileServicePaths = fileServicePaths;
+ }
}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/ExperimentDeliverer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/ExperimentDeliverer.java
index ad20d53b45c..9e19a723102 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/ExperimentDeliverer.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/ExperimentDeliverer.java
@@ -48,6 +48,7 @@ public class ExperimentDeliverer extends AbstractEntityWithPermIdDeliverer
XMLStreamWriter writer = context.getWriter();
String sessionToken = context.getSessionToken();
Set<String> spaces = context.getSpaces();
+ Set<String> fileServicePaths = context.getFileServicePaths();
IApplicationServerApi v3api = getV3Api();
List<ExperimentPermId> permIds = experiments.stream().map(ExperimentPermId::new).collect(Collectors.toList());
Collection<Experiment> fullExperiments = v3api.getExperiments(sessionToken, permIds, createFullFetchOptions()).values();
@@ -70,7 +71,7 @@ public class ExperimentDeliverer extends AbstractEntityWithPermIdDeliverer
addRegistrator(writer, experiment);
addSpace(writer, experiment.getProject().getSpace());
addType(writer, experiment.getType());
- addProperties(writer, experiment.getProperties());
+ addProperties(writer, experiment.getProperties(), fileServicePaths);
// ConnectionsBuilder connectionsBuilder = new ConnectionsBuilder();
// connectionsBuilder.addConnections(experiment.getSamples());
// connectionsBuilder.addConnections(experiment.getDataSets());
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/FileDeliverer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/FileDeliverer.java
new file mode 100644
index 00000000000..8e1006fdfec
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/FileDeliverer.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2019 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.datasource;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Base64;
+import java.util.Date;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamWriter;
+
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
+
+/**
+ * @author Franz-Josef Elmer
+ *
+ */
+public class FileDeliverer extends AbstractEntityDeliverer<String>
+{
+
+ FileDeliverer(DeliveryContext context)
+ {
+ super(context, "file");
+ }
+
+ @Override
+ public void deliverEntities(DeliveryExecutionContext context) throws XMLStreamException
+ {
+ Set<String> paths = context.getFileServicePaths();
+ if (paths.isEmpty())
+ {
+ return;
+ }
+ File repository = this.context.getFileServiceRepository();
+ XMLStreamWriter writer = context.getWriter();
+ int count = 0;
+ long totalSize = 0;
+ for (String path : paths)
+ {
+ File file = new File(repository, path);
+ if (file.isFile())
+ {
+ startUrlElement(writer, "FILE", path, new Date(file.lastModified()));
+ startXdElement(writer);
+ writer.writeAttribute("path", path);
+ byte[] content = FileUtilities.loadToByteArray(file);
+ String contentAsBase64String = Base64.getEncoder().encodeToString(content);
+ writer.writeCharacters(contentAsBase64String);
+ writer.writeEndElement();
+ writer.writeEndElement();
+ count++;
+ totalSize += content.length;
+ }
+ }
+ operationLog.info(count + " files (total size: " + FileUtilities.byteCountToDisplaySize(totalSize) + ") have been delivered.");
+ }
+
+}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/MaterialDeliverer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/MaterialDeliverer.java
index a6896053db0..fc77302d770 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/MaterialDeliverer.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/MaterialDeliverer.java
@@ -60,6 +60,7 @@ public class MaterialDeliverer extends AbstractEntityDeliverer<Material>
{
XMLStreamWriter writer = context.getWriter();
String sessionToken = context.getSessionToken();
+ Set<String> fileServicePaths = context.getFileServicePaths();
IApplicationServerApi v3api = getV3Api();
List<MaterialPermId> permIds = materials.stream().map(Material::getPermId).collect(Collectors.toList());
Collection<Material> fullMaterials = v3api.getMaterials(sessionToken, permIds, createFullFetchOptions()).values();
@@ -84,7 +85,7 @@ public class MaterialDeliverer extends AbstractEntityDeliverer<Material>
{
allProperties.put(entity.getKey(), entity.getValue().getPermId().toString());
}
- addProperties(writer, allProperties);
+ addProperties(writer, allProperties, fileServicePaths);
writer.writeEndElement();
writer.writeEndElement();
}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/SampleDeliverer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/SampleDeliverer.java
index f8e52aa2390..c093c56c975 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/SampleDeliverer.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/SampleDeliverer.java
@@ -48,6 +48,7 @@ public class SampleDeliverer extends AbstractEntityWithPermIdDeliverer
XMLStreamWriter writer = context.getWriter();
String sessionToken = context.getSessionToken();
Set<String> spaces = context.getSpaces();
+ Set<String> fileServicePaths = context.getFileServicePaths();
IApplicationServerApi v3api = getV3Api();
List<SamplePermId> permIds = samplePermIds.stream().map(SamplePermId::new).collect(Collectors.toList());
Collection<Sample> fullSamples = v3api.getSamples(sessionToken, permIds, createFullFetchOptions()).values();
@@ -73,7 +74,7 @@ public class SampleDeliverer extends AbstractEntityWithPermIdDeliverer
addRegistrator(writer, sample);
addSpace(writer, sample.getSpace());
addType(writer, sample.getType());
- addProperties(writer, sample.getProperties());
+ addProperties(writer, sample.getProperties(), fileServicePaths);
ConnectionsBuilder connectionsBuilder = new ConnectionsBuilder();
connectionsBuilder.addConnections(sample.getDataSets());
connectionsBuilder.addChildren(sample.getChildren());
--
GitLab