SSDM-7493: data source provides files of file service if mentioned in some entity properties

datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/AbstractEntityDeliverer.java

@@ -23,6 +23,8 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
 import java.util.function.Function;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import javax.xml.stream.XMLStreamException;
 import javax.xml.stream.XMLStreamWriter;
@@ -40,6 +42,7 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment;
 import ch.ethz.sis.openbis.generic.asapi.v3.dto.project.Project;
 import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample;
 import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.Space;
+import ch.ethz.sis.openbis.generic.server.FileServiceServlet;
 import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
 import ch.systemsx.cisd.common.logging.LogCategory;
 import ch.systemsx.cisd.common.logging.LogFactory;
@@ -53,6 +56,9 @@ import ch.systemsx.cisd.openbis.generic.server.batch.IBatchOperation;
 abstract class AbstractEntityDeliverer<T> implements IDeliverer
 {
     private static final int CHUNK_SIZE = 1000;
+    
+    private static final Pattern FILE_SERVICE_PATTERN = Pattern.compile("openbis/" + FileServiceServlet.FILE_SERVICE_PATH
+            + "/([^\"']*)");
 
     private static interface IConsumer<T>
     {
@@ -95,7 +101,7 @@ abstract class AbstractEntityDeliverer<T> implements IDeliverer
         return context.getV3api();
     }
 
-    protected void addProperties(XMLStreamWriter writer, Map<String, String> properties) throws XMLStreamException
+    protected void addProperties(XMLStreamWriter writer, Map<String, String> properties, Set<String> fileServicePaths) throws XMLStreamException
     {
         if (properties.isEmpty() == false)
         {
@@ -108,7 +114,13 @@ abstract class AbstractEntityDeliverer<T> implements IDeliverer
                 writer.writeCharacters(entry.getKey());
                 writer.writeEndElement();
                 writer.writeStartElement("x:value");
-                writer.writeCharacters(entry.getValue());
+                String value = entry.getValue();
+                Matcher matcher = FILE_SERVICE_PATTERN.matcher(value);
+                while (matcher.find())
+                {
+                    fileServicePaths.add(matcher.group(1));
+                }
+                writer.writeCharacters(value);
                 writer.writeEndElement();
                 writer.writeEndElement();
             }

datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DataSetDeliverer.java

@@ -52,6 +52,7 @@ public class DataSetDeliverer extends AbstractEntityWithPermIdDeliverer
         XMLStreamWriter writer = context.getWriter();
         String sessionToken = context.getSessionToken();
         Set<String> spaces = context.getSpaces();
+        Set<String> fileServicePaths = context.getFileServicePaths();
         IApplicationServerApi v3api = getV3Api();
         List<DataSetPermId> permIds = dataSets.stream().map(DataSetPermId::new).collect(Collectors.toList());
         Collection<DataSet> fullDataSets = v3api.getDataSets(sessionToken, permIds, createDataSetFetchOptions()).values();
@@ -77,7 +78,7 @@ public class DataSetDeliverer extends AbstractEntityWithPermIdDeliverer
                 addRegistrator(writer, dataSet);
                 addSample(writer, dataSet.getSample());
                 addType(writer, dataSet.getType());
-                addProperties(writer, dataSet.getProperties());
+                addProperties(writer, dataSet.getProperties(), fileServicePaths);
                 addPhysicalData(writer, dataSet, code);
                 addLinkedData(writer, dataSet, code);
                 ConnectionsBuilder connectionsBuilder = new ConnectionsBuilder();

datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DataSourceRequestHandler.java

@@ -26,6 +26,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
+import java.util.TreeSet;
 import java.util.stream.Collectors;
 
 import javax.servlet.http.HttpServletRequest;
@@ -76,6 +77,7 @@ public class DataSourceRequestHandler implements IRequestHandler
                 executionContext.setSessionToken(sessionToken);
                 executionContext.setSpaces(requestedSpaces);
                 executionContext.setWriter(context.getWriter());
+                executionContext.setFileServicePaths(context.getFileServicePaths());
                 deliverer.deliverEntities(executionContext);
             }
         };
@@ -168,6 +170,8 @@ public class DataSourceRequestHandler implements IRequestHandler
         deliveryContext.setServletPath(new File(PropertyUtils.getMandatoryProperty(properties, "path")).getParent());
         deliveryContext.setServerUrl(PropertyUtils.getMandatoryProperty(properties, "server-url"));
         deliveryContext.setDownloadUrl(PropertyUtils.getMandatoryProperty(properties, "download-url"));
+        String fileServiceRepositoryPath = PropertyUtils.getMandatoryProperty(properties, "file-service-repository-path");
+        deliveryContext.setFileServiceRepository(new File(fileServiceRepositoryPath));
         deliveryContext.setV3api(ServiceProvider.getV3ApplicationService());
         deliveryContext.setContentProvider(ServiceProvider.getHierarchicalContentProvider());
         deliveryContext.setOpenBisDataSourceName(properties.getProperty("openbis-data-source-name", "openbis-db"));
@@ -179,6 +183,7 @@ public class DataSourceRequestHandler implements IRequestHandler
         deliverers.addDeliverer(new ExperimentDeliverer(deliveryContext));
         deliverers.addDeliverer(new SampleDeliverer(deliveryContext));
         deliverers.addDeliverer(new DataSetDeliverer(deliveryContext));
+        deliverers.addDeliverer(new FileDeliverer(deliveryContext));
         deliverer = deliverers;
     }
 
@@ -277,6 +282,8 @@ public class DataSourceRequestHandler implements IRequestHandler
         private String sessionToken;
 
         private Date requestTimestamp;
+        
+        private Set<String> fileServicePaths = new TreeSet<>();
 
         public XMLStreamWriter getWriter()
         {
@@ -347,5 +354,10 @@ public class DataSourceRequestHandler implements IRequestHandler
         {
             this.requestTimestamp = requestTimestamp;
         }
+
+        public Set<String> getFileServicePaths()
+        {
+            return fileServicePaths;
+        }
     }
 }

datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DeliveryContext.java

@@ -16,6 +16,8 @@
 
 package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.datasource;
 
+import java.io.File;
+
 import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi;
 import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
 
@@ -36,6 +38,8 @@ class DeliveryContext
 
     private String openBisDataSourceName;
 
+    private File fileServiceRepository;
+
     public String getServerUrl()
     {
         return serverUrl;
@@ -96,4 +100,14 @@ class DeliveryContext
         this.openBisDataSourceName = openBisDataSourceName;
     }
 
+    public File getFileServiceRepository()
+    {
+        return fileServiceRepository;
+    }
+
+    public void setFileServiceRepository(File fileServiceRepository)
+    {
+        this.fileServiceRepository = fileServiceRepository;
+    }
+
 }

datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/DeliveryExecutionContext.java

@@ -38,6 +38,8 @@ class DeliveryExecutionContext
 
     private Date requestTimestamp;
 
+    private Set<String> fileServicePaths;
+
     public XMLStreamWriter getWriter()
     {
         return writer;
@@ -87,4 +89,14 @@ class DeliveryExecutionContext
     {
         this.requestTimestamp = requestTimestamp;
     }
+
+    public Set<String> getFileServicePaths()
+    {
+        return fileServicePaths;
+    }
+
+    public void setFileServicePaths(Set<String> fileServicePaths)
+    {
+        this.fileServicePaths = fileServicePaths;
+    }
 }

datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/ExperimentDeliverer.java

@@ -48,6 +48,7 @@ public class ExperimentDeliverer extends AbstractEntityWithPermIdDeliverer
         XMLStreamWriter writer = context.getWriter();
         String sessionToken = context.getSessionToken();
         Set<String> spaces = context.getSpaces();
+        Set<String> fileServicePaths = context.getFileServicePaths();
         IApplicationServerApi v3api = getV3Api();
         List<ExperimentPermId> permIds = experiments.stream().map(ExperimentPermId::new).collect(Collectors.toList());
         Collection<Experiment> fullExperiments = v3api.getExperiments(sessionToken, permIds, createFullFetchOptions()).values();
@@ -70,7 +71,7 @@ public class ExperimentDeliverer extends AbstractEntityWithPermIdDeliverer
                 addRegistrator(writer, experiment);
                 addSpace(writer, experiment.getProject().getSpace());
                 addType(writer, experiment.getType());
-                addProperties(writer, experiment.getProperties());
+                addProperties(writer, experiment.getProperties(), fileServicePaths);
 //                ConnectionsBuilder connectionsBuilder = new ConnectionsBuilder();
 //                connectionsBuilder.addConnections(experiment.getSamples());
 //                connectionsBuilder.addConnections(experiment.getDataSets());

datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/FileDeliverer.java

+/*
+ * Copyright 2019 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.datasource;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Base64;
+import java.util.Date;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamWriter;
+
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
+
+/**
+ * @author Franz-Josef Elmer
+ *
+ */
+public class FileDeliverer extends AbstractEntityDeliverer<String>
+{
+
+    FileDeliverer(DeliveryContext context)
+    {
+        super(context, "file");
+    }
+
+    @Override
+    public void deliverEntities(DeliveryExecutionContext context) throws XMLStreamException
+    {
+        Set<String> paths = context.getFileServicePaths();
+        if (paths.isEmpty())
+        {
+            return;
+        }
+        File repository = this.context.getFileServiceRepository();
+        XMLStreamWriter writer = context.getWriter();
+        int count = 0;
+        long totalSize = 0;
+        for (String path : paths)
+        {
+            File file = new File(repository, path);
+            if (file.isFile())
+            {
+                startUrlElement(writer, "FILE", path, new Date(file.lastModified()));
+                startXdElement(writer);
+                writer.writeAttribute("path", path);
+                byte[] content = FileUtilities.loadToByteArray(file);
+                String contentAsBase64String = Base64.getEncoder().encodeToString(content);
+                writer.writeCharacters(contentAsBase64String);
+                writer.writeEndElement();
+                writer.writeEndElement();
+                count++;
+                totalSize += content.length;
+            }
+        }
+        operationLog.info(count + " files (total size: " + FileUtilities.byteCountToDisplaySize(totalSize) + ") have been delivered.");
+    }
+
+}

datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/MaterialDeliverer.java

@@ -60,6 +60,7 @@ public class MaterialDeliverer extends AbstractEntityDeliverer<Material>
     {
         XMLStreamWriter writer = context.getWriter();
         String sessionToken = context.getSessionToken();
+        Set<String> fileServicePaths = context.getFileServicePaths();
         IApplicationServerApi v3api = getV3Api();
         List<MaterialPermId> permIds = materials.stream().map(Material::getPermId).collect(Collectors.toList());
         Collection<Material> fullMaterials = v3api.getMaterials(sessionToken, permIds, createFullFetchOptions()).values();
@@ -84,7 +85,7 @@ public class MaterialDeliverer extends AbstractEntityDeliverer<Material>
             {
                 allProperties.put(entity.getKey(), entity.getValue().getPermId().toString());
             }
-            addProperties(writer, allProperties);
+            addProperties(writer, allProperties, fileServicePaths);
             writer.writeEndElement();
             writer.writeEndElement();
         }

datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/SampleDeliverer.java

@@ -48,6 +48,7 @@ public class SampleDeliverer extends AbstractEntityWithPermIdDeliverer
         XMLStreamWriter writer = context.getWriter();
         String sessionToken = context.getSessionToken();
         Set<String> spaces = context.getSpaces();
+        Set<String> fileServicePaths = context.getFileServicePaths();
         IApplicationServerApi v3api = getV3Api();
         List<SamplePermId> permIds = samplePermIds.stream().map(SamplePermId::new).collect(Collectors.toList());
         Collection<Sample> fullSamples = v3api.getSamples(sessionToken, permIds, createFullFetchOptions()).values();
@@ -73,7 +74,7 @@ public class SampleDeliverer extends AbstractEntityWithPermIdDeliverer
                 addRegistrator(writer, sample);
                 addSpace(writer, sample.getSpace());
                 addType(writer, sample.getType());
-                addProperties(writer, sample.getProperties());
+                addProperties(writer, sample.getProperties(), fileServicePaths);
                 ConnectionsBuilder connectionsBuilder = new ConnectionsBuilder();
                 connectionsBuilder.addConnections(sample.getDataSets());
                 connectionsBuilder.addChildren(sample.getChildren());