diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/SyncEntityKind.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/SyncEntityKind.java
index 8c6bd5523674e8dfc5406d196223cf33f9c58559..e51146daaf94059062a5f96930264e668fbd880e 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/SyncEntityKind.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/SyncEntityKind.java
@@ -23,7 +23,7 @@ package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common;
*/
public enum SyncEntityKind
{
- SPACE("SP"), PROJECT("P"), EXPERIMENT("E"), SAMPLE("S"), DATA_SET("D"), MATERIAL("M");
+ SPACE("SP"), PROJECT("P"), EXPERIMENT("E"), SAMPLE("S"), DATA_SET("D"), MATERIAL("M"), FILE("F");
private final String abbreviation;
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/FileDeliverer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/FileDeliverer.java
index 8e1006fdfec29315d0e7f685c964f0a17499b18f..cb9c7e717abd4ecb08cd19f97042a2c89f268685 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/FileDeliverer.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/datasource/FileDeliverer.java
@@ -17,12 +17,9 @@
package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.datasource;
import java.io.File;
-import java.io.IOException;
import java.util.Base64;
import java.util.Date;
-import java.util.List;
import java.util.Set;
-import java.util.stream.Collectors;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamWriter;
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/config/SyncConfig.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/config/SyncConfig.java
index f060643e82b6777c24f2f64f1e9e8a07aa882303..6e2794ed245b888e28bc630ed0678cec1452cf87 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/config/SyncConfig.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/config/SyncConfig.java
@@ -124,6 +124,16 @@ public class SyncConfig
}
}
+ public String getFileServiceReporitoryPath()
+ {
+ return fileServiceReporitoryPath;
+ }
+
+ public void setFileServiceReporitoryPath(String fileServiceReporitoryPath)
+ {
+ this.fileServiceReporitoryPath = fileServiceReporitoryPath;
+ }
+
public String getHarvesterTempDir()
{
return harvesterTempDir;
@@ -220,6 +230,8 @@ public class SyncConfig
private String harvesterTempDir;
+ private String fileServiceReporitoryPath;
+
private List<EMailAddress> emailAddresses = new ArrayList<>();
private String logFilePath;
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/config/SynchronizationConfigReader.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/config/SynchronizationConfigReader.java
index 7fceca6222ea4011850f9a890043b54520941647..770cd19bf9eb8bbc3b6a8820ade1b12e44b24ff9 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/config/SynchronizationConfigReader.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/config/SynchronizationConfigReader.java
@@ -59,6 +59,8 @@ public class SynchronizationConfigReader
private static final String HARVESTER_TEMP_DIR_PROPERTY_NAME = "harvester-tmp-dir";
+ private static final String FILE_SERVICE_REPOSITORY_PATH_PROPERTY_NAME = "file-service-repository-path";
+
private static final String HARVESTER_LAST_SYNC_TIMESTAMP_FILE_PROPERTY_NAME = "last-sync-timestamp-file";
private static final String HARVESTER_NOT_SYNCED_ENTITIES_FILE_NAME = "not-synced-entities-file";
@@ -143,6 +145,7 @@ public class SynchronizationConfigReader
}
config.setHarvesterTempDir(reader.getString(section, HARVESTER_TEMP_DIR_PROPERTY_NAME, DEFAULT_HARVESTER_TEMP_DIR, false));
+ config.setFileServiceReporitoryPath(reader.getString(section, FILE_SERVICE_REPOSITORY_PATH_PROPERTY_NAME, null, true));
config.setTranslateUsingDataSourceAlias(reader.getBoolean(section, TRANSLATE_USING_DATA_SOURCE_ALIAS_PROPERTY_NAME, true));
// read full-sync configuration
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
index a18b0a5a785a374a8380280bf0f4e9a93c300f71..f3b91b6146160c55cab3ace0972e59ea9754fdf0 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
@@ -31,6 +31,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import java.util.stream.Collectors;
@@ -205,6 +206,7 @@ public class EntitySynchronizer
registerMasterData(data.getMasterData());
MultiKeyMap<String, String> newEntities = registerEntities(data);
List<String> notSyncedAttachmentsHolders = registerAttachments(data, newEntities);
+ populateFileServiceRepository(data);
registerDataSets(data, notSyncedAttachmentsHolders);
if (config.keepOriginalTimestampsAndUsers())
@@ -649,6 +651,26 @@ public class EntitySynchronizer
monitor.log();
return notSyncedAttachmentsHolders;
}
+
+ private void populateFileServiceRepository(ResourceListParserData data)
+ {
+ Monitor monitor = new Monitor("Populate file service repository", operationLog);
+ operationLog.info("Processing files...");
+ File fileRepo = new File(config.getFileServiceReporitoryPath());
+ int count = 0;
+ long totalSize = 0;
+ for (Entry<String, byte[]> entry : data.getFileToProcess().entrySet())
+ {
+ String path = entry.getKey();
+ byte[] fileContent = entry.getValue();
+ File file = new File(fileRepo, path);
+ file.getParentFile().mkdirs();
+ FileUtilities.writeToFile(file, fileContent);
+ count++;
+ totalSize += fileContent.length;
+ }
+ monitor.log(count + " files (total size: " + FileUtilities.byteCountToDisplaySize(totalSize) + ") have been saved.");
+ }
private MultiKeyMap<String, String> registerEntities(ResourceListParserData data)
{
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParser.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParser.java
index ecbd21e00be3cf0880c9e0bdcd7de1c3beeec113..9517cd0f82c645303600400c394f45ceda38b85c 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParser.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParser.java
@@ -22,6 +22,7 @@ import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
+import java.util.Base64;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
@@ -271,6 +272,9 @@ public class ResourceListParser
} else if (SyncEntityKind.MATERIAL.toString().equals(entityKind))
{
parseMaterialMetaData(extractMaterialCodeFromURI(uri), xdNode, lastModificationDate);
+ } else if (SyncEntityKind.FILE.toString().equals(entityKind))
+ {
+ parseFileData(xdNode, lastModificationDate);
}
}
@@ -504,6 +508,14 @@ public class ResourceListParser
expCode);
}
+ private void parseFileData(Node xdNode, Date lastModificationDate)
+ {
+ String path = extractAttribute(xdNode, "path", false);
+ String base64EncodedFileContent = xdNode.getTextContent();
+ byte[] content = Base64.getDecoder().decode(base64EncodedFileContent);
+ data.getFileToProcess().put(path, content);
+ }
+
private void parseSpaceMetaData(Node xdNode, Date lastModificationDate)
{
String code = nameTranslator.translate(extractCode(xdNode));
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParserData.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParserData.java
index 40f63b44ea73349f08a556f96f06b9ebbe228c25..7b3de02e2fcbf9bab4c657ad877c8abc7145dd35 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParserData.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParserData.java
@@ -52,6 +52,8 @@ public class ResourceListParserData
private Map<String, IncomingDataSet> dataSetsToProcess = new HashMap<String, IncomingDataSet>();
private MultiKeyMap<String, IncomingMaterial> materialsToProcess = new MultiKeyMap<String, IncomingMaterial>();
+
+ private Map<String, byte[]> filesToProcess = new HashMap<>();
public MasterData getMasterData()
{
@@ -67,6 +69,11 @@ public class ResourceListParserData
{
this.resourceListTimestamp = resourceListTimestamp;
}
+
+ public Map<String, byte[]> getFileToProcess()
+ {
+ return filesToProcess;
+ }
public Set<String> getHarvesterSpaceList()
{