From 6d0f3b516d45bc2d891e3e4bef4cb2a559d84e91 Mon Sep 17 00:00:00 2001
From: juanf <juanf@ethz.ch>
Date: Wed, 8 Mar 2023 11:27:15 +0100
Subject: [PATCH] SSDM-13422: Zip support for master data imports
---
.../v1/impl/MasterDataRegistrationHelper.java | 54 ++++++++++++++++++-
.../1/as/services/xls-import/xls-import.py | 46 +++++++++++-----
2 files changed, 85 insertions(+), 15 deletions(-)
diff --git a/server-application-server/source/java/ch/systemsx/cisd/openbis/generic/server/jython/api/v1/impl/MasterDataRegistrationHelper.java b/server-application-server/source/java/ch/systemsx/cisd/openbis/generic/server/jython/api/v1/impl/MasterDataRegistrationHelper.java
index aa023a84384..29278ae699e 100644
--- a/server-application-server/source/java/ch/systemsx/cisd/openbis/generic/server/jython/api/v1/impl/MasterDataRegistrationHelper.java
+++ b/server-application-server/source/java/ch/systemsx/cisd/openbis/generic/server/jython/api/v1/impl/MasterDataRegistrationHelper.java
@@ -17,8 +17,13 @@ package ch.systemsx.cisd.openbis.generic.server.jython.api.v1.impl;
import java.io.File;
import java.io.IOException;
+import java.io.InputStream;
import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.*;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
import org.apache.log4j.Logger;
@@ -100,7 +105,7 @@ public class MasterDataRegistrationHelper {
return result;
}
- private void gatherScripts(Map<String, String> scripts, File rootFolder, File file) {
+ private static void gatherScripts(Map<String, String> scripts, File rootFolder, File file) {
if (file.isFile()) {
String scriptPath = FileUtilities.getRelativeFilePath(rootFolder, file);
scripts.put(scriptPath, FileUtilities.loadToString(file));
@@ -114,4 +119,51 @@ public class MasterDataRegistrationHelper {
}
}
+ public static Map<String, String> getAllScripts(Path path) {
+ Map<String, String> result = new TreeMap<>();
+ File scriptsFolder = new File(path.toFile(), "scripts");
+ if (scriptsFolder.isDirectory()) {
+ gatherScripts(result, scriptsFolder, scriptsFolder);
+ }
+ return result;
+ }
+
+ public static List<byte[]> getByteArrays(Path path, String findName) {
+ List<byte[]> byteArrays = new ArrayList<>();
+ for (File file : path.toFile().listFiles()) {
+ String name = file.getName();
+ if (name.contains(findName)) {
+ operationLog.info("load master data " + file.getName());
+ byteArrays.add(FileUtilities.loadToByteArray(file));
+ }
+ }
+ return byteArrays;
+ }
+
+ public static void extractToDestination(byte[] zip, String tempPathAsString) throws IOException
+ {
+ // Write temp file
+ Path tempZipPath = Paths.get(tempPathAsString, "temp.zip");
+ Files.write(tempZipPath, zip);
+
+ try (ZipFile zipFile = new ZipFile(tempZipPath.toFile())) {
+ Enumeration<? extends ZipEntry> entries = zipFile.entries();
+ while (entries.hasMoreElements()) {
+ ZipEntry entry = entries.nextElement();
+ File entryDestination = new File(tempPathAsString, entry.getName());
+ if (entry.isDirectory()) {
+ entryDestination.mkdirs();
+ } else {
+ entryDestination.getParentFile().mkdirs();
+ try (InputStream in = zipFile.getInputStream(entry)) {
+ Files.copy(in, entryDestination.toPath());
+ }
+ }
+ }
+ }
+
+ // Delete temp file leaving on the folder only the uncompressed content
+ Files.delete(tempZipPath);
+ }
+
}
diff --git a/ui-admin/src/core-plugins/admin/1/as/services/xls-import/xls-import.py b/ui-admin/src/core-plugins/admin/1/as/services/xls-import/xls-import.py
index 17d8c6267fd..ef0e4dbedeb 100644
--- a/ui-admin/src/core-plugins/admin/1/as/services/xls-import/xls-import.py
+++ b/ui-admin/src/core-plugins/admin/1/as/services/xls-import/xls-import.py
@@ -1,10 +1,15 @@
import base64
+from ch.systemsx.cisd.openbis.generic.server.jython.api.v1.impl import MasterDataRegistrationHelper
from ch.ethz.sis.openbis.generic.server.xls.importer import ImportOptions
from ch.ethz.sis.openbis.generic.server.xls.importer import XLSImport
from ch.ethz.sis.openbis.generic.server.xls.importer.enums import ImportModes
from ch.systemsx.cisd.common.exceptions import UserFailureException
from java.util import ArrayList
-
+from org.apache.commons.io import FileUtils
+from java.io import File
+from java.lang import Long
+from java.lang import System
+from java.nio.file import Path
def get_update_mode(parameters):
update_mode = parameters.get('update_mode', 'FAIL_IF_EXISTS')
@@ -41,8 +46,29 @@ def process(context, parameters):
result = None
if method == "import":
+ zip = parameters.get('zip', False)
+ temp = None
+
+
+ if zip: # Zip mode uses xls_base64 for all multiple XLS + script files
+ zip_bytes = base64.b64decode(parameters.get('xls_base64'))
+ temp = File.createTempFile("temp", Long.toString(System.nanoTime()))
+ temp.delete()
+ temp.mkdir()
+ tempPath = temp.getAbsolutePath()
+ MasterDataRegistrationHelper.extractToDestination(zip_bytes, tempPath)
+ byteArrays = MasterDataRegistrationHelper.getByteArrays(Path.of(tempPath), ".xls")
+ parameters.put('xls', byteArrays)
+ allScripts = MasterDataRegistrationHelper.getAllScripts(Path.of(tempPath))
+ parameters.put('scripts', allScripts)
+ else:
+ # Check if xls_base64 is used for a single XLS
+ xls_base64_string = parameters.get('xls_base64', None)
+ if xls_base64_string is not None:
+ parameters.set('xls', [ base64.b64decode(xls_base64_string) ])
result = _import(context, parameters)
-
+ if temp is not None:
+ FileUtils.deleteDirectory(temp)
return result
@@ -73,25 +99,17 @@ def _import(context, parameters):
}
:return: Openbis's execute operations result string. It should contain report on what was created.
"""
- api, session_token = context.applicationService, context.sessionToken
-
- xls_byte_arrays = parameters.get('xls', None)
- xls_base64_string = parameters.get('xls_base64', None)
- xls_name = parameters.get('xls_name', None)
- zip = parameters.get('zip', False)
+ session_token = context.sessionToken
+ api = context.applicationService
scripts = parameters.get('scripts', {})
mode = get_update_mode(parameters)
options = get_import_options(parameters)
-
- if zip:
- raise UserFailureException('Zip imports not yet supported');
-
- if xls_byte_arrays is None and xls_base64_string is not None:
- xls_byte_arrays = [ base64.b64decode(xls_base64_string) ]
+ xls_name = parameters.get('xls_name', None)
importXls = XLSImport(session_token, api, scripts, mode, options, xls_name)
ids = ArrayList()
+ xls_byte_arrays = parameters.get('xls', None)
for xls_byte_array in xls_byte_arrays:
ids.addAll(importXls.importXLS(xls_byte_array))
--
GitLab