SSDM-13422: Zip support for master data imports

server-application-server/source/java/ch/systemsx/cisd/openbis/generic/server/jython/api/v1/impl/MasterDataRegistrationHelper.java

@@ -17,8 +17,13 @@ package ch.systemsx.cisd.openbis.generic.server.jython.api.v1.impl;
 
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.*;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
 
 import org.apache.log4j.Logger;
 
@@ -100,7 +105,7 @@ public class MasterDataRegistrationHelper {
         return result;
     }
 
-    private void gatherScripts(Map<String, String> scripts, File rootFolder, File file) {
+    private static void gatherScripts(Map<String, String> scripts, File rootFolder, File file) {
         if (file.isFile()) {
             String scriptPath = FileUtilities.getRelativeFilePath(rootFolder, file);
             scripts.put(scriptPath, FileUtilities.loadToString(file));
@@ -114,4 +119,51 @@ public class MasterDataRegistrationHelper {
         }
     }
 
+    public static Map<String, String> getAllScripts(Path path) {
+        Map<String, String> result = new TreeMap<>();
+        File scriptsFolder = new File(path.toFile(), "scripts");
+        if (scriptsFolder.isDirectory()) {
+            gatherScripts(result, scriptsFolder, scriptsFolder);
+        }
+        return result;
+    }
+
+    public static List<byte[]> getByteArrays(Path path, String findName) {
+        List<byte[]> byteArrays = new ArrayList<>();
+        for (File file : path.toFile().listFiles()) {
+            String name = file.getName();
+            if (name.contains(findName)) {
+                operationLog.info("load master data " + file.getName());
+                byteArrays.add(FileUtilities.loadToByteArray(file));
+            }
+        }
+        return byteArrays;
+    }
+
+    public static void extractToDestination(byte[] zip, String tempPathAsString) throws IOException
+    {
+        // Write temp file
+        Path tempZipPath = Paths.get(tempPathAsString, "temp.zip");
+        Files.write(tempZipPath, zip);
+
+        try (ZipFile zipFile = new ZipFile(tempZipPath.toFile())) {
+            Enumeration<? extends ZipEntry> entries = zipFile.entries();
+            while (entries.hasMoreElements()) {
+                ZipEntry entry = entries.nextElement();
+                File entryDestination = new File(tempPathAsString,  entry.getName());
+                if (entry.isDirectory()) {
+                    entryDestination.mkdirs();
+                } else {
+                    entryDestination.getParentFile().mkdirs();
+                    try (InputStream in = zipFile.getInputStream(entry)) {
+                        Files.copy(in, entryDestination.toPath());
+                    }
+                }
+            }
+        }
+
+        // Delete temp file leaving on the folder only the uncompressed content
+        Files.delete(tempZipPath);
+    }
+
 }

ui-admin/src/core-plugins/admin/1/as/services/xls-import/xls-import.py

 import base64
+from ch.systemsx.cisd.openbis.generic.server.jython.api.v1.impl import MasterDataRegistrationHelper
 from ch.ethz.sis.openbis.generic.server.xls.importer import ImportOptions
 from ch.ethz.sis.openbis.generic.server.xls.importer import XLSImport
 from ch.ethz.sis.openbis.generic.server.xls.importer.enums import ImportModes
 from ch.systemsx.cisd.common.exceptions import UserFailureException
 from java.util import ArrayList
-
+from org.apache.commons.io import FileUtils
+from java.io import File
+from java.lang import Long
+from java.lang import System
+from java.nio.file import Path
 
 def get_update_mode(parameters):
     update_mode = parameters.get('update_mode', 'FAIL_IF_EXISTS')
@@ -41,8 +46,29 @@ def process(context, parameters):
     result = None
 
     if method == "import":
+        zip = parameters.get('zip', False)
+        temp = None
+
+
+        if zip: # Zip mode uses xls_base64 for all multiple XLS + script files
+            zip_bytes = base64.b64decode(parameters.get('xls_base64'))
+            temp = File.createTempFile("temp", Long.toString(System.nanoTime()))
+            temp.delete()
+            temp.mkdir()
+            tempPath = temp.getAbsolutePath()
+            MasterDataRegistrationHelper.extractToDestination(zip_bytes, tempPath)
+            byteArrays = MasterDataRegistrationHelper.getByteArrays(Path.of(tempPath), ".xls")
+            parameters.put('xls', byteArrays)
+            allScripts = MasterDataRegistrationHelper.getAllScripts(Path.of(tempPath))
+            parameters.put('scripts', allScripts)
+        else:
+            # Check if xls_base64 is used for a single XLS
+            xls_base64_string = parameters.get('xls_base64', None)
+            if xls_base64_string is not None:
+                parameters.set('xls', [ base64.b64decode(xls_base64_string) ])
         result = _import(context, parameters)
-
+        if temp is not None:
+            FileUtils.deleteDirectory(temp)
     return result
 
 
@@ -73,25 +99,17 @@ def _import(context, parameters):
             }
         :return: Openbis's execute operations result string. It should contain report on what was created.
     """
-    api, session_token = context.applicationService, context.sessionToken
-
-    xls_byte_arrays = parameters.get('xls', None)
-    xls_base64_string = parameters.get('xls_base64', None)
-    xls_name = parameters.get('xls_name', None)
-    zip = parameters.get('zip', False)
+    session_token = context.sessionToken
+    api = context.applicationService
     scripts = parameters.get('scripts', {})
     mode = get_update_mode(parameters)
     options = get_import_options(parameters)
-
-    if zip:
-        raise UserFailureException('Zip imports not yet supported');
-
-    if xls_byte_arrays is None and xls_base64_string is not None:
-        xls_byte_arrays = [ base64.b64decode(xls_base64_string) ]
+    xls_name = parameters.get('xls_name', None)
 
     importXls = XLSImport(session_token, api, scripts, mode, options, xls_name)
 
     ids = ArrayList()
+    xls_byte_arrays = parameters.get('xls', None)
     for xls_byte_array in xls_byte_arrays:
         ids.addAll(importXls.importXLS(xls_byte_array))