Merge branch 'stored-method-in-zip-export'

openbis_standard_technologies/dist/core-plugins/eln-lims/1/dss/reporting-plugins/exports-api/exportsApi.py

@@ -21,14 +21,13 @@ import jarray
 from ch.systemsx.cisd.openbis.dss.generic.server import DataStoreServer
 from ch.systemsx.cisd.openbis.generic.client.web.client.exception import UserFailureException
 # Zip Format
-from java.io import File
+from java.io import File, BufferedInputStream
 from java.io import FileInputStream
 from java.io import FileOutputStream
 from java.lang import String
 from java.lang import StringBuilder
 from java.util import ArrayList
-from java.util.zip import ZipEntry, Deflater
-from java.util.zip import ZipOutputStream
+from java.util.zip import ZipEntry, Deflater, ZipOutputStream, CRC32
 from org.apache.commons.io import FileUtils
 # Java Core
 from org.apache.commons.io import IOUtils
@@ -263,7 +262,7 @@ def cleanUp(tempDirPath, tempZipFilePath):
     FileUtils.forceDelete(File(tempZipFilePath));
 
 
-def generateFilesInZip(zos, entities, includeRoot, sessionToken, tempDirPath):
+def generateFilesInZip(zos, entities, includeRoot, sessionToken, tempDirPath, deflated=True):
     # Services used during the export process
     v3 = ServiceProvider.getV3ApplicationService();
     v3d = ServiceProvider.getApplicationContext().getBean(V3_DSS_BEAN);
@@ -364,7 +363,7 @@ def generateFilesInZip(zos, entities, includeRoot, sessionToken, tempDirPath):
             rawFile = File(tempDirPath + filePath + ".json");
             rawFile.getParentFile().mkdirs();
             IOUtils.copyLarge(rawFileInputStream, FileOutputStream(rawFile));
-            addToZipFile(filePath, rawFile, zos);
+            addToZipFile(filePath, rawFile, zos, deflated=deflated);
             emptyZip = False
 
         # To avoid empty directories on the zip file, it makes the first found entity the base directory
@@ -382,20 +381,20 @@ def generateFilesInZip(zos, entities, includeRoot, sessionToken, tempDirPath):
         if entityObj is not None and entityFilePath is not None:
             # JSON
             entityJson = String(objectMapper.writeValueAsString(entityObj));
-            fileMetadatum = addFile(tempDirPath, entityFilePath, "json", entityJson.getBytes(), zos);
+            fileMetadatum = addFile(tempDirPath, entityFilePath, "json", entityJson.getBytes(), zos, deflated=deflated);
             fileMetadata.append(fileMetadatum)
             emptyZip = False
             # TEXT
             entityTXT = String(getTXT(entityObj, v3, sessionToken, False));
-            fileMetadatum = addFile(tempDirPath, entityFilePath, "txt", entityTXT.getBytes(), zos);
+            fileMetadatum = addFile(tempDirPath, entityFilePath, "txt", entityTXT.getBytes(), zos, deflated=deflated);
             fileMetadata.append(fileMetadatum)
             # DOCX
             entityDOCX = getDOCX(entityObj, v3, sessionToken, False);
-            fileMetadatum = addFile(tempDirPath, entityFilePath, "docx", entityDOCX, zos);
+            fileMetadatum = addFile(tempDirPath, entityFilePath, "docx", entityDOCX, zos, deflated=deflated);
             fileMetadata.append(fileMetadatum)
             # HTML
             entityHTML = getDOCX(entityObj, v3, sessionToken, True);
-            fileMetadatum = addFile(tempDirPath, entityFilePath, "html", entityHTML, zos);
+            fileMetadatum = addFile(tempDirPath, entityFilePath, "html", entityHTML, zos, deflated=deflated);
             fileMetadata.append(fileMetadatum)
             operationLog.info("--> Entity type: " + type + " permId: " + permId + " post html.");
     if emptyZip:
@@ -651,12 +650,12 @@ def objToStrArray(objArray):
     return result
 
 
-def addFile(tempDirPath, entityFilePath, extension, fileContent, zos):
+def addFile(tempDirPath, entityFilePath, extension, fileContent, zos, deflated=True):
     entityFileNameWithExtension = entityFilePath + "." + extension
     entityFile = File(tempDirPath + entityFileNameWithExtension);
     entityFile.getParentFile().mkdirs();
     IOUtils.write(fileContent, FileOutputStream(entityFile));
-    addToZipFile(entityFileNameWithExtension, entityFile, zos);
+    addToZipFile(entityFileNameWithExtension, entityFile, zos, deflated=deflated);
     FileUtils.forceDelete(entityFile);
 
     extensionToMimeType = {
@@ -685,9 +684,17 @@ def getFilePath(spaceCode, projCode, expCode, sampCode, dataCode):
         fileName += "/" + dataCode;
     return fileName;
 
-def addToZipFile(path, file, zos):
+def addToZipFile(path, file, zos, deflated=True):
     fis = FileInputStream(file);
     zipEntry = ZipEntry(path[1:]); # Making paths relative to make them compatible with Windows zip implementation
+    if not deflated:
+        zipEntry.setMethod(ZipOutputStream.STORED)
+        zipEntry.setSize(file.length())
+        zipEntry.setCompressedSize(-1)
+        crc = getFileCRC(file)
+        zipEntry.setCrc(crc)
+    else:
+        zipEntry.setMethod(ZipOutputStream.DEFLATED)
     zos.putNextEntry(zipEntry);
 
     bytes = jarray.zeros(1024, "b");
@@ -699,6 +706,21 @@ def addToZipFile(path, file, zos):
     zos.closeEntry();
     fis.close();
 
+def getFileCRC(file):
+    bis = None
+    crc = CRC32()
+    try:
+        bis = BufferedInputStream(FileInputStream(file), 1024)
+        b = jarray.zeros(1024, "b")
+        i = bis.read(b)
+        while i != -1:
+            crc.update(b, 0, i)
+            i = bis.read(b)
+    finally:
+        if bis is not None:
+            bis.close()
+    return crc.getValue()
+
 def getConfigurationProperty(transaction, propertyName):
     threadProperties = transaction.getGlobalState().getThreadParameters().getThreadProperties();
     try:
@@ -719,9 +741,10 @@ def generateZipFile(entities, params, tempDirPath, tempZipFilePath, deflated=Tru
         fos = FileOutputStream(tempZipFilePath)
         zos = ZipOutputStream(fos)
         if not deflated:
+            zos.setMethod(ZipOutputStream.STORED)
             zos.setLevel(Deflater.NO_COMPRESSION)
 
-        fileMetadata = generateFilesInZip(zos, entities, includeRoot, sessionToken, tempDirPath)
+        fileMetadata = generateFilesInZip(zos, entities, includeRoot, sessionToken, tempDirPath, deflated=deflated)
     finally:
         if zos is not None:
             zos.close()

openbis_standard_technologies/dist/core-plugins/eln-lims/1/dss/reporting-plugins/rc-exports-api/rcExports.py

@@ -228,9 +228,10 @@ def generateExternalZipFile(params, exportDirPath, contentZipFilePath, contentZi
         fos = FileOutputStream(exportZipFileName)
         zos = ZipOutputStream(fos)
         if not deflated:
+            zos.setMethod(ZipOutputStream.STORED)
             zos.setLevel(Deflater.NO_COMPRESSION)
 
-        addToZipFile(' ' + contentZipFileName, File(contentZipFilePath), zos)
+        addToZipFile(' ' + contentZipFileName, File(contentZipFilePath), zos, deflated)
 
         generateXML(zipOutputStream=zos, fileMetadata=fileMetadata, exportDirPath=exportDirPath,
                     userInformation=userInformation, entities=entities, params=params)