From 6dea0fa3cd5946855f1274dfd6b9bf08d09a3d58 Mon Sep 17 00:00:00 2001
From: vkovtun <vkovtun@ethz.ch>
Date: Thu, 18 Apr 2024 17:39:17 +0200
Subject: [PATCH] BIS-1043: Appending "#UUID" to the names of the folders for
export for reimport. Renamed "meta.json" to "metadata.json".
---
.../v3/executor/exporter/ExportExecutor.java | 50 ++++++------
.../executor/exporter/ExportExecutorTest.java | 81 +++++++------------
.../eln-lims-dropbox/eln-lims-dropbox.py | 7 +-
3 files changed, 60 insertions(+), 78 deletions(-)
diff --git a/server-application-server/source/java/ch/ethz/sis/openbis/generic/server/asapi/v3/executor/exporter/ExportExecutor.java b/server-application-server/source/java/ch/ethz/sis/openbis/generic/server/asapi/v3/executor/exporter/ExportExecutor.java
index 3d8790d6e5d..ff7d9ba1775 100644
--- a/server-application-server/source/java/ch/ethz/sis/openbis/generic/server/asapi/v3/executor/exporter/ExportExecutor.java
+++ b/server-application-server/source/java/ch/ethz/sis/openbis/generic/server/asapi/v3/executor/exporter/ExportExecutor.java
@@ -79,8 +79,6 @@ import java.util.zip.ZipEntry;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
-import com.openhtmltopdf.extend.FSSupplier;
-
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
import org.apache.log4j.Logger;
@@ -97,6 +95,7 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.fasterxml.jackson.databind.node.TextNode;
+import com.openhtmltopdf.extend.FSSupplier;
import com.openhtmltopdf.pdfboxout.PdfRendererBuilder;
import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi;
@@ -175,16 +174,12 @@ public class ExportExecutor implements IExportExecutor
public static final String EXPORT_FILE_PREFIX = "export";
- public static final String METADATA_FILE_NAME = "metadata" + XLSExport.XLSX_EXTENSION;
-
public static final String XLSX_DIRECTORY = "xlsx";
public static final String PDF_DIRECTORY = "pdf";
public static final String DATA_DIRECTORY = "data";
- public static final String META_FILE_NAME = "meta.json";
-
public static final String SHARED_SAMPLES_DIRECTORY = "(shared)";
public static final String HTML_EXTENSION = ".html";
@@ -193,6 +188,10 @@ public class ExportExecutor implements IExportExecutor
public static final String JSON_EXTENSION = ".json";
+ public static final String METADATA_FILE_NAME = "metadata" + XLSExport.XLSX_EXTENSION;
+
+ public static final String METADATA_JSON_FILE_NAME = "metadata" + JSON_EXTENSION;
+
static final String NAME_PROPERTY_NAME = "$NAME";
private static final String TYPE_EXPORT_FIELD_KEY = "TYPE";
@@ -577,9 +576,10 @@ public class ExportExecutor implements IExportExecutor
final String dataSetCode = dataSet.getCode();
final String dataSetTypeCode = dataSet.getType().getCode();
final String dataSetName = getEntityName(dataSet);
+ final String dataDirectorySuffix = "#" + UUID.randomUUID();
final File metadataJsonFile = createMetadataJsonFile(parentDataDirectory, prefix, spaceCode, projectCode, containerCode, code,
- dataSetTypeCode, dataSetCode, dataSetName, codeHolderJson, compatibleWithImport);
+ dataSetTypeCode, dataSetCode, dataSetName, dataDirectorySuffix, codeHolderJson, compatibleWithImport);
createDocFilesForDataSet(sessionToken, metadataJsonFile.getParentFile(), null, dataSet, EnumSet.of(ExportFormat.PDF));
if (dataSet.getKind() != DataSetKind.LINK)
@@ -604,7 +604,7 @@ public class ExportExecutor implements IExportExecutor
while ((file = reader.read()) != null)
{
createNextDataFile(parentDataDirectory, prefix, spaceCode, projectCode,
- containerCode, code, dataSetTypeCode, dataSetCode, dataSetName, file, compatibleWithImport);
+ containerCode, code, dataSetTypeCode, dataSetCode, dataSetName, dataDirectorySuffix, file, compatibleWithImport);
}
}
} else
@@ -628,33 +628,34 @@ public class ExportExecutor implements IExportExecutor
private static File createMetadataJsonFile(final File parentDataDirectory, final char prefix,
final String spaceCode, final String projectCode, final String containerCode, final String code, final String dataSetTypeCode,
- final String dataSetCode, final String dataSetName, final String codeHolderJson, final boolean compatibleWithImport) throws IOException
+ final String dataSetCode, final String dataSetName, final String dataDirectorySuffix, final String codeHolderJson,
+ final boolean compatibleWithImport) throws IOException
{
- final File metadataFile;
+ final File metadataJsonFile;
if (compatibleWithImport)
{
final File dataDirectory = new File(parentDataDirectory, DATA_DIRECTORY + '/');
mkdirs(dataDirectory);
- metadataFile = new File(dataDirectory,
- getDataDirectoryName(prefix, spaceCode, projectCode, containerCode, code, dataSetTypeCode, dataSetCode, dataSetName,
- META_FILE_NAME));
+ metadataJsonFile = new File(dataDirectory,
+ getDataDirectoryName(prefix, spaceCode, projectCode, containerCode, code, dataSetTypeCode,
+ dataDirectorySuffix, METADATA_JSON_FILE_NAME));
} else
{
final File dataDirectory = new File(parentDataDirectory, getFullEntityName(dataSetCode, dataSetName));
mkdirs(dataDirectory);
- metadataFile = new File(new File(dataDirectory, DATA_DIRECTORY), META_FILE_NAME);
+ metadataJsonFile = new File(new File(dataDirectory, DATA_DIRECTORY), METADATA_JSON_FILE_NAME);
}
- final File dataSubdirectory = metadataFile.getParentFile();
+ final File dataSubdirectory = metadataJsonFile.getParentFile();
mkdirs(dataSubdirectory);
- try (final OutputStream os = new BufferedOutputStream(new FileOutputStream(metadataFile)))
+ try (final OutputStream os = new BufferedOutputStream(new FileOutputStream(metadataJsonFile)))
{
writeInChunks(os, codeHolderJson.getBytes(StandardCharsets.UTF_8));
}
- return metadataFile;
+ return metadataJsonFile;
}
private void exportSpacesDoc(final String sessionToken, final Map<String, Map<String, List<Map<String, String>>>> exportFields,
@@ -1232,7 +1233,8 @@ public class ExportExecutor implements IExportExecutor
private static void createNextDataFile(final File parentDataDirectory, final char prefix, final String spaceCode, final String projectCode,
final String containerCode, final String entityCode, final String dataSetTypeCode, final String dataSetCode,
- final String dataSetName, final DataSetFileDownload dataSetFileDownload, final boolean compatibleWithImport) throws IOException
+ final String dataSetName, final String dataDirectorySuffix, final DataSetFileDownload dataSetFileDownload,
+ final boolean compatibleWithImport) throws IOException
{
final DataSetFile dataSetFile = dataSetFileDownload.getDataSetFile();
final String filePath = dataSetFile.getPath();
@@ -1244,7 +1246,7 @@ public class ExportExecutor implements IExportExecutor
final File dataDirectory = new File(parentDataDirectory, DATA_DIRECTORY + '/');
mkdirs(dataDirectory);
dataSetFsEntry = new File(dataDirectory, getDataDirectoryName(prefix, spaceCode, projectCode, containerCode, entityCode,
- dataSetTypeCode, dataSetCode, dataSetName, filePath) + (isDirectory ? "/" : ""));
+ dataSetTypeCode, dataDirectorySuffix, filePath) + (isDirectory ? "/" : ""));
} else
{
final File datasetDirectory = new File(parentDataDirectory, getFullEntityName(dataSetCode, dataSetName));
@@ -1272,7 +1274,7 @@ public class ExportExecutor implements IExportExecutor
static String getDataDirectoryName(final char prefix, final String spaceCode, final String projectCode,
final String containerCode, final String entityCode, final String dataSetTypeCode,
- final String dataSetCode, final String dataSetName, final String fileName)
+ final String dataDirectorySuffix, final String fileName)
{
if (prefix != 'O' && prefix != 'E')
{
@@ -1322,13 +1324,9 @@ public class ExportExecutor implements IExportExecutor
throw new IllegalArgumentException("Data set type code is mandatory");
}
- if (dataSetCode != null)
- {
- entryBuilder.append('+');
- addFullEntityName(entryBuilder, null, dataSetCode, dataSetName);
- } else
+ if (dataDirectorySuffix != null)
{
- throw new IllegalArgumentException("Data set code is mandatory");
+ entryBuilder.append(dataDirectorySuffix);
}
if (fileName != null)
diff --git a/server-application-server/sourceTest/java/ch/ethz/sis/openbis/generic/server/asapi/v3/executor/exporter/ExportExecutorTest.java b/server-application-server/sourceTest/java/ch/ethz/sis/openbis/generic/server/asapi/v3/executor/exporter/ExportExecutorTest.java
index aa8e5edb343..d67b599f33c 100644
--- a/server-application-server/sourceTest/java/ch/ethz/sis/openbis/generic/server/asapi/v3/executor/exporter/ExportExecutorTest.java
+++ b/server-application-server/sourceTest/java/ch/ethz/sis/openbis/generic/server/asapi/v3/executor/exporter/ExportExecutorTest.java
@@ -49,6 +49,8 @@ public class ExportExecutorTest
private static final String DATA_SET_CODE = "TEST_DATA_SET";
+ public static final String UUID_SUFFIX = "#123e4567-e89b-12d3-a456-426614174000";
+
private static final Object[][] NEXT_ZIP_ENTRY_DATA = {
{
null, null, null, null, null, null, null, null, null, "/"
@@ -237,82 +239,58 @@ public class ExportExecutorTest
private static final Object[][] FOLDER_NAME_DATA = {
{
- 'O', "DEFAULT_LAB_NOTEBOOK", "DEFAULT_PROJECT", null, "OBJ1", "ANALYZED_DATA", "20231201115004780-27", "my dataset", null,
- "O+DEFAULT_LAB_NOTEBOOK+DEFAULT_PROJECT+OBJ1+ANALYZED_DATA+my dataset (20231201115004780-27)"
- },
- {
- 'O', "DEFAULT_LAB_NOTEBOOK", "DEFAULT_PROJECT", null, "OBJ1", "ANALYZED_DATA", "20231201115004780-27", "my dataset", "file",
- "O+DEFAULT_LAB_NOTEBOOK+DEFAULT_PROJECT+OBJ1+ANALYZED_DATA+my dataset (20231201115004780-27)/file"
- },
- {
- 'O', "DEFAULT_LAB_NOTEBOOK", "DEFAULT_PROJECT", null, "OBJ1", "ANALYZED_DATA", "20231201115004780-27", null, "file",
- "O+DEFAULT_LAB_NOTEBOOK+DEFAULT_PROJECT+OBJ1+ANALYZED_DATA+20231201115004780-27/file"
- },
- {
- 'O', "DEFAULT_LAB_NOTEBOOK", null, null, "OBJ1", "ANALYZED_DATA", "20231201115004780-27", "my dataset", "file",
- "O+DEFAULT_LAB_NOTEBOOK+OBJ1+ANALYZED_DATA+my dataset (20231201115004780-27)/file"
+ 'O', "DEFAULT_LAB_NOTEBOOK", "DEFAULT_PROJECT", null, "OBJ1", "ANALYZED_DATA", null,
+ "O+DEFAULT_LAB_NOTEBOOK+DEFAULT_PROJECT+OBJ1+ANALYZED_DATA" + UUID_SUFFIX
},
{
- 'O', "DEFAULT_LAB_NOTEBOOK", null, null, "OBJ1", "ANALYZED_DATA", "20231201115004780-27", null, "file",
- "O+DEFAULT_LAB_NOTEBOOK+OBJ1+ANALYZED_DATA+20231201115004780-27/file"
+ 'O', "DEFAULT_LAB_NOTEBOOK", "DEFAULT_PROJECT", null, "OBJ1", "ANALYZED_DATA", "file",
+ "O+DEFAULT_LAB_NOTEBOOK+DEFAULT_PROJECT+OBJ1+ANALYZED_DATA" + UUID_SUFFIX + "/file"
},
{
- 'O', null, null, null, "OBJ1", "ANALYZED_DATA", "20231201115004780-27", "my dataset", "file",
- "O+OBJ1+ANALYZED_DATA+my dataset (20231201115004780-27)/file"
+ 'O', "DEFAULT_LAB_NOTEBOOK", null, null, "OBJ1", "ANALYZED_DATA", "file",
+ "O+DEFAULT_LAB_NOTEBOOK+OBJ1+ANALYZED_DATA" + UUID_SUFFIX + "/file"
},
{
- 'O', null, null, null, "OBJ1", "ANALYZED_DATA", "20231201115004780-27", null, "file",
- "O+OBJ1+ANALYZED_DATA+20231201115004780-27/file"
+ 'O', null, null, null, "OBJ1", "ANALYZED_DATA", "file",
+ "O+OBJ1+ANALYZED_DATA" + UUID_SUFFIX + "/file"
},
{
- 'E', "DEFAULT_LAB_NOTEBOOK", "DEFAULT_PROJECT", null, "EXP1", "ANALYZED_DATA", "20231201115004780-27", "my dataset", "file",
- "E+DEFAULT_LAB_NOTEBOOK+DEFAULT_PROJECT+EXP1+ANALYZED_DATA+my dataset (20231201115004780-27)/file"
+ 'E', "DEFAULT_LAB_NOTEBOOK", "DEFAULT_PROJECT", null, "EXP1", "ANALYZED_DATA", "file",
+ "E+DEFAULT_LAB_NOTEBOOK+DEFAULT_PROJECT+EXP1+ANALYZED_DATA" + UUID_SUFFIX + "/file"
},
{
- 'O', "DEFAULT_LAB_NOTEBOOK", "DEFAULT_PROJECT", "OBJ_CONTAINER", "OBJ1", "ANALYZED_DATA", "20231201115004780-27", "my dataset",
- "file", "O+DEFAULT_LAB_NOTEBOOK+DEFAULT_PROJECT+OBJ_CONTAINER*OBJ1+ANALYZED_DATA+my dataset (20231201115004780-27)/file"
+ 'O', "DEFAULT_LAB_NOTEBOOK", "DEFAULT_PROJECT", "OBJ_CONTAINER", "OBJ1", "ANALYZED_DATA",
+ "file", "O+DEFAULT_LAB_NOTEBOOK+DEFAULT_PROJECT+OBJ_CONTAINER*OBJ1+ANALYZED_DATA" + UUID_SUFFIX + "/file"
},
};
private static final Object[][] ERRONEOUS_FOLDER_NAME_DATA = {
{
- 'P', "DEFAULT_LAB_NOTEBOOK", "DEFAULT_PROJECT", null, "OBJ1", "ANALYZED_DATA", "20231201115004780-27", "my dataset", "file"
+ 'P', "DEFAULT_LAB_NOTEBOOK", "DEFAULT_PROJECT", null, "OBJ1", "ANALYZED_DATA", "file"
},
{
- 'O', null, "DEFAULT_PROJECT", null, "OBJ1", "ANALYZED_DATA", "20231201115004780-27", "my dataset", "file"
+ 'O', null, "DEFAULT_PROJECT", null, "OBJ1", "ANALYZED_DATA", "file"
},
{
- 'O', "DEFAULT_LAB_NOTEBOOK", "DEFAULT_PROJECT", null, null, "ANALYZED_DATA", "20231201115004780-27", "my dataset", "file"
+ 'O', "DEFAULT_LAB_NOTEBOOK", "DEFAULT_PROJECT", null, null, "ANALYZED_DATA", "file"
},
{
- 'O', "DEFAULT_LAB_NOTEBOOK", "DEFAULT_PROJECT", "OBJ_CONTAINER", null, "ANALYZED_DATA", "20231201115004780-27", "my dataset",
- "file"
+ 'O', "DEFAULT_LAB_NOTEBOOK", "DEFAULT_PROJECT", "OBJ_CONTAINER", null, "ANALYZED_DATA", "file"
},
{
- 'O', "DEFAULT_LAB_NOTEBOOK", "DEFAULT_PROJECT", null, "OBJ1", null, "20231201115004780-27", "my dataset", "file"
+ 'O', "DEFAULT_LAB_NOTEBOOK", "DEFAULT_PROJECT", null, "OBJ1", null, "file"
},
- {
- 'O', "DEFAULT_LAB_NOTEBOOK", "DEFAULT_PROJECT", null, "OBJ1", "ANALYZED_DATA", null, "my dataset", "file"
- },
- {
- 'O', "DEFAULT_LAB_NOTEBOOK", "DEFAULT_PROJECT", null, "OBJ1", "ANALYZED_DATA", null, null, "file"
- },
- {
- 'O', "DEFAULT_LAB_NOTEBOOK", "DEFAULT_PROJECT", null, "OBJ1", "ANALYZED_DATA", null, null, null
- },
-
{
// Experiments cannot have containers
- 'E', "DEFAULT_LAB_NOTEBOOK", "DEFAULT_PROJECT", "EXP_CONTAINER", "EXP1", "ANALYZED_DATA", "20231201115004780-27", "my dataset",
- "file"
+ 'E', "DEFAULT_LAB_NOTEBOOK", "DEFAULT_PROJECT", "EXP_CONTAINER", "EXP1", "ANALYZED_DATA", "file"
},
{
// Experiments cannot be on the space level
- 'E', "DEFAULT_LAB_NOTEBOOK", null, null, "EXP1", "ANALYZED_DATA", "20231201115004780-27", "my dataset", "file"
+ 'E', "DEFAULT_LAB_NOTEBOOK", null, null, "EXP1", "ANALYZED_DATA", "file"
},
{
// Experiments cannot be shared (w/o space)
- 'E', null, null, null, "EXP1", "ANALYZED_DATA", "20231201115004780-27", "my dataset", "file"
+ 'E', null, null, null, "EXP1", "ANALYZED_DATA", "file"
},
};
@@ -360,19 +338,20 @@ public class ExportExecutorTest
@Test(dataProvider = FOLDER_NAME_DATA_PROVIDER)
public void testGetDataDirectoryName(final char prefix, final String spaceCode, final String projectCode,
- final String containerCode, final String entityCode, final String dataSetTypeCode, final String dataSetCode,
- final String dataSetName, final String fileName, final String expectedResult)
+ final String containerCode, final String entityCode, final String dataSetTypeCode,
+ final String fileName, final String expectedResult)
{
- assertEquals(ExportExecutor.getDataDirectoryName(prefix, spaceCode, projectCode, containerCode, entityCode, dataSetTypeCode, dataSetCode,
- dataSetName, fileName), expectedResult);
+ assertEquals(ExportExecutor.getDataDirectoryName(prefix, spaceCode, projectCode, containerCode, entityCode, dataSetTypeCode,
+ UUID_SUFFIX, fileName), expectedResult);
}
@Test(dataProvider = ERRONEOUS_FOLDER_NAME_DATA_PROVIDER, expectedExceptions = IllegalArgumentException.class)
public void testGetDataDirectoryNameError(final char prefix, final String spaceCode, final String projectCode,
- final String containerCode, final String entityCode, final String dataSetTypeCode, final String dataSetCode,
- final String dataSetName, final String fileName)
+ final String containerCode, final String entityCode, final String dataSetTypeCode,
+ final String fileName)
{
- ExportExecutor.getDataDirectoryName(prefix, spaceCode, projectCode, containerCode, entityCode, dataSetTypeCode, dataSetCode, dataSetName, fileName);
+ ExportExecutor.getDataDirectoryName(prefix, spaceCode, projectCode, containerCode, entityCode, dataSetTypeCode,
+ UUID_SUFFIX, fileName);
}
@Test()
diff --git a/ui-eln-lims/src/core-plugins/eln-lims/1/dss/drop-boxes/eln-lims-dropbox/eln-lims-dropbox.py b/ui-eln-lims/src/core-plugins/eln-lims/1/dss/drop-boxes/eln-lims-dropbox/eln-lims-dropbox.py
index 45db4f0ff5d..fb8f8700956 100644
--- a/ui-eln-lims/src/core-plugins/eln-lims/1/dss/drop-boxes/eln-lims-dropbox/eln-lims-dropbox.py
+++ b/ui-eln-lims/src/core-plugins/eln-lims/1/dss/drop-boxes/eln-lims-dropbox/eln-lims-dropbox.py
@@ -30,7 +30,7 @@ errorMessages = []
def process(transaction):
incoming = transaction.getIncoming();
- folderName = incoming.getName();
+ folderName = substring_up_to_hash(incoming.getName().split("#"));
emailAddress = None
try:
@@ -199,6 +199,11 @@ def process(transaction):
reportAllIssues(transaction, emailAddress)
+def substring_up_to_hash(input_string):
+ hash_index = input_string.rfind('#')
+ return input_string[:hash_index] if hash_index >=0 else input_string
+
+
def pathListToStr(list):
return "\n".join(list)
--
GitLab