From 51b1d2a1b2bdff742cd2d19b57797da9d3c2d561 Mon Sep 17 00:00:00 2001
From: juanf <juanf@ethz.ch>
Date: Thu, 30 May 2024 14:27:34 +0200
Subject: [PATCH] BIS-1059: Fix master data scripts
---
.../1/as/initialize-master-data.py | 2 +-
.../1/as/initialize-master-data.py | 2 +-
.../1/as/initialize-master-data.py | 2 +-
.../imaging/1/as/initialize-master-data.py | 2 +-
.../1/as/initialize-master-data.py | 2 +-
.../v1/impl/MasterDataRegistrationHelper.java | 164 +-----------------
.../eln-lims/1/as/initialize-master-data.py | 8 +-
7 files changed, 12 insertions(+), 170 deletions(-)
diff --git a/core-plugin-openbis/dist/core-plugins/eln-lims-template-types/1/as/initialize-master-data.py b/core-plugin-openbis/dist/core-plugins/eln-lims-template-types/1/as/initialize-master-data.py
index 131266e99d7..24d8775c589 100644
--- a/core-plugin-openbis/dist/core-plugins/eln-lims-template-types/1/as/initialize-master-data.py
+++ b/core-plugin-openbis/dist/core-plugins/eln-lims-template-types/1/as/initialize-master-data.py
@@ -29,7 +29,7 @@ api = CommonServiceProvider.getApplicationContext().getBean(ApplicationServerApi
sessionToken = api.loginAsSystem()
sessionWorkspaceFiles = helper.uploadToAsSessionWorkspace(sessionToken, "eln-types-template.xlsx", "scripts/date_range_validation.py")
-importData = ImportData(ImportFormat.EXCEL, sessionWorkspaceFiles[0])
+importData = ImportData(ImportFormat.EXCEL, [sessionWorkspaceFiles[0]])
importOptions = ImportOptions(ImportMode.UPDATE_IF_EXISTS)
importResult = api.executeImport(sessionToken, importData, importOptions)
diff --git a/core-plugin-openbis/dist/core-plugins/imaging-nanonis/1/as/initialize-master-data.py b/core-plugin-openbis/dist/core-plugins/imaging-nanonis/1/as/initialize-master-data.py
index 1971099332f..4abb942824b 100644
--- a/core-plugin-openbis/dist/core-plugins/imaging-nanonis/1/as/initialize-master-data.py
+++ b/core-plugin-openbis/dist/core-plugins/imaging-nanonis/1/as/initialize-master-data.py
@@ -28,7 +28,7 @@ helper = MasterDataRegistrationHelper(sys.path)
api = CommonServiceProvider.getApplicationContext().getBean(ApplicationServerApi.INTERNAL_SERVICE_NAME)
sessionToken = api.loginAsSystem()
sessionWorkspaceFiles = helper.uploadToAsSessionWorkspace(sessionToken, "imaging-test-data-model.xls")
-importData = ImportData(ImportFormat.EXCEL, sessionWorkspaceFiles[0])
+importData = ImportData(ImportFormat.EXCEL, [sessionWorkspaceFiles[0]])
importOptions = ImportOptions(ImportMode.UPDATE_IF_EXISTS)
importResult = api.executeImport(sessionToken, importData, importOptions)
diff --git a/core-plugin-openbis/dist/core-plugins/imaging-test/1/as/initialize-master-data.py b/core-plugin-openbis/dist/core-plugins/imaging-test/1/as/initialize-master-data.py
index 48e14efcc1b..90a68543be6 100644
--- a/core-plugin-openbis/dist/core-plugins/imaging-test/1/as/initialize-master-data.py
+++ b/core-plugin-openbis/dist/core-plugins/imaging-test/1/as/initialize-master-data.py
@@ -26,7 +26,7 @@ helper = MasterDataRegistrationHelper(sys.path)
api = CommonServiceProvider.getApplicationContext().getBean(ApplicationServerApi.INTERNAL_SERVICE_NAME)
sessionToken = api.loginAsSystem()
sessionWorkspaceFiles = helper.uploadToAsSessionWorkspace(sessionToken, "imaging-test-data-model.xls")
-importData = ImportData(ImportFormat.EXCEL, sessionWorkspaceFiles[0])
+importData = ImportData(ImportFormat.EXCEL, [sessionWorkspaceFiles[0]])
importOptions = ImportOptions(ImportMode.UPDATE_IF_EXISTS)
importResult = api.executeImport(sessionToken, importData, importOptions)
diff --git a/core-plugin-openbis/dist/core-plugins/imaging/1/as/initialize-master-data.py b/core-plugin-openbis/dist/core-plugins/imaging/1/as/initialize-master-data.py
index 06b1dbec36b..0a9255c46be 100644
--- a/core-plugin-openbis/dist/core-plugins/imaging/1/as/initialize-master-data.py
+++ b/core-plugin-openbis/dist/core-plugins/imaging/1/as/initialize-master-data.py
@@ -27,7 +27,7 @@ helper = MasterDataRegistrationHelper(sys.path)
api = CommonServiceProvider.getApplicationContext().getBean(ApplicationServerApi.INTERNAL_SERVICE_NAME)
sessionToken = api.loginAsSystem()
sessionWorkspaceFiles = helper.uploadToAsSessionWorkspace(sessionToken, "imaging-data-model.xls")
-importData = ImportData(ImportFormat.EXCEL, sessionWorkspaceFiles[0])
+importData = ImportData(ImportFormat.EXCEL, [sessionWorkspaceFiles[0]])
importOptions = ImportOptions(ImportMode.UPDATE_IF_EXISTS)
importResult = api.executeImport(sessionToken, importData, importOptions)
diff --git a/core-plugin-openbis/dist/core-plugins/search-store/1/as/initialize-master-data.py b/core-plugin-openbis/dist/core-plugins/search-store/1/as/initialize-master-data.py
index b616903f2c8..27db57e9218 100644
--- a/core-plugin-openbis/dist/core-plugins/search-store/1/as/initialize-master-data.py
+++ b/core-plugin-openbis/dist/core-plugins/search-store/1/as/initialize-master-data.py
@@ -27,7 +27,7 @@ helper = MasterDataRegistrationHelper(sys.path)
api = CommonServiceProvider.getApplicationContext().getBean(ApplicationServerApi.INTERNAL_SERVICE_NAME)
sessionToken = api.loginAsSystem()
sessionWorkspaceFiles = helper.uploadToAsSessionWorkspace(sessionToken, "data-model.xls")
-importData = ImportData(ImportFormat.EXCEL, sessionWorkspaceFiles[0])
+importData = ImportData(ImportFormat.EXCEL, [sessionWorkspaceFiles[0]])
importOptions = ImportOptions(ImportMode.UPDATE_IF_EXISTS)
importResult = api.executeImport(sessionToken, importData, importOptions)
api.logout(sessionToken)
diff --git a/server-application-server/source/java/ch/systemsx/cisd/openbis/generic/server/jython/api/v1/impl/MasterDataRegistrationHelper.java b/server-application-server/source/java/ch/systemsx/cisd/openbis/generic/server/jython/api/v1/impl/MasterDataRegistrationHelper.java
index ab757129794..3edec2bea6f 100644
--- a/server-application-server/source/java/ch/systemsx/cisd/openbis/generic/server/jython/api/v1/impl/MasterDataRegistrationHelper.java
+++ b/server-application-server/source/java/ch/systemsx/cisd/openbis/generic/server/jython/api/v1/impl/MasterDataRegistrationHelper.java
@@ -74,177 +74,19 @@ public class MasterDataRegistrationHelper {
throw new IllegalArgumentException("Does not contain path to the core plugin: " + systemPaths);
}
- public List<byte[]> listXlsByteArrays() {
- List<byte[]> result = new ArrayList<>();
- for (File file : masterDataFolder.listFiles()) {
- String name = file.getName();
- if (name.endsWith(".xls") || name.endsWith(".xlsx")) {
- operationLog.info("load master data " + file.getName());
- result.add(FileUtilities.loadToByteArray(file));
- }
- }
- return result;
- }
-
- public Map<String, InputStream> getXlsInputStreamMap()
- {
- return Arrays.stream(masterDataFolder.listFiles()).filter(file -> file.getName().endsWith(".xls") || file.getName().endsWith(".xlsx"))
- .collect(Collectors.toMap(File::getName, file1 ->
- {
- try
- {
- return new FileInputStream(file1);
- } catch (final FileNotFoundException e)
- {
- throw new RuntimeException(e);
- }
- }));
- }
-
- private static String[] getFilePaths(final String... fileNames)
- {
- return Arrays.stream(fileNames).map(fileName -> MasterDataRegistrationHelper.class.getResource("master-data/" + fileName).getPath())
- .toArray(String[]::new);
- }
-
public String[] uploadToAsSessionWorkspace(final String sessionToken, final String... relativeFilePaths) throws IOException
{
- final String[] canonicalFilePaths = getFilePaths(relativeFilePaths);
final ISessionWorkspaceProvider sessionWorkspaceProvider = CommonServiceProvider.getSessionWorkspaceProvider();
final String uploadId = UUID.randomUUID().toString();
- final String[] destinations = new String[canonicalFilePaths.length];
+ final String[] destinations = new String[relativeFilePaths.length];
- for (int i = 0; i < canonicalFilePaths.length; i++)
+ for (int i = 0; i < relativeFilePaths.length; i++)
{
destinations[i] = uploadId + "/" + relativeFilePaths[i];
- sessionWorkspaceProvider.write(sessionToken, destinations[i], new FileInputStream(canonicalFilePaths[i]));
+ sessionWorkspaceProvider.write(sessionToken, destinations[i], new FileInputStream(masterDataFolder.getCanonicalPath() + "/" + relativeFilePaths[i]));
}
return destinations;
}
- public List<byte[]> getByteArray(String findName) {
- for (File file : masterDataFolder.listFiles()) {
- String name = file.getName();
- if (name.equals(findName)) {
- operationLog.info("load master data " + file.getName());
- return Arrays.asList(FileUtilities.loadToByteArray(file));
- }
- }
- return null;
- }
-
- public List<byte[]> listCsvByteArrays() throws IOException {
- List<byte[]> result = new ArrayList<>();
- for (File file : masterDataFolder.listFiles()) {
- String name = file.getName();
- if (name.endsWith(".csv")) {
- operationLog.info("load master data " + file.getName());
- result.add(Files.readAllBytes(file.toPath()));
- }
- }
- return result;
- }
-
- public Map<String, String> getAllScripts() {
- Map<String, String> result = new TreeMap<>();
- File scriptsFolder = new File(masterDataFolder, "scripts");
- if (scriptsFolder.isDirectory()) {
- gatherScripts(result, scriptsFolder, scriptsFolder);
- }
- return result;
- }
-
- private static void gatherScripts(Map<String, String> scripts, File rootFolder, File file) {
- if (file.isFile()) {
- String scriptPath = FileUtilities.getRelativeFilePath(rootFolder, file);
- scripts.put(scriptPath, FileUtilities.loadToString(file));
- operationLog.info("Script " + scriptPath + " loaded");
- }
- if (file.isDirectory()) {
- File[] files = file.listFiles();
- for (File child : files) {
- gatherScripts(scripts, rootFolder, child);
- }
- }
- }
-
- public static Map<String, String> getAllScripts(Path path) {
- Map<String, String> result = new TreeMap<>();
- File scriptsFolder = new File(path.toFile(), "scripts");
- if (scriptsFolder.isDirectory()) {
- gatherScripts(result, scriptsFolder, scriptsFolder);
- }
- return result;
- }
-
- public static Map<String, String> getAllLargeValues(Path path)
- {
- final Map<String, String> result = new TreeMap<>();
- final File valuesFolder = new File(path.toFile(), "data");
- if (valuesFolder.isDirectory())
- {
- gatherFileValues(result, valuesFolder, valuesFolder);
- }
- return result;
- }
-
- private static void gatherFileValues(final Map<String, String> values, final File rootFolder, final File file) {
- if (file.isFile())
- {
- final String filePath = FileUtilities.getRelativeFilePath(rootFolder, file);
- values.put(filePath, FileUtilities.loadToString(file));
- operationLog.info("File " + filePath + " loaded");
- }
- if (file.isDirectory())
- {
- final File[] files = file.listFiles();
- for (final File child : files)
- {
- gatherScripts(values, rootFolder, child);
- }
- }
- }
-
- public static List<byte[]> getByteArrays(Path path, String findName) {
- List<byte[]> byteArrays = new ArrayList<>();
- for (File file : path.toFile().listFiles()) {
- String name = file.getName();
- if (name.contains(findName)) {
- operationLog.info("load master data " + file.getName());
- byteArrays.add(FileUtilities.loadToByteArray(file));
- }
- }
- return byteArrays;
- }
-
- public static void extractToDestination(byte[] zip, String tempPathAsString) throws IOException
- {
- // Write temp file
- Path tempZipPath = Paths.get(tempPathAsString, "temp.zip");
- Files.write(tempZipPath, zip);
-
- try (ZipFile zipFile = new ZipFile(tempZipPath.toFile())) {
- Enumeration<? extends ZipEntry> entries = zipFile.entries();
- while (entries.hasMoreElements()) {
- ZipEntry entry = entries.nextElement();
- File entryDestination = new File(tempPathAsString, entry.getName());
- if (entry.isDirectory())
- {
- entryDestination.mkdirs();
- } else
- {
- entryDestination.getParentFile().mkdirs();
- try (InputStream in = zipFile.getInputStream(entry))
- {
- Files.copy(in, entryDestination.toPath(), StandardCopyOption.REPLACE_EXISTING);
- }
- }
- }
- }
-
- // Delete temp file leaving on the folder only the uncompressed content
- Files.delete(tempZipPath);
- }
-
}
diff --git a/ui-eln-lims/src/core-plugins/eln-lims/1/as/initialize-master-data.py b/ui-eln-lims/src/core-plugins/eln-lims/1/as/initialize-master-data.py
index fe4c3ad61d5..ea4e9d19dad 100644
--- a/ui-eln-lims/src/core-plugins/eln-lims/1/as/initialize-master-data.py
+++ b/ui-eln-lims/src/core-plugins/eln-lims/1/as/initialize-master-data.py
@@ -38,19 +38,19 @@ if ELNFixes.isELNInstalled():
helper = MasterDataRegistrationHelper(sys.path)
sessionWorkspaceFiles = helper.uploadToAsSessionWorkspace(sessionToken, "common-data-model.xls", "scripts/date_range_validation.py",
"scripts/storage_position_validation.py")
-importData = ImportData(ImportFormat.EXCEL, sessionWorkspaceFiles[0])
+importData = ImportData(ImportFormat.EXCEL, [sessionWorkspaceFiles[0]])
importOptions = ImportOptions(ImportMode.UPDATE_IF_EXISTS)
importResult = api.executeImport(sessionToken, importData, importOptions)
if not ELNFixes.isMultiGroup():
sessionWorkspaceFiles = helper.uploadToAsSessionWorkspace(sessionToken, "single-group-data-model.xls", "scripts/date_range_validation.py",
- "storage_position_validation.py")
- importData = ImportData(ImportFormat.EXCEL, sessionWorkspaceFiles[0])
+ "scripts/storage_position_validation.py")
+ importData = ImportData(ImportFormat.EXCEL, [sessionWorkspaceFiles[0]])
importOptions = ImportOptions(ImportMode.UPDATE_IF_EXISTS)
importResult = api.executeImport(sessionToken, importData, importOptions)
ELNCollectionTypeMigration.afterUpgrade()
api.logout(sessionToken)
print("======================== master-data xls ingestion result ========================")
-print(result)
+print(importResult.getObjectIds())
print("======================== master-data xls ingestion result ========================")
--
GitLab