From f27def9af9633636f2e4ca6146095fa0197251b0 Mon Sep 17 00:00:00 2001
From: alaskowski <alaskowski@ethz.ch>
Date: Thu, 13 Jun 2024 11:05:53 +0200
Subject: [PATCH] BIS-1093: Added discard files pattern support
---
.../etc/service.properties | 10 +++++-
.../cisd/etlserver/ThreadParameters.java | 34 ++++++++++++++++---
2 files changed, 39 insertions(+), 5 deletions(-)
diff --git a/server-original-data-store/etc/service.properties b/server-original-data-store/etc/service.properties
index 671e5b04826..0675357b086 100644
--- a/server-original-data-store/etc/service.properties
+++ b/server-original-data-store/etc/service.properties
@@ -385,4 +385,12 @@ jython-version=2.7
#
default-incoming-share-id=1
default-incoming-share-minimum-free-space-in-gb=10
-discard-hidden-files=true
+eln-lims-dropbox-discard-hidden-files=false
+eln-lims-dropbox-allow-hidden-files=true
+#eln-lims-dropbox-discard-files-patterns =
+#
+# Imaging plugin configuration
+#
+imaging.nanonis.sxm-script-path=${core-plugins-folder}/imaging-nanonis/1/dss/services/imaging-nanonis/nanonis_sxm.py
+imaging.nanonis.dat-script-path=${core-plugins-folder}/imaging-nanonis/1/dss/services/imaging-nanonis/nanonis_dat.py
+imaging.imaging-test.script-path=${core-plugins-folder}/imaging-test/1/dss/services/imaging-test/test_adaptor_script.py
\ No newline at end of file
diff --git a/server-original-data-store/source/java/ch/systemsx/cisd/etlserver/ThreadParameters.java b/server-original-data-store/source/java/ch/systemsx/cisd/etlserver/ThreadParameters.java
index b200d9f70a6..cbf1ef49489 100644
--- a/server-original-data-store/source/java/ch/systemsx/cisd/etlserver/ThreadParameters.java
+++ b/server-original-data-store/source/java/ch/systemsx/cisd/etlserver/ThreadParameters.java
@@ -16,9 +16,9 @@
package ch.systemsx.cisd.etlserver;
import java.io.File;
-import java.util.Arrays;
-import java.util.Properties;
-import java.util.Vector;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Pattern;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
@@ -133,6 +133,10 @@ public final class ThreadParameters
public static final String DISCARD_HIDDEN_FILES = "discard-hidden-files";
+ public static final String ALLOW_HIDDEN_FILES = "allow-hidden-files";
+
+ public static final String DISCARD_FILES_PATTERNS = "discard-files-patterns";
+
/**
* The (local) directory to monitor for new files and directories to move to the remote side. The directory where data to be processed by the ETL
* server become available.
@@ -187,6 +191,12 @@ public final class ThreadParameters
private final boolean discardHiddenFiles;
+ private final boolean allowHiddenFiles;
+
+ private final String discardFilesPatterns;
+ private final Map<String, Pattern> discardFilesPatternsCache = new ConcurrentHashMap<>();
+
+
/**
* @param threadProperties parameters for one processing thread together with general parameters.
*/
@@ -196,6 +206,7 @@ public final class ThreadParameters
this.h5Folders = PropertyUtils.getBoolean(threadProperties, H5_FOLDERS, false);
this.h5arFolders = PropertyUtils.getBoolean(threadProperties, H5AR_FOLDERS, true);
this.discardHiddenFiles = PropertyUtils.getBoolean(threadProperties, DISCARD_HIDDEN_FILES, true);
+ this.allowHiddenFiles = PropertyUtils.getBoolean(threadProperties, ALLOW_HIDDEN_FILES, true);
this.createIncomingDirectories =
PropertyUtils.getBoolean(threadProperties, INCOMING_DIR_CREATE, true);
this.threadProperties = threadProperties;
@@ -275,7 +286,7 @@ public final class ThreadParameters
ON_ERROR_DECISION_KEY + ".class", ex.getMessage());
}
this.onErrorDecisionClassOrNull = onErrorClass;
-
+ this.discardFilesPatterns = PropertyUtils.getProperty(threadProperties, DISCARD_FILES_PATTERNS, "");
}
private DataSetRegistrationPreStagingBehavior getOriginalnputDataSetBehaviour(
@@ -578,6 +589,21 @@ public final class ThreadParameters
return discardHiddenFiles;
}
+ public boolean allowHiddenFiles()
+ {
+ return allowHiddenFiles;
+ }
+
+ public String discardFilesPatterns()
+ {
+ return discardFilesPatterns;
+ }
+
+ public Map<String, Pattern> discardFilesPatternsCache()
+ {
+ return discardFilesPatternsCache;
+ }
+
public boolean hasH5ArAsFolders()
{
return h5arFolders;
--
GitLab