BIS-1093: Added discard files pattern support

server-original-data-store/etc/service.properties

@@ -385,4 +385,12 @@ jython-version=2.7
 #
 default-incoming-share-id=1
 default-incoming-share-minimum-free-space-in-gb=10
-discard-hidden-files=true
+eln-lims-dropbox-discard-hidden-files=false
+eln-lims-dropbox-allow-hidden-files=true
+#eln-lims-dropbox-discard-files-patterns =
+#
+# Imaging plugin configuration
+#
+imaging.nanonis.sxm-script-path=${core-plugins-folder}/imaging-nanonis/1/dss/services/imaging-nanonis/nanonis_sxm.py
+imaging.nanonis.dat-script-path=${core-plugins-folder}/imaging-nanonis/1/dss/services/imaging-nanonis/nanonis_dat.py
+imaging.imaging-test.script-path=${core-plugins-folder}/imaging-test/1/dss/services/imaging-test/test_adaptor_script.py
\ No newline at end of file

server-original-data-store/source/java/ch/systemsx/cisd/etlserver/ThreadParameters.java

@@ -16,9 +16,9 @@
 package ch.systemsx.cisd.etlserver;
 
 import java.io.File;
-import java.util.Arrays;
-import java.util.Properties;
-import java.util.Vector;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Pattern;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -133,6 +133,10 @@ public final class ThreadParameters
 
     public static final String DISCARD_HIDDEN_FILES = "discard-hidden-files";
 
+    public static final String ALLOW_HIDDEN_FILES = "allow-hidden-files";
+
+    public static final String DISCARD_FILES_PATTERNS = "discard-files-patterns";
+
     /**
      * The (local) directory to monitor for new files and directories to move to the remote side. The directory where data to be processed by the ETL
      * server become available.
@@ -187,6 +191,12 @@ public final class ThreadParameters
 
     private final boolean discardHiddenFiles;
 
+    private final boolean allowHiddenFiles;
+
+    private final String discardFilesPatterns;
+    private final Map<String, Pattern> discardFilesPatternsCache = new ConcurrentHashMap<>();
+
+
     /**
      * @param threadProperties parameters for one processing thread together with general parameters.
      */
@@ -196,6 +206,7 @@ public final class ThreadParameters
         this.h5Folders = PropertyUtils.getBoolean(threadProperties, H5_FOLDERS, false);
         this.h5arFolders = PropertyUtils.getBoolean(threadProperties, H5AR_FOLDERS, true);
         this.discardHiddenFiles = PropertyUtils.getBoolean(threadProperties, DISCARD_HIDDEN_FILES, true);
+        this.allowHiddenFiles = PropertyUtils.getBoolean(threadProperties, ALLOW_HIDDEN_FILES, true);
         this.createIncomingDirectories =
                 PropertyUtils.getBoolean(threadProperties, INCOMING_DIR_CREATE, true);
         this.threadProperties = threadProperties;
@@ -275,7 +286,7 @@ public final class ThreadParameters
                     ON_ERROR_DECISION_KEY + ".class", ex.getMessage());
         }
         this.onErrorDecisionClassOrNull = onErrorClass;
-
+        this.discardFilesPatterns = PropertyUtils.getProperty(threadProperties, DISCARD_FILES_PATTERNS, "");
     }
 
     private DataSetRegistrationPreStagingBehavior getOriginalnputDataSetBehaviour(
@@ -578,6 +589,21 @@ public final class ThreadParameters
         return discardHiddenFiles;
     }
 
+    public boolean allowHiddenFiles()
+    {
+        return allowHiddenFiles;
+    }
+
+    public String discardFilesPatterns()
+    {
+        return discardFilesPatterns;
+    }
+
+    public Map<String, Pattern> discardFilesPatternsCache()
+    {
+        return discardFilesPatternsCache;
+    }
+
     public boolean hasH5ArAsFolders()
     {
         return h5arFolders;