SSDM-9972: Allow to configure free space for each drop box separately. Test...

SSDM-9972: Allow to configure free space for each drop box separately. Test share folder instead of incoming folder.

datastore_server/source/java/ch/systemsx/cisd/etlserver/ETLDaemon.java

@@ -30,6 +30,7 @@ import java.util.Set;
 import java.util.Timer;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.filefilter.FileFilterUtils;
 import org.apache.commons.io.filefilter.NameFileFilter;
 import org.apache.log4j.Level;
@@ -317,8 +318,13 @@ public final class ETLDaemon
         }
     }
 
-    private final static long getHighwaterMark(final Properties properties)
+    private final static long getHighwaterMark(ThreadParameters threadParameters, final Properties properties)
     {
+        Long incomingShareMinimumFreeSpace = threadParameters.getIncomingShareMinimumFreeSpace();
+        if (incomingShareMinimumFreeSpace != null)
+        {
+            return incomingShareMinimumFreeSpace / FileUtils.ONE_KB;
+        }
         return PropertyUtils.getLong(properties,
                 HostAwareFileWithHighwaterMark.HIGHWATER_MARK_PROPERTY_KEY, -1L);
     }
@@ -339,7 +345,7 @@ public final class ETLDaemon
             final IDataSetValidator dataSetValidator, final boolean notifySuccessfulRegistration)
     {
         final HighwaterMarkWatcher highwaterMarkWatcher =
-                new HighwaterMarkWatcher(getHighwaterMark(parameters.getProperties()));
+                new HighwaterMarkWatcher(getHighwaterMark(threadParameters, parameters.getProperties()));
         final File incomingDataDirectory = threadParameters.getIncomingDataDirectory();
         final File recoveryStateDirectory =
                 DssPropertyParametersUtil.getDssRecoveryStateDir(parameters.getProperties());
@@ -353,9 +359,11 @@ public final class ETLDaemon
                 createTopLevelDataSetRegistrator(parameters.getProperties(), threadParameters,
                         authorizedLimsService, mailClient, dataSetValidator,
                         new DataSourceQueryService(), notifySuccessfulRegistration);
+        TopLevelDataSetRegistratorGlobalState globalState = pathHandler.getGlobalState();
+        HostAwareFile shareFolder = new HostAwareFile(new File(globalState.getStoreRootDir(), globalState.getShareId()));
         final HighwaterMarkDirectoryScanningHandler directoryScanningHandler =
                 createDirectoryScanningHandler(pathHandler, highwaterMarkWatcher,
-                        hostAwareIncomingDataDirectory, hostAwareRecoveryStateDirectory,
+                        hostAwareIncomingDataDirectory, shareFolder, hostAwareRecoveryStateDirectory,
                         threadParameters.reprocessFaultyDatasets(),
                         parameters.getCheckIntervalMillis(), pathHandler);
         FileFilter fileFilter =
@@ -594,7 +602,7 @@ public final class ETLDaemon
 
     private final static HighwaterMarkDirectoryScanningHandler createDirectoryScanningHandler(
             final IStopSignaler stopSignaler, final HighwaterMarkWatcher highwaterMarkWatcher,
-            final HostAwareFile incomingDataDirectory, final HostAwareFile recoveryStateDirectory,
+            final HostAwareFile incomingDataDirectory, HostAwareFile shareFolder, final HostAwareFile recoveryStateDirectory,
             boolean reprocessFaultyDatasets, final long checkIntervalMillis,
             IFaultyPathDirectoryScanningHandlerDelegate faultyPathHandlerDelegate)
     {
@@ -603,7 +611,7 @@ public final class ETLDaemon
                         reprocessFaultyDatasets, checkIntervalMillis, faultyPathHandlerDelegate);
         return new HighwaterMarkDirectoryScanningHandler(faultyPathHandler, highwaterMarkWatcher,
                 new HostAwareFile[]
-                { incomingDataDirectory, recoveryStateDirectory });
+                { shareFolder, recoveryStateDirectory });
     }
 
     private static IDirectoryScanningHandler createFaultyPathHandler(
@@ -656,7 +664,7 @@ public final class ETLDaemon
                 {
                     if (scannedStore.existsOrError(storeItem))
                     {
-                        if (faultyItems.containsValue(storeItem.getName()) == false)
+                        if (faultyItems.containsKey(storeItem.getName()) == false)
                         {
                             StringBuffer sb = new StringBuffer();
                             sb.append("The thread configuration setting "

datastore_server/source/java/ch/systemsx/cisd/etlserver/ThreadParameters.java

@@ -21,6 +21,7 @@ import java.util.Arrays;
 import java.util.Properties;
 import java.util.Vector;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.log4j.Logger;
 
@@ -99,6 +100,8 @@ public final class ThreadParameters
     @Private
     public static final String INCOMING_SHARE_ID = "incoming-share-id";
 
+    private static final String INCOMING_SHARE_MINIMUM_FREE_SPACE_IN_GB = "incoming-share-minimum-free-space-in-gb";
+
     /*
      * The properties that control the process of retrying registration by jython dropboxes
      */
@@ -175,6 +178,8 @@ public final class ThreadParameters
 
     private final Integer incomingShareId;
 
+    private final Long incomingShareMinimumFreeSpace;
+
     private final boolean h5Folders;
 
     private final boolean h5arFolders;
@@ -221,6 +226,7 @@ public final class ThreadParameters
         this.dataSetRegistrationPreStagingBehavior =
                 getOriginalnputDataSetBehaviour(threadProperties);
         this.incomingShareId = tryGetIncomingShareId(threadProperties);
+        this.incomingShareMinimumFreeSpace = tryGetIncomingShareMinimumFreeSpace(threadProperties);
 
         boolean developmentMode =
                 PropertyUtils.getBoolean(threadProperties, RECOVERY_DEVELOPMENT_MODE, false);
@@ -288,6 +294,11 @@ public final class ThreadParameters
         return incomingShareId;
     }
 
+    public Long getIncomingShareMinimumFreeSpace()
+    {
+        return incomingShareMinimumFreeSpace;
+    }
+
     // true if marker file should be used, false if autodetection should be used, exceprion when the
     // value is invalid.
     private static boolean parseCompletenessCondition(String completenessCondition)
@@ -398,6 +409,12 @@ public final class ThreadParameters
         return Integer.parseInt(shareId);
     }
 
+    private static Long tryGetIncomingShareMinimumFreeSpace(Properties properties)
+    {
+        double min = PropertyUtils.getDouble(properties, INCOMING_SHARE_MINIMUM_FREE_SPACE_IN_GB, -1);
+        return min < 0 ? null : (long) (min * FileUtils.ONE_GB);
+    }
+
     private static String nullIfEmpty(String value)
     {
         return StringUtils.defaultIfEmpty(value, null);