SSDM-9973: PreStagingCleanUpMaintenanceTask introduced and automatically set up at DSS start up

datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/PreStagingCleanUpMaintenanceTask.java

+/*
+ * Copyright 2020 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins;
+
+import java.io.File;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Properties;
+
+import org.apache.commons.lang3.time.DateUtils;
+import org.apache.log4j.Logger;
+
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.maintenance.IMaintenanceTask;
+import ch.systemsx.cisd.common.properties.PropertyUtils;
+import ch.systemsx.cisd.common.utilities.ITimeProvider;
+import ch.systemsx.cisd.common.utilities.SystemTimeProvider;
+import ch.systemsx.cisd.etlserver.TopLevelDataSetRegistratorGlobalState;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.utils.SegmentedStoreUtils;
+
+/**
+ * @author Franz-Josef Elmer
+ */
+public class PreStagingCleanUpMaintenanceTask implements IMaintenanceTask
+{
+
+    public static final String DEFAULT_MAINTENANCE_TASK_NAME = "pre-staging-clean-up-task";
+
+    public static final int DEFAULT_MAINTENANCE_TASK_INTERVAL = (int) (DateUtils.MILLIS_PER_DAY / 1000);
+
+    public static final String MINIMUM_AGE_IN_DAYS = "minimum-age-in-days";
+
+    public static final int DEFAULT_MINIMUM_AGE_IN_DAYS = 30;
+
+    private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
+            PreStagingCleanUpMaintenanceTask.class);
+
+    private long minimumAge;
+
+    private File storeRoot;
+
+    private ITimeProvider timeProvider;
+
+    public PreStagingCleanUpMaintenanceTask()
+    {
+    }
+
+    PreStagingCleanUpMaintenanceTask(File storeRoot, ITimeProvider timeProvider)
+    {
+        this.storeRoot = storeRoot;
+        this.timeProvider = timeProvider;
+    }
+
+    @Override
+    public void setUp(String pluginName, Properties properties)
+    {
+        minimumAge = PropertyUtils.getInt(properties, MINIMUM_AGE_IN_DAYS, DEFAULT_MINIMUM_AGE_IN_DAYS) * DateUtils.MILLIS_PER_DAY;
+
+    }
+
+    @Override
+    public void execute()
+    {
+        File[] shares = SegmentedStoreUtils.getShares(getStoreRoot());
+        for (File share : shares)
+        {
+            File[] files = new File(share, TopLevelDataSetRegistratorGlobalState.DEFAULT_PRE_STAGING_DIR).listFiles();
+            if (files != null)
+            {
+                for (File file : files)
+                {
+                    String dateStamp = file.getName().split("_")[0];
+                    try
+                    {
+                        Date date = new SimpleDateFormat("yyyy-MM-dd").parse(dateStamp);
+                        if (date.getTime() + minimumAge < getTimeProvider().getTimeInMilliseconds())
+                        {
+                            FileUtilities.deleteRecursively(file);
+                            operationLog.info("Stale folder deleted: " + file.getAbsolutePath());
+                        }
+                    } catch (ParseException e)
+                    {
+                        // such files are not of interest
+                    }
+                }
+            }
+        }
+    }
+
+    private File getStoreRoot()
+    {
+        if (storeRoot == null)
+        {
+            storeRoot = ServiceProvider.getConfigProvider().getStoreRoot();
+        }
+        return storeRoot;
+    }
+
+    private ITimeProvider getTimeProvider()
+    {
+        if (timeProvider == null)
+        {
+            timeProvider = SystemTimeProvider.SYSTEM_TIME_PROVIDER;
+        }
+        return timeProvider;
+    }
+
+}
\ No newline at end of file

datastore_server/source/java/ch/systemsx/cisd/etlserver/ETLDaemon.java

@@ -35,6 +35,7 @@ import org.apache.commons.io.filefilter.NameFileFilter;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
 
+import ch.ethz.sis.openbis.generic.server.dss.plugins.PreStagingCleanUpMaintenanceTask;
 import ch.rinn.restrictions.Private;
 import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
 import ch.systemsx.cisd.base.exceptions.InterruptedExceptionUnchecked;
@@ -64,6 +65,7 @@ import ch.systemsx.cisd.common.logging.LogCategory;
 import ch.systemsx.cisd.common.logging.LogFactory;
 import ch.systemsx.cisd.common.mail.IMailClient;
 import ch.systemsx.cisd.common.mail.MailClient;
+import ch.systemsx.cisd.common.maintenance.IMaintenanceTask;
 import ch.systemsx.cisd.common.maintenance.MaintenancePlugin;
 import ch.systemsx.cisd.common.maintenance.MaintenanceTaskParameters;
 import ch.systemsx.cisd.common.maintenance.MaintenanceTaskUtils;
@@ -712,7 +714,11 @@ public final class ETLDaemon
         MaintenanceTaskUtils.startupMaintenancePlugins(maintenancePlugins);
 
         injectPostRegistrationMaintenanceTaskIfNecessary(maintenancePlugins);
-        injectDeleteDataSetsAlreadyDeletedInApplicationServerMaintenanceTaskIfNecessary(maintenancePlugins);
+        injectMaintenanceTask(maintenancePlugins, DeleteDataSetsAlreadyDeletedInApplicationServerMaintenanceTask.class,
+                300, "injected-delete-datasets-already-deleted-from-application-server-task");
+        injectMaintenanceTask(maintenancePlugins, PreStagingCleanUpMaintenanceTask.class,
+                PreStagingCleanUpMaintenanceTask.DEFAULT_MAINTENANCE_TASK_INTERVAL,
+                PreStagingCleanUpMaintenanceTask.DEFAULT_MAINTENANCE_TASK_NAME);
 
         operationLog.info("Data Store Server ready and waiting for data.");
     }
@@ -756,38 +762,31 @@ public final class ETLDaemon
         }
 
         PostRegistrationMaintenanceTask task = new PostRegistrationMaintenanceTask();
-        Properties props = new Properties();
-        props.setProperty(MaintenanceTaskParameters.CLASS_KEY, task.getClass().getName());
-        // Have the task run every second
-        props.setProperty(MaintenanceTaskParameters.INTERVAL_KEY,
-                Integer.toString(INJECTED_POST_REGISTRATION_TASK_INTERVAL));
-        MaintenanceTaskParameters parameters =
-                new MaintenanceTaskParameters(props, "injected-post-registration-task");
         task.setUpEmpty();
-
-        MaintenancePlugin plugin = new MaintenancePlugin(task, parameters);
-        MaintenanceTaskUtils.injectMaintenancePlugin(plugin);
+        injectMaintenanceTask(PostRegistrationMaintenanceTask.class, task,
+                INJECTED_POST_REGISTRATION_TASK_INTERVAL, "injected-post-registration-task");
     }
 
-    private static void injectDeleteDataSetsAlreadyDeletedInApplicationServerMaintenanceTaskIfNecessary(
-            MaintenanceTaskParameters[] maintenancePlugins)
+    private static void injectMaintenanceTask(MaintenanceTaskParameters[] maintenancePlugins,
+            Class<? extends IMaintenanceTask> taskClass, int interval, String pluginName)
     {
-        if (hasMaintenanceTaskOfClass(maintenancePlugins,
-                DeleteDataSetsAlreadyDeletedInApplicationServerMaintenanceTask.class))
+        if (hasMaintenanceTaskOfClass(maintenancePlugins, taskClass))
         {
             // Nothing additional to do.
             return;
         }
 
+        injectMaintenanceTask(taskClass, null, interval, pluginName);
+    }
+
+    private static void injectMaintenanceTask(Class<? extends IMaintenanceTask> taskClass, IMaintenanceTask task, int interval, String pluginName)
+    {
         Properties props = new Properties();
-        props.setProperty(MaintenanceTaskParameters.CLASS_KEY,
-                DeleteDataSetsAlreadyDeletedInApplicationServerMaintenanceTask.class.getName());
-        props.setProperty(MaintenanceTaskParameters.INTERVAL_KEY, Integer.toString(300));
-        MaintenanceTaskParameters parameters =
-                new MaintenanceTaskParameters(props,
-                        "injected-delete-datasets-already-deleted-from-application-server-task");
-
-        MaintenancePlugin plugin = new MaintenancePlugin(parameters);
+        props.setProperty(MaintenanceTaskParameters.CLASS_KEY, taskClass.getName());
+        props.setProperty(MaintenanceTaskParameters.INTERVAL_KEY, Integer.toString(interval));
+        MaintenanceTaskParameters parameters = new MaintenanceTaskParameters(props, pluginName);
+
+        MaintenancePlugin plugin = task == null ? new MaintenancePlugin(parameters) : new MaintenancePlugin(task, parameters);
         MaintenanceTaskUtils.injectMaintenancePlugin(plugin);
     }
 

datastore_server/source/java/ch/systemsx/cisd/etlserver/TopLevelDataSetRegistratorGlobalState.java

@@ -316,6 +316,8 @@ public class TopLevelDataSetRegistratorGlobalState implements IReleasable
     public static final String STAGING_DIR = "staging-dir";
 
     public static final String PRE_STAGING_DIR = "pre-staging-dir";
+    
+    public static final String DEFAULT_PRE_STAGING_DIR = "pre-staging";
 
     public static final String PRE_COMMIT_DIR = "pre-commit-dir";
 
@@ -329,7 +331,7 @@ public class TopLevelDataSetRegistratorGlobalState implements IReleasable
     private static File getPreStagingDir(File storeRoot, String shareId, Properties threadProperties)
     {
         return getShareLocalDir(storeRoot, shareId, threadProperties, PRE_STAGING_DIR,
-                "pre-staging");
+                DEFAULT_PRE_STAGING_DIR);
     }
 
     private static File getPreCommitDir(File storeRoot, String shareId, Properties threadProperties)