From c50ee40f0027c142d3742bd104e8695ecaf9fcfa Mon Sep 17 00:00:00 2001
From: felmer <franz-josef.elmer@id.ethz.ch>
Date: Tue, 27 Dec 2022 08:12:49 +0100
Subject: [PATCH] SSDM-11605: bug (caused by empty shares) fixed and tested.
Log messages improved
---
.../DataSetRegistrationCleanUpTask.java | 19 ++++++++++++++-----
.../DataSetRegistrationCleanUpTaskTest.java | 16 +++++++++++-----
2 files changed, 25 insertions(+), 10 deletions(-)
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/DataSetRegistrationCleanUpTask.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/DataSetRegistrationCleanUpTask.java
index a71e7e2d416..da97d23a103 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/DataSetRegistrationCleanUpTask.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/DataSetRegistrationCleanUpTask.java
@@ -95,6 +95,8 @@ public class DataSetRegistrationCleanUpTask implements IMaintenanceTask
private SimpleDateFormat dateFormat;
+ private int minimumAgeInDays;
+
public DataSetRegistrationCleanUpTask()
{
}
@@ -111,7 +113,8 @@ public class DataSetRegistrationCleanUpTask implements IMaintenanceTask
@Override
public void setUp(String pluginName, Properties properties)
{
- minimumAge = PropertyUtils.getInt(properties, MINIMUM_AGE_IN_DAYS, DEFAULT_MINIMUM_AGE_IN_DAYS) * DateUtils.MILLIS_PER_DAY;
+ minimumAgeInDays = PropertyUtils.getInt(properties, MINIMUM_AGE_IN_DAYS, DEFAULT_MINIMUM_AGE_IN_DAYS);
+ minimumAge = minimumAgeInDays * DateUtils.MILLIS_PER_DAY;
preStagingDirectories = new MonitoredDirectories("pre-staging");
stagingDirectories = new MonitoredDirectories("staging");
preCommitDirectories = new MonitoredDirectories("pre-commit");
@@ -147,8 +150,8 @@ public class DataSetRegistrationCleanUpTask implements IMaintenanceTask
File[] files = dir.listFiles(filter);
if (files != null && files.length > 0)
{
- operationLog.info(dirs.getName() + " directory " + dir.getAbsolutePath()
- + " has " + files.length + " files");
+ operationLog.info(dirs.getName() + " directory " + dir.getAbsolutePath() + " has "
+ + files.length + " files which are older than " + minimumAgeInDays + " days:");
for (File file : files)
{
operationLog.info(file.getName() + " (last modified: "
@@ -168,6 +171,10 @@ public class DataSetRegistrationCleanUpTask implements IMaintenanceTask
for (File share : shares)
{
Path shareCore = new File(share, instanceHome).toPath();
+ if (Files.isDirectory(shareCore) == false)
+ {
+ continue;
+ }
int shareCorePathLevels = shareCore.getNameCount();
EmptyFoldersGatherer gatherer = new EmptyFoldersGatherer(shareCorePathLevels, fileFilter);
try (Stream<Path> paths = Files.walk(shareCore, SHARDING_LEVELS + 2))
@@ -191,7 +198,8 @@ public class DataSetRegistrationCleanUpTask implements IMaintenanceTask
{
return;
}
- operationLog.info(emptyDataSets.size() + " old empty data set folders found in share " + shareId + ": " + emptyDataSets);
+ operationLog.info(emptyDataSets.size() + " empty data set folders found in share " + shareId
+ + " which are older than " + minimumAgeInDays + " days: " + emptyDataSets);
List<String> dataSetCodes = emptyDataSets.stream().map(path -> path.getFileName().toString())
.collect(Collectors.toList());
List<AbstractExternalData> dataSets = getOpenBisService().listDataSetsByCode(dataSetCodes);
@@ -211,7 +219,8 @@ public class DataSetRegistrationCleanUpTask implements IMaintenanceTask
{
return;
}
- operationLog.info(emptyFolders.size() + " empty sharding folders found in share " + shareId + ": " + emptyFolders);
+ operationLog.info(emptyFolders.size() + " empty sharding folders found in share " + shareId
+ + " which are older than " + minimumAgeInDays + " days: " + emptyFolders);
for (Path path : emptyFolders)
{
deleteFolder(path, "Empty sharding folder");
diff --git a/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/server/dss/plugins/DataSetRegistrationCleanUpTaskTest.java b/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/server/dss/plugins/DataSetRegistrationCleanUpTaskTest.java
index 241ec3ed0a4..d0e2d5124ac 100644
--- a/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/server/dss/plugins/DataSetRegistrationCleanUpTaskTest.java
+++ b/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/server/dss/plugins/DataSetRegistrationCleanUpTaskTest.java
@@ -102,6 +102,7 @@ public class DataSetRegistrationCleanUpTaskTest extends AbstractFileSystemTestCa
File incoming2 = new File(storeRoot, "2/in");
incoming2.mkdirs();
new File(incoming2.getParent(), databaseInstance.getUuid()).mkdirs();
+ new File(storeRoot, "9").mkdirs();
thread2Properties.setProperty(ThreadParameters.INCOMING_DIR, incoming2.getAbsolutePath());
threadParameters = new ThreadParameters[] {
new ThreadParameters(thread1Properties, "t1"),
@@ -140,7 +141,8 @@ public class DataSetRegistrationCleanUpTaskTest extends AbstractFileSystemTestCa
task.execute();
// Then
- assertEquals("pre-staging directory " + storeRoot.getAbsolutePath() + "/1/pre-staging has 1 files\n"
+ assertEquals("pre-staging directory " + storeRoot.getAbsolutePath()
+ + "/1/pre-staging has 1 files which are older than 70 days:\n"
+ "1970-01-01_01-00-00-000_test_test (last modified: 1970-01-01 01:00:50)\n"
+ "Deleting directory '" + storeRoot + "/1/pre-staging/1970-01-01_01-00-00-000_test_test'\n"
+ "Stale folder deleted: " + storeRoot.getAbsolutePath()
@@ -174,7 +176,8 @@ public class DataSetRegistrationCleanUpTaskTest extends AbstractFileSystemTestCa
task.execute();
// Then
- assertEquals("staging directory " + storeRoot.getAbsolutePath() + "/1/staging has 1 files\n"
+ assertEquals("staging directory " + storeRoot.getAbsolutePath()
+ + "/1/staging has 1 files which are older than 70 days:\n"
+ "19700101010000000-0 (last modified: 1970-01-01 01:00:00)\n"
+ "Deleting directory '" + storeRoot + "/1/staging/19700101010000000-0'\n"
+ "Stale folder deleted: " + storeRoot.getAbsolutePath()
@@ -215,7 +218,8 @@ public class DataSetRegistrationCleanUpTaskTest extends AbstractFileSystemTestCa
task.execute();
// Then
- assertEquals("pre-commit directory " + storeRoot.getAbsolutePath() + "/1/pre-commit has 1 files\n"
+ assertEquals("pre-commit directory " + storeRoot.getAbsolutePath()
+ + "/1/pre-commit has 1 files which are older than 70 days:\n"
+ "19700101010000000-0 (last modified: 1970-01-01 01:00:00)\n"
+ "Deleting directory '" + storeRoot + "/1/pre-commit/19700101010000000-0'\n"
+ "Stale folder deleted: " + storeRoot.getAbsolutePath()
@@ -251,11 +255,13 @@ public class DataSetRegistrationCleanUpTaskTest extends AbstractFileSystemTestCa
task.execute();
// Then
- assertEquals("2 old empty data set folders found in share 3: [" + dir5 + ", " + dir7 + "]\n"
+ assertEquals("2 empty data set folders found in share 3 which are older than 30 days: ["
+ + dir5 + ", " + dir7 + "]\n"
+ "Deleting directory '" + dir5 + "'\n"
+ "Stale data set folder deleted: " + dir5.getAbsolutePath()
+ " (last modified: 1970-01-01T00:00:00Z)\n"
- + "3 empty sharding folders found in share 3: [" + dir1 + ", " + dir4 + ", " + dir3 + "]\n"
+ + "3 empty sharding folders found in share 3 which are older than 30 days: ["
+ + dir1 + ", " + dir4 + ", " + dir3 + "]\n"
+ "Deleting directory '" + dir1 + "'\n"
+ "Empty sharding folder deleted: " + dir1.getAbsolutePath()
+ " (last modified: 1970-01-01T00:00:00Z)\n"
--
GitLab