From f3d90bc711d0fe23520de753a4f9979db8c46450 Mon Sep 17 00:00:00 2001
From: pkupczyk <pkupczyk>
Date: Fri, 14 Mar 2014 15:37:44 +0000
Subject: [PATCH] SP-1240 / BIS-677 : Data Set Size Filling Maintenance Task -
detect that all data sets have been fixed and log appropriate message
SVN: 31147
---
...penbisDBFromPathInfoDBMaintenanceTask.java | 120 +++++++++++-------
...isDBFromPathInfoDBMaintenanceTaskTest.java | 119 ++++++++++++++++-
2 files changed, 192 insertions(+), 47 deletions(-)
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.java
index 76a8acf9607..e64c141db0d 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.java
@@ -17,6 +17,7 @@
package ch.systemsx.cisd.etlserver.plugins;
import java.io.File;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -91,9 +92,9 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask impl
public FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask()
{
service = ServiceProvider.getOpenBISService();
- dao = createDAO();
+ dao = QueryTool.getQuery(PathInfoDataSourceProvider.getDataSource(), IPathsInfoDAO.class);
timeProvider = SystemTimeProvider.SYSTEM_TIME_PROVIDER;
- configProvider = createConfigProvider();
+ configProvider = ServiceProvider.getConfigProvider();
}
public FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask(IEncapsulatedOpenBISService service, IPathsInfoDAO dao,
@@ -129,86 +130,91 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask impl
StringBuilder logBuilder = new StringBuilder();
- logBuilder.append(pluginName + " initialized with\n");
- logBuilder.append(CHUNK_SIZE_PROPERTY + ": " + chunkSize + "\n");
- logBuilder.append(TIME_LIMIT_PROPERTY + ": " + DateTimeUtils.renderDuration(timeLimit) + "\n");
- logBuilder.append(LAST_SEEN_DATA_SET_FILE_PROPERTY + ": " + lastSeenDataSetFile.getAbsolutePath() + "\n");
- logBuilder.append(DROP_LAST_SEEN_DATA_SET_FILE_INTERVAL_PROPERTY + ": " + DateTimeUtils.renderDuration(dropLastSeenDataSetFileInterval));
+ logBuilder.append("Maintenance task '" + pluginName + "' was initialized with the following property values:\n");
+ logBuilder.append("\t" + CHUNK_SIZE_PROPERTY + ": " + chunkSize + "\n");
+ logBuilder.append("\t" + TIME_LIMIT_PROPERTY + ": " + DateTimeUtils.renderDuration(timeLimit) + "\n");
+ logBuilder.append("\t" + LAST_SEEN_DATA_SET_FILE_PROPERTY + ": " + lastSeenDataSetFile.getAbsolutePath() + "\n");
+ logBuilder.append("\t" + DROP_LAST_SEEN_DATA_SET_FILE_INTERVAL_PROPERTY + ": "
+ + DateTimeUtils.renderDuration(dropLastSeenDataSetFileInterval));
operationLog.info(logBuilder.toString());
}
- @SuppressWarnings("null")
@Override
public void execute()
{
operationLog.info("Start filling.");
List<SimpleDataSetInformationDTO> dataSets = null;
+ boolean fixedAllDataSets = true;
long startTime = timeProvider.getTimeInMilliseconds();
- boolean foundDataSets = false;
boolean reachedTimeLimit = false;
int chunkIndex = 1;
prepareLastSeenDataSetFile();
+ LastSeenDataSetFileContent initialLastSeenContent = LastSeenDataSetFileContent.readFromFile(lastSeenDataSetFile);
+
do
{
- LastSeenDataSetFileContent lastSeenContent = LastSeenDataSetFileContent.readFromFile(lastSeenDataSetFile);
+ LastSeenDataSetFileContent currentLastSeenContent = LastSeenDataSetFileContent.readFromFile(lastSeenDataSetFile);
- dataSets = service.listPhysicalDataSetsWithUnknownSize(chunkSize, lastSeenContent.getLastSeenDataSetCode());
- foundDataSets = dataSets != null && false == dataSets.isEmpty();
+ dataSets = findDataSetsWithUnknownSizeInOpenbisDB(currentLastSeenContent.getLastSeenDataSetCode());
- operationLog.info("Last seen data set code: " + lastSeenContent.getLastSeenDataSetCode() + " (chunk: " + chunkIndex + ").");
+ logWithChunkInfo("Found " + dataSets.size() + " dataset(s) with unknown size in openbis database", chunkIndex,
+ currentLastSeenContent.getLastSeenDataSetCode());
- if (foundDataSets)
+ if (false == dataSets.isEmpty())
{
- operationLog.info("Found " + dataSets.size() + " dataset(s) with unknown size in openbis database (chunk: " + chunkIndex + ").");
-
Set<String> codes = new HashSet<String>();
+ Set<String> fixedCodes = new HashSet<String>();
for (SimpleDataSetInformationDTO dataSet : dataSets)
{
codes.add(dataSet.getDataSetCode());
}
- List<PathEntryDTO> pathInfoEntries = dao.listDataSetsSize(codes.toArray(new String[codes.size()]));
+ Map<String, Long> sizeMap = findDataSetsSizeInPathInfoDB(codes);
- if (pathInfoEntries != null && false == pathInfoEntries.isEmpty())
+ logWithChunkInfo("Found " + sizeMap.size() + " size(s) in pathinfo database", chunkIndex,
+ currentLastSeenContent.getLastSeenDataSetCode());
+
+ if (false == sizeMap.isEmpty())
{
- Map<String, Long> sizeMap = new HashMap<String, Long>();
-
- for (PathEntryDTO pathInfoEntry : pathInfoEntries)
- {
- if (pathInfoEntry.getSizeInBytes() != null)
- {
- sizeMap.put(pathInfoEntry.getDataSetCode(), pathInfoEntry.getSizeInBytes());
- }
- }
-
- operationLog.info("Found sizes for " + sizeMap.size() + " dataset(s) in pathinfo database (chunk: " + chunkIndex + ").");
-
- if (false == sizeMap.isEmpty())
- {
- service.updatePhysicalDataSetsSize(sizeMap);
- }
+ service.updatePhysicalDataSetsSize(sizeMap);
+ fixedCodes.addAll(sizeMap.keySet());
}
+ fixedAllDataSets = fixedAllDataSets && codes.equals(fixedCodes);
+
LastSeenDataSetFileContent newLastSeenContent = new LastSeenDataSetFileContent();
- newLastSeenContent.setFileCreationTime(lastSeenContent.getFileCreationTime());
+ newLastSeenContent.setFileCreationTime(currentLastSeenContent.getFileCreationTime());
newLastSeenContent.setLastSeenDataSetCode(Collections.max(codes));
newLastSeenContent.writeToFile(lastSeenDataSetFile);
+ }
- } else
+ reachedTimeLimit = timeProvider.getTimeInMilliseconds() > startTime + timeLimit;
+
+ if (reachedTimeLimit)
{
- operationLog.info("Did not find any datasets with unknown size in openbis database (chunk: " + chunkIndex + ").");
+ logWithChunkInfo("Reached time limit of " + DateTimeUtils.renderDuration(timeLimit) + ".", chunkIndex,
+ currentLastSeenContent.getLastSeenDataSetCode());
}
- reachedTimeLimit = timeProvider.getTimeInMilliseconds() > startTime + timeLimit;
chunkIndex++;
- } while (foundDataSets && false == reachedTimeLimit);
+ } while (false == dataSets.isEmpty() && false == reachedTimeLimit);
+
+ if (initialLastSeenContent.getLastSeenDataSetCode() == null && fixedAllDataSets && false == reachedTimeLimit)
+ {
+ operationLog
+ .info("All data sets with unknown size in openbis database have been fixed. The maintenance task can be now disabled.");
+ } else
+ {
+ operationLog
+ .info("Some data sets with unknown size in openbis database have not been fixed yet. Do not disable the maintenance task yet.");
+ }
operationLog.info("Filling finished.");
}
@@ -240,14 +246,41 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask impl
}
}
- private static IPathsInfoDAO createDAO()
+ private List<SimpleDataSetInformationDTO> findDataSetsWithUnknownSizeInOpenbisDB(String lastSeenDataSetCode)
{
- return QueryTool.getQuery(PathInfoDataSourceProvider.getDataSource(), IPathsInfoDAO.class);
+ List<SimpleDataSetInformationDTO> dataSets = service.listPhysicalDataSetsWithUnknownSize(chunkSize, lastSeenDataSetCode);
+
+ if (dataSets == null)
+ {
+ return Collections.emptyList();
+ } else
+ {
+ return dataSets;
+ }
}
- private IConfigProvider createConfigProvider()
+ private Map<String, Long> findDataSetsSizeInPathInfoDB(Collection<String> dataSetCodes)
{
- return ServiceProvider.getConfigProvider();
+ List<PathEntryDTO> entries = dao.listDataSetsSize(dataSetCodes.toArray(new String[dataSetCodes.size()]));
+ Map<String, Long> map = new HashMap<String, Long>();
+
+ if (entries != null && false == entries.isEmpty())
+ {
+ for (PathEntryDTO pathInfoEntry : entries)
+ {
+ if (pathInfoEntry.getSizeInBytes() != null)
+ {
+ map.put(pathInfoEntry.getDataSetCode(), pathInfoEntry.getSizeInBytes());
+ }
+ }
+ }
+
+ return map;
+ }
+
+ private void logWithChunkInfo(String msg, int chunkIndex, String lastSeenDataSetCode)
+ {
+ operationLog.info(msg + " (chunkIndex: " + chunkIndex + ", lastSeenDataSetCode: " + lastSeenDataSetCode + ").");
}
public static class LastSeenDataSetFileContent
@@ -359,7 +392,6 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask impl
throw new IllegalArgumentException("Could not write the last seen data set file", e);
}
}
-
}
}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest.java
index 6c3a7a8fbe1..416e728fa36 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest.java
@@ -19,17 +19,19 @@ package ch.systemsx.cisd.etlserver.plugins;
import static ch.systemsx.cisd.common.test.ArrayContainsExactlyMatcher.containsExactly;
import static ch.systemsx.cisd.etlserver.plugins.FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.CHUNK_SIZE_DEFAULT;
import static ch.systemsx.cisd.etlserver.plugins.FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.CHUNK_SIZE_PROPERTY;
-import static ch.systemsx.cisd.etlserver.plugins.FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.TIME_LIMIT_PROPERTY;
-import static ch.systemsx.cisd.etlserver.plugins.FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.LAST_SEEN_DATA_SET_FILE_PROPERTY;
import static ch.systemsx.cisd.etlserver.plugins.FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.DROP_LAST_SEEN_DATA_SET_FILE_INTERVAL_PROPERTY;
+import static ch.systemsx.cisd.etlserver.plugins.FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.LAST_SEEN_DATA_SET_FILE_PROPERTY;
+import static ch.systemsx.cisd.etlserver.plugins.FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.TIME_LIMIT_PROPERTY;
import java.io.File;
+import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.Collections;
import java.util.Map;
import java.util.Properties;
import org.apache.commons.collections.map.HashedMap;
+import org.apache.log4j.Level;
import org.jmock.Expectations;
import org.jmock.Mockery;
import org.testng.Assert;
@@ -38,6 +40,9 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import ch.systemsx.cisd.common.filesystem.FileUtilities;
+import ch.systemsx.cisd.common.logging.BufferedAppender;
+import ch.systemsx.cisd.common.logging.LogInitializer;
+import ch.systemsx.cisd.common.test.AssertionUtil;
import ch.systemsx.cisd.common.utilities.ITimeProvider;
import ch.systemsx.cisd.common.utilities.TestResources;
import ch.systemsx.cisd.etlserver.path.IPathsInfoDAO;
@@ -73,9 +78,14 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
private PathEntryDTO entry2;
+ private BufferedAppender logRecorder;
+
@BeforeMethod
public void beforeMethod()
{
+ LogInitializer.init();
+ logRecorder = new BufferedAppender("%-5p %c - %m%n", Level.DEBUG);
+
context = new Mockery();
service = context.mock(IEncapsulatedOpenBISService.class);
dao = context.mock(IPathsInfoDAO.class);
@@ -115,8 +125,13 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
}
@AfterMethod
- public void afterMethod()
+ public void afterMethod(Method method)
{
+ System.out.println("======= Log content for " + method.getName() + "():");
+ System.out.println(logRecorder.getLogContent());
+ System.out.println("=======");
+ logRecorder.reset();
+
context.assertIsSatisfied();
}
@@ -135,6 +150,8 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
});
execute(null, null, null, null);
+
+ assertLogThatAllDataSetsHaveBeenFixed();
}
@Test
@@ -152,6 +169,8 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
});
execute(null, null, null, null);
+
+ assertLogThatAllDataSetsHaveBeenFixed();
}
@Test
@@ -175,6 +194,8 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
});
execute(null, null, null, null);
+
+ assertLogThatSomeDataSetsHaveNotBeenFixedYet();
}
@Test
@@ -198,6 +219,8 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
});
execute(null, null, null, null);
+
+ assertLogThatSomeDataSetsHaveNotBeenFixedYet();
}
@Test
@@ -228,6 +251,8 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
});
execute(null, null, null, null);
+
+ assertLogThatSomeDataSetsHaveNotBeenFixedYet();
}
@Test
@@ -265,6 +290,8 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
});
execute(null, chunkSize, null, null);
+
+ assertLogThatSomeDataSetsHaveNotBeenFixedYet();
}
@Test
@@ -308,6 +335,44 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
});
execute(timeLimit, chunkSize, null, null);
+
+ assertLogThatSomeDataSetsHaveNotBeenFixedYet();
+ }
+
+ @Test
+ public void testExecuteWhenAllDataSetsGetFixedButTimeLimitIsReached()
+ {
+ final long timeLimit = 10L;
+ final int chunkSize = 1;
+
+ context.checking(new Expectations()
+ {
+ {
+ one(timeProvider).getTimeInMilliseconds();
+ will(returnValue(0L));
+
+ one(timeProvider).getTimeInMilliseconds();
+ will(returnValue(0L));
+
+ one(service).listPhysicalDataSetsWithUnknownSize(chunkSize, null);
+ will(returnValue(Arrays.asList(dataSet1)));
+
+ one(dao).listDataSetsSize(new String[] { dataSet1.getDataSetCode() });
+ will(returnValue(Arrays.asList(entry1)));
+
+ Map<String, Long> sizeMap = new HashedMap<String, Long>();
+ sizeMap.put(dataSet1.getDataSetCode(), entry1.getSizeInBytes());
+
+ one(service).updatePhysicalDataSetsSize(sizeMap);
+
+ one(timeProvider).getTimeInMilliseconds();
+ will(returnValue(12L));
+ }
+ });
+
+ execute(timeLimit, chunkSize, null, null);
+
+ assertLogThatSomeDataSetsHaveNotBeenFixedYet();
}
@Test
@@ -346,6 +411,8 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
lastSeenContent = LastSeenDataSetFileContent.readFromFile(lastSeenFile);
Assert.assertEquals(lastSeenContent.getFileCreationTime(), Long.valueOf(lastSeenCreationTime));
Assert.assertEquals(lastSeenContent.getLastSeenDataSetCode(), dataSet3.getDataSetCode());
+
+ assertLogThatSomeDataSetsHaveNotBeenFixedYet();
}
@Test
@@ -389,6 +456,8 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
lastSeenContent = LastSeenDataSetFileContent.readFromFile(lastSeenFile);
Assert.assertEquals(lastSeenContent.getFileCreationTime(), Long.valueOf(lastSeenCreationTime + 2 * dropLastSeenFileInterval));
Assert.assertEquals(lastSeenContent.getLastSeenDataSetCode(), dataSet3.getDataSetCode());
+
+ assertLogThatSomeDataSetsHaveNotBeenFixedYet();
}
@Test(expectedExceptions = { IllegalArgumentException.class }, expectedExceptionsMessageRegExp = "Could not read the last seen data set file")
@@ -409,6 +478,36 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
execute(null, null, lastSeenFile, null);
}
+ @Test
+ public void testExecuteWhenAllDataSetsGetFixed()
+ {
+ context.checking(new Expectations()
+ {
+ {
+ allowing(timeProvider).getTimeInMilliseconds();
+ will(returnValue(0L));
+
+ one(service).listPhysicalDataSetsWithUnknownSize(CHUNK_SIZE_DEFAULT, null);
+ will(returnValue(Arrays.asList(dataSet1)));
+
+ one(dao).listDataSetsSize(new String[] { dataSet1.getDataSetCode() });
+ will(returnValue(Arrays.asList(entry1)));
+
+ Map<String, Long> sizeMap = new HashedMap<String, Long>();
+ sizeMap.put(dataSet1.getDataSetCode(), entry1.getSizeInBytes());
+
+ one(service).updatePhysicalDataSetsSize(sizeMap);
+
+ one(service).listPhysicalDataSetsWithUnknownSize(CHUNK_SIZE_DEFAULT, dataSet1.getDataSetCode());
+ will(returnValue(Collections.emptyList()));
+ }
+ });
+
+ execute(null, null, null, null);
+
+ assertLogThatAllDataSetsHaveBeenFixed();
+ }
+
private void execute(Long timeLimit, Integer chunkSize, File lastSeenFile, Long dropLastSeenFileInterval)
{
FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask task =
@@ -442,4 +541,18 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
return new File(System.getProperty("java.io.tmpdir") + File.separator + getClass().getName() + "Store");
}
+ private void assertLogThatAllDataSetsHaveBeenFixed()
+ {
+ AssertionUtil.assertContainsLines("INFO OPERATION.FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask - "
+ + "All data sets with unknown size in openbis database have been fixed. The maintenance task can be now disabled.",
+ logRecorder.getLogContent());
+ }
+
+ private void assertLogThatSomeDataSetsHaveNotBeenFixedYet()
+ {
+ AssertionUtil.assertContainsLines("INFO OPERATION.FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask - "
+ + "Some data sets with unknown size in openbis database have not been fixed yet. Do not disable the maintenance task yet.",
+ logRecorder.getLogContent());
+ }
+
}
--
GitLab