Skip to content
Snippets Groups Projects
Commit f3d90bc7 authored by pkupczyk's avatar pkupczyk
Browse files

SP-1240 / BIS-677 : Data Set Size Filling Maintenance Task - detect that all... 

SP-1240 / BIS-677 : Data Set Size Filling Maintenance Task - detect that all data sets have been fixed and log appropriate message

SVN: 31147
parent a875e081
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.java
+ 76
− 44
View file @ f3d90bc7
......@@ -17,6 +17,7 @@
package ch.systemsx.cisd.etlserver.plugins;
import java.io.File;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
......@@ -91,9 +92,9 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask impl
public FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask()
{
service = ServiceProvider.getOpenBISService();
dao = createDAO();
dao = QueryTool.getQuery(PathInfoDataSourceProvider.getDataSource(), IPathsInfoDAO.class);
timeProvider = SystemTimeProvider.SYSTEM_TIME_PROVIDER;
configProvider = createConfigProvider();
configProvider = ServiceProvider.getConfigProvider();
}
public FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask(IEncapsulatedOpenBISService service, IPathsInfoDAO dao,
......@@ -129,86 +130,91 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask impl
StringBuilder logBuilder = new StringBuilder();
logBuilder.append(pluginName + " initialized with\n");
logBuilder.append(CHUNK_SIZE_PROPERTY + ": " + chunkSize + "\n");
logBuilder.append(TIME_LIMIT_PROPERTY + ": " + DateTimeUtils.renderDuration(timeLimit) + "\n");
logBuilder.append(LAST_SEEN_DATA_SET_FILE_PROPERTY + ": " + lastSeenDataSetFile.getAbsolutePath() + "\n");
logBuilder.append(DROP_LAST_SEEN_DATA_SET_FILE_INTERVAL_PROPERTY + ": " + DateTimeUtils.renderDuration(dropLastSeenDataSetFileInterval));
logBuilder.append("Maintenance task '" + pluginName + "' was initialized with the following property values:\n");
logBuilder.append("\t" + CHUNK_SIZE_PROPERTY + ": " + chunkSize + "\n");
logBuilder.append("\t" + TIME_LIMIT_PROPERTY + ": " + DateTimeUtils.renderDuration(timeLimit) + "\n");
logBuilder.append("\t" + LAST_SEEN_DATA_SET_FILE_PROPERTY + ": " + lastSeenDataSetFile.getAbsolutePath() + "\n");
logBuilder.append("\t" + DROP_LAST_SEEN_DATA_SET_FILE_INTERVAL_PROPERTY + ": "
+ DateTimeUtils.renderDuration(dropLastSeenDataSetFileInterval));
operationLog.info(logBuilder.toString());
}
@SuppressWarnings("null")
@Override
public void execute()
{
operationLog.info("Start filling.");
List<SimpleDataSetInformationDTO> dataSets = null;
boolean fixedAllDataSets = true;
long startTime = timeProvider.getTimeInMilliseconds();
boolean foundDataSets = false;
boolean reachedTimeLimit = false;
int chunkIndex = 1;
prepareLastSeenDataSetFile();
LastSeenDataSetFileContent initialLastSeenContent = LastSeenDataSetFileContent.readFromFile(lastSeenDataSetFile);
do
{
LastSeenDataSetFileContent lastSeenContent = LastSeenDataSetFileContent.readFromFile(lastSeenDataSetFile);
LastSeenDataSetFileContent currentLastSeenContent = LastSeenDataSetFileContent.readFromFile(lastSeenDataSetFile);
dataSets = service.listPhysicalDataSetsWithUnknownSize(chunkSize, lastSeenContent.getLastSeenDataSetCode());
foundDataSets = dataSets != null && false == dataSets.isEmpty();
dataSets = findDataSetsWithUnknownSizeInOpenbisDB(currentLastSeenContent.getLastSeenDataSetCode());
operationLog.info("Last seen data set code: " + lastSeenContent.getLastSeenDataSetCode() + " (chunk: " + chunkIndex + ").");
logWithChunkInfo("Found " + dataSets.size() + " dataset(s) with unknown size in openbis database", chunkIndex,
currentLastSeenContent.getLastSeenDataSetCode());
if (foundDataSets)
if (false == dataSets.isEmpty())
{
operationLog.info("Found " + dataSets.size() + " dataset(s) with unknown size in openbis database (chunk: " + chunkIndex + ").");
Set<String> codes = new HashSet<String>();
Set<String> fixedCodes = new HashSet<String>();
for (SimpleDataSetInformationDTO dataSet : dataSets)
{
codes.add(dataSet.getDataSetCode());
}
List<PathEntryDTO> pathInfoEntries = dao.listDataSetsSize(codes.toArray(new String[codes.size()]));
Map<String, Long> sizeMap = findDataSetsSizeInPathInfoDB(codes);
if (pathInfoEntries != null && false == pathInfoEntries.isEmpty())
logWithChunkInfo("Found " + sizeMap.size() + " size(s) in pathinfo database", chunkIndex,
currentLastSeenContent.getLastSeenDataSetCode());
if (false == sizeMap.isEmpty())
{
Map<String, Long> sizeMap = new HashMap<String, Long>();
for (PathEntryDTO pathInfoEntry : pathInfoEntries)
{
if (pathInfoEntry.getSizeInBytes() != null)
{
sizeMap.put(pathInfoEntry.getDataSetCode(), pathInfoEntry.getSizeInBytes());
}
}
operationLog.info("Found sizes for " + sizeMap.size() + " dataset(s) in pathinfo database (chunk: " + chunkIndex + ").");
if (false == sizeMap.isEmpty())
{
service.updatePhysicalDataSetsSize(sizeMap);
}
service.updatePhysicalDataSetsSize(sizeMap);
fixedCodes.addAll(sizeMap.keySet());
}
fixedAllDataSets = fixedAllDataSets && codes.equals(fixedCodes);
LastSeenDataSetFileContent newLastSeenContent = new LastSeenDataSetFileContent();
newLastSeenContent.setFileCreationTime(lastSeenContent.getFileCreationTime());
newLastSeenContent.setFileCreationTime(currentLastSeenContent.getFileCreationTime());
newLastSeenContent.setLastSeenDataSetCode(Collections.max(codes));
newLastSeenContent.writeToFile(lastSeenDataSetFile);
}
} else
reachedTimeLimit = timeProvider.getTimeInMilliseconds() > startTime + timeLimit;
if (reachedTimeLimit)
{
operationLog.info("Did not find any datasets with unknown size in openbis database (chunk: " + chunkIndex + ").");
logWithChunkInfo("Reached time limit of " + DateTimeUtils.renderDuration(timeLimit) + ".", chunkIndex,
currentLastSeenContent.getLastSeenDataSetCode());
}
reachedTimeLimit = timeProvider.getTimeInMilliseconds() > startTime + timeLimit;
chunkIndex++;
} while (foundDataSets && false == reachedTimeLimit);
} while (false == dataSets.isEmpty() && false == reachedTimeLimit);
if (initialLastSeenContent.getLastSeenDataSetCode() == null && fixedAllDataSets && false == reachedTimeLimit)
{
operationLog
.info("All data sets with unknown size in openbis database have been fixed. The maintenance task can be now disabled.");
} else
{
operationLog
.info("Some data sets with unknown size in openbis database have not been fixed yet. Do not disable the maintenance task yet.");
}
operationLog.info("Filling finished.");
}
......@@ -240,14 +246,41 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask impl
}
}
private static IPathsInfoDAO createDAO()
private List<SimpleDataSetInformationDTO> findDataSetsWithUnknownSizeInOpenbisDB(String lastSeenDataSetCode)
{
return QueryTool.getQuery(PathInfoDataSourceProvider.getDataSource(), IPathsInfoDAO.class);
List<SimpleDataSetInformationDTO> dataSets = service.listPhysicalDataSetsWithUnknownSize(chunkSize, lastSeenDataSetCode);
if (dataSets == null)
{
return Collections.emptyList();
} else
{
return dataSets;
}
}
private IConfigProvider createConfigProvider()
private Map<String, Long> findDataSetsSizeInPathInfoDB(Collection<String> dataSetCodes)
{
return ServiceProvider.getConfigProvider();
List<PathEntryDTO> entries = dao.listDataSetsSize(dataSetCodes.toArray(new String[dataSetCodes.size()]));
Map<String, Long> map = new HashMap<String, Long>();
if (entries != null && false == entries.isEmpty())
{
for (PathEntryDTO pathInfoEntry : entries)
{
if (pathInfoEntry.getSizeInBytes() != null)
{
map.put(pathInfoEntry.getDataSetCode(), pathInfoEntry.getSizeInBytes());
}
}
}
return map;
}
private void logWithChunkInfo(String msg, int chunkIndex, String lastSeenDataSetCode)
{
operationLog.info(msg + " (chunkIndex: " + chunkIndex + ", lastSeenDataSetCode: " + lastSeenDataSetCode + ").");
}
public static class LastSeenDataSetFileContent
......@@ -359,7 +392,6 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask impl
throw new IllegalArgumentException("Could not write the last seen data set file", e);
}
}
}
}
datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest.java
+ 116
− 3
View file @ f3d90bc7
......@@ -19,17 +19,19 @@ package ch.systemsx.cisd.etlserver.plugins;
import static ch.systemsx.cisd.common.test.ArrayContainsExactlyMatcher.containsExactly;
import static ch.systemsx.cisd.etlserver.plugins.FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.CHUNK_SIZE_DEFAULT;
import static ch.systemsx.cisd.etlserver.plugins.FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.CHUNK_SIZE_PROPERTY;
import static ch.systemsx.cisd.etlserver.plugins.FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.TIME_LIMIT_PROPERTY;
import static ch.systemsx.cisd.etlserver.plugins.FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.LAST_SEEN_DATA_SET_FILE_PROPERTY;
import static ch.systemsx.cisd.etlserver.plugins.FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.DROP_LAST_SEEN_DATA_SET_FILE_INTERVAL_PROPERTY;
import static ch.systemsx.cisd.etlserver.plugins.FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.LAST_SEEN_DATA_SET_FILE_PROPERTY;
import static ch.systemsx.cisd.etlserver.plugins.FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.TIME_LIMIT_PROPERTY;
import java.io.File;
import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.Collections;
import java.util.Map;
import java.util.Properties;
import org.apache.commons.collections.map.HashedMap;
import org.apache.log4j.Level;
import org.jmock.Expectations;
import org.jmock.Mockery;
import org.testng.Assert;
......@@ -38,6 +40,9 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.common.logging.BufferedAppender;
import ch.systemsx.cisd.common.logging.LogInitializer;
import ch.systemsx.cisd.common.test.AssertionUtil;
import ch.systemsx.cisd.common.utilities.ITimeProvider;
import ch.systemsx.cisd.common.utilities.TestResources;
import ch.systemsx.cisd.etlserver.path.IPathsInfoDAO;
......@@ -73,9 +78,14 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
private PathEntryDTO entry2;
private BufferedAppender logRecorder;
@BeforeMethod
public void beforeMethod()
{
LogInitializer.init();
logRecorder = new BufferedAppender("%-5p %c - %m%n", Level.DEBUG);
context = new Mockery();
service = context.mock(IEncapsulatedOpenBISService.class);
dao = context.mock(IPathsInfoDAO.class);
......@@ -115,8 +125,13 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
}
@AfterMethod
public void afterMethod()
public void afterMethod(Method method)
{
System.out.println("======= Log content for " + method.getName() + "():");
System.out.println(logRecorder.getLogContent());
System.out.println("=======");
logRecorder.reset();
context.assertIsSatisfied();
}
......@@ -135,6 +150,8 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
});
execute(null, null, null, null);
assertLogThatAllDataSetsHaveBeenFixed();
}
@Test
......@@ -152,6 +169,8 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
});
execute(null, null, null, null);
assertLogThatAllDataSetsHaveBeenFixed();
}
@Test
......@@ -175,6 +194,8 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
});
execute(null, null, null, null);
assertLogThatSomeDataSetsHaveNotBeenFixedYet();
}
@Test
......@@ -198,6 +219,8 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
});
execute(null, null, null, null);
assertLogThatSomeDataSetsHaveNotBeenFixedYet();
}
@Test
......@@ -228,6 +251,8 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
});
execute(null, null, null, null);
assertLogThatSomeDataSetsHaveNotBeenFixedYet();
}
@Test
......@@ -265,6 +290,8 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
});
execute(null, chunkSize, null, null);
assertLogThatSomeDataSetsHaveNotBeenFixedYet();
}
@Test
......@@ -308,6 +335,44 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
});
execute(timeLimit, chunkSize, null, null);
assertLogThatSomeDataSetsHaveNotBeenFixedYet();
}
@Test
public void testExecuteWhenAllDataSetsGetFixedButTimeLimitIsReached()
{
final long timeLimit = 10L;
final int chunkSize = 1;
context.checking(new Expectations()
{
{
one(timeProvider).getTimeInMilliseconds();
will(returnValue(0L));
one(timeProvider).getTimeInMilliseconds();
will(returnValue(0L));
one(service).listPhysicalDataSetsWithUnknownSize(chunkSize, null);
will(returnValue(Arrays.asList(dataSet1)));
one(dao).listDataSetsSize(new String[] { dataSet1.getDataSetCode() });
will(returnValue(Arrays.asList(entry1)));
Map<String, Long> sizeMap = new HashedMap<String, Long>();
sizeMap.put(dataSet1.getDataSetCode(), entry1.getSizeInBytes());
one(service).updatePhysicalDataSetsSize(sizeMap);
one(timeProvider).getTimeInMilliseconds();
will(returnValue(12L));
}
});
execute(timeLimit, chunkSize, null, null);
assertLogThatSomeDataSetsHaveNotBeenFixedYet();
}
@Test
......@@ -346,6 +411,8 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
lastSeenContent = LastSeenDataSetFileContent.readFromFile(lastSeenFile);
Assert.assertEquals(lastSeenContent.getFileCreationTime(), Long.valueOf(lastSeenCreationTime));
Assert.assertEquals(lastSeenContent.getLastSeenDataSetCode(), dataSet3.getDataSetCode());
assertLogThatSomeDataSetsHaveNotBeenFixedYet();
}
@Test
......@@ -389,6 +456,8 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
lastSeenContent = LastSeenDataSetFileContent.readFromFile(lastSeenFile);
Assert.assertEquals(lastSeenContent.getFileCreationTime(), Long.valueOf(lastSeenCreationTime + 2 * dropLastSeenFileInterval));
Assert.assertEquals(lastSeenContent.getLastSeenDataSetCode(), dataSet3.getDataSetCode());
assertLogThatSomeDataSetsHaveNotBeenFixedYet();
}
@Test(expectedExceptions = { IllegalArgumentException.class }, expectedExceptionsMessageRegExp = "Could not read the last seen data set file")
......@@ -409,6 +478,36 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
execute(null, null, lastSeenFile, null);
}
@Test
public void testExecuteWhenAllDataSetsGetFixed()
{
context.checking(new Expectations()
{
{
allowing(timeProvider).getTimeInMilliseconds();
will(returnValue(0L));
one(service).listPhysicalDataSetsWithUnknownSize(CHUNK_SIZE_DEFAULT, null);
will(returnValue(Arrays.asList(dataSet1)));
one(dao).listDataSetsSize(new String[] { dataSet1.getDataSetCode() });
will(returnValue(Arrays.asList(entry1)));
Map<String, Long> sizeMap = new HashedMap<String, Long>();
sizeMap.put(dataSet1.getDataSetCode(), entry1.getSizeInBytes());
one(service).updatePhysicalDataSetsSize(sizeMap);
one(service).listPhysicalDataSetsWithUnknownSize(CHUNK_SIZE_DEFAULT, dataSet1.getDataSetCode());
will(returnValue(Collections.emptyList()));
}
});
execute(null, null, null, null);
assertLogThatAllDataSetsHaveBeenFixed();
}
private void execute(Long timeLimit, Integer chunkSize, File lastSeenFile, Long dropLastSeenFileInterval)
{
FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask task =
......@@ -442,4 +541,18 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
return new File(System.getProperty("java.io.tmpdir") + File.separator + getClass().getName() + "Store");
}
private void assertLogThatAllDataSetsHaveBeenFixed()
{
AssertionUtil.assertContainsLines("INFO OPERATION.FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask - "
+ "All data sets with unknown size in openbis database have been fixed. The maintenance task can be now disabled.",
logRecorder.getLogContent());
}
private void assertLogThatSomeDataSetsHaveNotBeenFixedYet()
{
AssertionUtil.assertContainsLines("INFO OPERATION.FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask - "
+ "Some data sets with unknown size in openbis database have not been fixed yet. Do not disable the maintenance task yet.",
logRecorder.getLogContent());
}
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment