Skip to content
Snippets Groups Projects
Commit 63bff999 authored by kaloyane's avatar kaloyane
Browse files

[LMS-2583] 

- make estimated data set sizes configurable,
- change ExperimentBasedArchiver task to work with estimated data set sizes
- we no longer monitor the free space of a share

SVN: 23415
parent 8b525858
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/ExperimentBasedArchivingTask.java
+ 75
− 95
View file @ 63bff999
...@@ -24,8 +24,10 @@ import java.util.Collections; ...@@ -24,8 +24,10 @@ import java.util.Collections;
import java.util.Comparator; import java.util.Comparator;
import java.util.Date; import java.util.Date;
import java.util.EnumSet; import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.Properties; import java.util.Properties;
import java.util.Set; import java.util.Set;
...@@ -37,6 +39,8 @@ import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException; ...@@ -37,6 +39,8 @@ import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
import ch.systemsx.cisd.common.filesystem.HostAwareFile; import ch.systemsx.cisd.common.filesystem.HostAwareFile;
import ch.systemsx.cisd.common.filesystem.IFreeSpaceProvider; import ch.systemsx.cisd.common.filesystem.IFreeSpaceProvider;
import ch.systemsx.cisd.common.filesystem.SimpleFreeSpaceProvider; import ch.systemsx.cisd.common.filesystem.SimpleFreeSpaceProvider;
import ch.systemsx.cisd.common.logging.ISimpleLogger;
import ch.systemsx.cisd.common.logging.Log4jSimpleLogger;
import ch.systemsx.cisd.common.logging.LogCategory; import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory; import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.maintenance.IDataStoreLockingMaintenanceTask; import ch.systemsx.cisd.common.maintenance.IDataStoreLockingMaintenanceTask;
...@@ -45,7 +49,6 @@ import ch.systemsx.cisd.common.utilities.ExtendedProperties; ...@@ -45,7 +49,6 @@ import ch.systemsx.cisd.common.utilities.ExtendedProperties;
import ch.systemsx.cisd.common.utilities.PropertyParametersUtil; import ch.systemsx.cisd.common.utilities.PropertyParametersUtil;
import ch.systemsx.cisd.common.utilities.PropertyUtils; import ch.systemsx.cisd.common.utilities.PropertyUtils;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService; import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
import ch.systemsx.cisd.openbis.dss.generic.shared.IShareIdManager;
import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider; import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSet; import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSet;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetArchivingStatus; import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetArchivingStatus;
...@@ -68,19 +71,21 @@ public class ExperimentBasedArchivingTask implements IDataStoreLockingMaintenanc ...@@ -68,19 +71,21 @@ public class ExperimentBasedArchivingTask implements IDataStoreLockingMaintenanc
private static final Logger notificationLog = LogFactory.getLogger(LogCategory.NOTIFY, private static final Logger notificationLog = LogFactory.getLogger(LogCategory.NOTIFY,
ExperimentBasedArchivingTask.class); ExperimentBasedArchivingTask.class);
private static final long ONE_KB_IN_BYTES = 1024L;
static final String MINIMUM_FREE_SPACE_KEY = "minimum-free-space-in-MB"; static final String MINIMUM_FREE_SPACE_KEY = "minimum-free-space-in-MB";
static final String STOREROOT_DIR_KEY = "storeroot-dir";
static final String MONITORED_SHARE_KEY = "monitored-share";
static final String EXCLUDED_DATA_SET_TYPES_KEY = "excluded-data-set-types"; static final String EXCLUDED_DATA_SET_TYPES_KEY = "excluded-data-set-types";
static final String MONITORED_DIR = "monitored-dir"; static final String MONITORED_DIR = "monitored-dir";
static final String FREE_SPACE_PROVIDER_PREFIX = "free-space-provider."; static final String FREE_SPACE_PROVIDER_PREFIX = "free-space-provider.";
static final String DATA_SET_SIZE_PREFIX = "data-set-size.";
static final String DEFAULT_DATA_SET_TYPE = "DEFAULT";
private static final EnumSet<DataSetArchivingStatus> ARCHIVE_STATES = EnumSet.of( private static final EnumSet<DataSetArchivingStatus> ARCHIVE_STATES = EnumSet.of(
DataSetArchivingStatus.ARCHIVE_PENDING, DataSetArchivingStatus.ARCHIVED); DataSetArchivingStatus.ARCHIVE_PENDING, DataSetArchivingStatus.ARCHIVED);
...@@ -88,27 +93,22 @@ public class ExperimentBasedArchivingTask implements IDataStoreLockingMaintenanc ...@@ -88,27 +93,22 @@ public class ExperimentBasedArchivingTask implements IDataStoreLockingMaintenanc
private IFreeSpaceProvider freeSpaceProvider; private IFreeSpaceProvider freeSpaceProvider;
private File storeRoot;
private File monitoredDirectory; private File monitoredDirectory;
private long minimumFreeSpace; private long minimumFreeSpace;
private String shareIDOrNull;
private final IShareIdManager shareIdManager;
private Set<String> excludedDataSetTypes; private Set<String> excludedDataSetTypes;
private Map<String, Long> estimatedDataSetSizes;
public ExperimentBasedArchivingTask() public ExperimentBasedArchivingTask()
{ {
this(ServiceProvider.getOpenBISService(), ServiceProvider.getShareIdManager()); this(ServiceProvider.getOpenBISService());
} }
ExperimentBasedArchivingTask(IEncapsulatedOpenBISService service, IShareIdManager shareIdManager) ExperimentBasedArchivingTask(IEncapsulatedOpenBISService service)
{ {
this.service = service; this.service = service;
this.shareIdManager = shareIdManager;
} }
public boolean requiresDataStoreLock() public boolean requiresDataStoreLock()
...@@ -118,54 +118,27 @@ public class ExperimentBasedArchivingTask implements IDataStoreLockingMaintenanc ...@@ -118,54 +118,27 @@ public class ExperimentBasedArchivingTask implements IDataStoreLockingMaintenanc
public void setUp(String pluginName, Properties properties) public void setUp(String pluginName, Properties properties)
{ {
storeRoot = setUpStoreRoot(properties);
freeSpaceProvider = setUpFreeSpaceProvider(properties); freeSpaceProvider = setUpFreeSpaceProvider(properties);
setUpMonitoredShareOrPath(properties); monitoredDirectory = setUpMonitoredDirectory(properties);
minimumFreeSpace = minimumFreeSpace =
FileUtils.ONE_MB * PropertyUtils.getLong(properties, MINIMUM_FREE_SPACE_KEY, 1024); FileUtils.ONE_MB * PropertyUtils.getLong(properties, MINIMUM_FREE_SPACE_KEY, 1024);
excludedDataSetTypes = excludedDataSetTypes =
new HashSet<String>(Arrays.asList(PropertyParametersUtil.parseItemisedProperty( new HashSet<String>(Arrays.asList(PropertyParametersUtil.parseItemisedProperty(
properties.getProperty(EXCLUDED_DATA_SET_TYPES_KEY, ""), properties.getProperty(EXCLUDED_DATA_SET_TYPES_KEY, ""),
EXCLUDED_DATA_SET_TYPES_KEY))); EXCLUDED_DATA_SET_TYPES_KEY)));
estimatedDataSetSizes = setUpEstimatedDataSetSizes(properties);
} }
private File setUpStoreRoot(Properties properties) private File setUpMonitoredDirectory(Properties properties)
{
String storeRootFileName =
PropertyUtils.getMandatoryProperty(properties, STOREROOT_DIR_KEY);
File resultStoreRoot = new File(storeRootFileName);
if (resultStoreRoot.isDirectory() == false)
{
throw new ConfigurationFailureException(
"Store root doesn't exists or isn't a directory: " + storeRoot);
}
return resultStoreRoot;
}
private void setUpMonitoredShareOrPath(Properties properties)
{ {
final String monitoredDirPath = PropertyUtils.getProperty(properties, MONITORED_DIR); final String monitoredDirPath = PropertyUtils.getProperty(properties, MONITORED_DIR);
if (monitoredDirPath == null) File monitoredDir = new File(monitoredDirPath);
{ if (monitoredDir.isDirectory() == false)
shareIDOrNull = PropertyUtils.getMandatoryProperty(properties, MONITORED_SHARE_KEY);
monitoredDirectory = new File(storeRoot, shareIDOrNull);
} else
{ {
shareIDOrNull = null; throw new ConfigurationFailureException("Directory '" + monitoredDirPath
monitoredDirectory = new File(monitoredDirPath); + "' doesn't exists or isn't a directory.");
}
if (monitoredDirectory.isDirectory() == false)
{
if (monitorDataStoreShare())
{
throw new ConfigurationFailureException("Share " + shareIDOrNull
+ " doesn't exists or isn't a directory.");
} else
{
throw new ConfigurationFailureException("Directory '" + monitoredDirPath
+ "' doesn't exists or isn't a directory.");
}
} }
return monitoredDir;
} }
private IFreeSpaceProvider setUpFreeSpaceProvider(Properties properties) private IFreeSpaceProvider setUpFreeSpaceProvider(Properties properties)
...@@ -197,6 +170,27 @@ public class ExperimentBasedArchivingTask implements IDataStoreLockingMaintenanc ...@@ -197,6 +170,27 @@ public class ExperimentBasedArchivingTask implements IDataStoreLockingMaintenanc
} }
private Map<String, Long> setUpEstimatedDataSetSizes(Properties properties)
{
ISimpleLogger log = new Log4jSimpleLogger(operationLog);
Properties dataSetSizeProps =
ExtendedProperties.getSubset(properties, DATA_SET_SIZE_PREFIX, true);
Map<String, Long> result = new HashMap<String, Long>();
for (Object key : dataSetSizeProps.keySet())
{
String dataSetType = ((String) key).toUpperCase();
long estimatedSizeInBytes =
ONE_KB_IN_BYTES
* PropertyUtils.getPosLong(dataSetSizeProps, dataSetType, 0L, log);
if (estimatedSizeInBytes > 0)
{
result.put(dataSetType, estimatedSizeInBytes);
}
}
return result;
}
public void execute() public void execute()
{ {
long freeSpace = getFreeSpace(); long freeSpace = getFreeSpace();
...@@ -232,22 +226,13 @@ public class ExperimentBasedArchivingTask implements IDataStoreLockingMaintenanc ...@@ -232,22 +226,13 @@ public class ExperimentBasedArchivingTask implements IDataStoreLockingMaintenanc
StringBuilder archivingMessages = new StringBuilder(); StringBuilder archivingMessages = new StringBuilder();
try try
{ {
if (monitorDataStoreShare()) for (int i = 0; i < infos.size() && freeSpace < minimumFreeSpace; i++)
{
for (int i = 0; i < infos.size() && freeSpace < minimumFreeSpace; i++)
{
ExperimentDataSetsInfo info = infos.get(i);
freeSpace += info.calculateSize();
archive(info, archivingMessages);
}
} else
{ {
for (int i = 0; i < infos.size() && freeSpace < minimumFreeSpace; i++) ExperimentDataSetsInfo info = infos.get(i);
long estimatedSpaceFreed = info.estimateSize();
if (archive(info, archivingMessages))
{ {
if (archive(infos.get(i), archivingMessages)) freeSpace += estimatedSpaceFreed;
{
freeSpace = getFreeSpace();
}
} }
} }
} finally } finally
...@@ -259,16 +244,12 @@ public class ExperimentBasedArchivingTask implements IDataStoreLockingMaintenanc ...@@ -259,16 +244,12 @@ public class ExperimentBasedArchivingTask implements IDataStoreLockingMaintenanc
} }
} }
private boolean monitorDataStoreShare()
{
return shareIDOrNull != null;
}
private long getFreeSpace() private long getFreeSpace()
{ {
try try
{ {
return 1024L * freeSpaceProvider.freeSpaceKb(new HostAwareFile(monitoredDirectory)); return ONE_KB_IN_BYTES
* freeSpaceProvider.freeSpaceKb(new HostAwareFile(monitoredDirectory));
} catch (IOException ex) } catch (IOException ex)
{ {
throw CheckedExceptionTunnel.wrapIfNecessary(ex); throw CheckedExceptionTunnel.wrapIfNecessary(ex);
...@@ -314,11 +295,6 @@ public class ExperimentBasedArchivingTask implements IDataStoreLockingMaintenanc ...@@ -314,11 +295,6 @@ public class ExperimentBasedArchivingTask implements IDataStoreLockingMaintenanc
continue; continue;
} }
DataSet realDataSet = (DataSet) dataSet; DataSet realDataSet = (DataSet) dataSet;
if (shareIDOrNull != null
&& realDataSet.getShareId().equals(shareIDOrNull) == false)
{
continue;
}
if (excludedDataSetTypes.contains(realDataSet.getDataSetType().getCode())) if (excludedDataSetTypes.contains(realDataSet.getDataSetType().getCode()))
{ {
continue; continue;
...@@ -345,32 +321,37 @@ public class ExperimentBasedArchivingTask implements IDataStoreLockingMaintenanc ...@@ -345,32 +321,37 @@ public class ExperimentBasedArchivingTask implements IDataStoreLockingMaintenanc
} }
} }
long calculateSize() public long estimateSize()
{ {
long sum = 0; long sum = 0L;
for (DataSet dataSet : dataSetsToBeArchived) for (DataSet dataSetToBeArchived : getDataSetsToBeArchived())
{ {
Long size = dataSet.getSize(); sum += estimateSize(dataSetToBeArchived);
if (size == null)
{
String dataSetCode = dataSet.getCode();
shareIdManager.lock(dataSetCode);
try
{
File shareRoot =
new File(storeRoot, shareIdManager.getShareId(dataSetCode));
String location = dataSet.getLocation();
size = FileUtils.sizeOfDirectory(new File(shareRoot, location));
} finally
{
shareIdManager.releaseLock(dataSetCode);
}
}
sum += size;
} }
return sum; return sum;
} }
private long estimateSize(DataSet dataSet)
{
String dataSetType = dataSet.getDataSetType().getCode().toUpperCase();
Long estimatedDataSetSize = estimatedDataSetSizes.get(dataSetType);
if (estimatedDataSetSize == null)
{
estimatedDataSetSize = estimatedDataSetSizes.get(DEFAULT_DATA_SET_TYPE);
}
if (estimatedDataSetSize == null)
{
throw ConfigurationFailureException
.fromTemplate(
"Failed to estimate the average size for data set type '%s'. "
+ "Please, configure the maintenance task with a '%s' or a '%s' property. ",
dataSetType, DATA_SET_SIZE_PREFIX + dataSetType,
DATA_SET_SIZE_PREFIX + DEFAULT_DATA_SET_TYPE);
}
return estimatedDataSetSize.longValue();
}
public String getExperimentIdentifier() public String getExperimentIdentifier()
{ {
return experimentIdentifier; return experimentIdentifier;
...@@ -409,5 +390,4 @@ public class ExperimentBasedArchivingTask implements IDataStoreLockingMaintenanc ...@@ -409,5 +390,4 @@ public class ExperimentBasedArchivingTask implements IDataStoreLockingMaintenanc
return 0; return 0;
} }
} }
} }
datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/ExperimentBasedArchivingTaskTest.java
+ 57
− 112
View file @ 63bff999
...@@ -24,7 +24,6 @@ import java.util.Date; ...@@ -24,7 +24,6 @@ import java.util.Date;
import java.util.List; import java.util.List;
import java.util.Properties; import java.util.Properties;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.Level; import org.apache.log4j.Level;
import org.jmock.Expectations; import org.jmock.Expectations;
import org.jmock.Mockery; import org.jmock.Mockery;
...@@ -39,7 +38,6 @@ import ch.systemsx.cisd.common.filesystem.IFreeSpaceProvider; ...@@ -39,7 +38,6 @@ import ch.systemsx.cisd.common.filesystem.IFreeSpaceProvider;
import ch.systemsx.cisd.common.logging.BufferedAppender; import ch.systemsx.cisd.common.logging.BufferedAppender;
import ch.systemsx.cisd.common.logging.LogCategory; import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService; import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
import ch.systemsx.cisd.openbis.dss.generic.shared.IShareIdManager;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSet; import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSet;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetArchivingStatus; import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetArchivingStatus;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Experiment; import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Experiment;
...@@ -82,8 +80,6 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase ...@@ -82,8 +80,6 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase
private static final String LOCATION_PREFIX = "abc/"; private static final String LOCATION_PREFIX = "abc/";
private static final String SHARE_ID = "3";
private BufferedAppender logRecorder; private BufferedAppender logRecorder;
private Mockery context; private Mockery context;
...@@ -92,14 +88,12 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase ...@@ -92,14 +88,12 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase
private IFreeSpaceProvider freeSpaceProvider; private IFreeSpaceProvider freeSpaceProvider;
private IShareIdManager shareIdManager;
private ExperimentBasedArchivingTask task; private ExperimentBasedArchivingTask task;
private File share;
private Properties properties; private Properties properties;
private File monitoredDir;
private Experiment e1; private Experiment e1;
private Experiment e2; private Experiment e2;
...@@ -110,8 +104,6 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase ...@@ -110,8 +104,6 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase
private ExternalData notARealDataSet; private ExternalData notARealDataSet;
private DataSet dataSetInAShareToBeIgnored;
private DataSet dataSetOfIgnoredType; private DataSet dataSetOfIgnoredType;
private DataSet dataSetWithNoModificationDate; private DataSet dataSetWithNoModificationDate;
...@@ -131,10 +123,9 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase ...@@ -131,10 +123,9 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase
context = new Mockery(); context = new Mockery();
service = context.mock(IEncapsulatedOpenBISService.class); service = context.mock(IEncapsulatedOpenBISService.class);
freeSpaceProvider = context.mock(IFreeSpaceProvider.class); freeSpaceProvider = context.mock(IFreeSpaceProvider.class);
shareIdManager = context.mock(IShareIdManager.class);
MockFreeSpaceProvider.mock = freeSpaceProvider; MockFreeSpaceProvider.mock = freeSpaceProvider;
task = new ExperimentBasedArchivingTask(service, shareIdManager); task = new ExperimentBasedArchivingTask(service);
assertEquals(true, task.requiresDataStoreLock()); assertEquals(true, task.requiresDataStoreLock());
e1 = new ExperimentBuilder().id(41).identifier("/S/P/E1").getExperiment(); e1 = new ExperimentBuilder().id(41).identifier("/S/P/E1").getExperiment();
...@@ -144,9 +135,6 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase ...@@ -144,9 +135,6 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase
lockedDataSet = lockedDataSet =
dataSet("lockedDataSet").modificationDate(new Date(100)) dataSet("lockedDataSet").modificationDate(new Date(100))
.status(DataSetArchivingStatus.LOCKED).getDataSet(); .status(DataSetArchivingStatus.LOCKED).getDataSet();
dataSetInAShareToBeIgnored =
dataSet("dataSetInAShareToBeIgnored").modificationDate(new Date(120)).shareID("42")
.getDataSet();
dataSetOfIgnoredType = dataSet("dataSetOfIgnoredType").type("ABC").getDataSet(); dataSetOfIgnoredType = dataSet("dataSetOfIgnoredType").type("ABC").getDataSet();
dataSetWithNoModificationDate = dataSet("dataSetWithNoModificationDate").getDataSet(); dataSetWithNoModificationDate = dataSet("dataSetWithNoModificationDate").getDataSet();
oldDataSet = dataSet("oldDataSet").modificationDate(new Date(300)).getDataSet(); oldDataSet = dataSet("oldDataSet").modificationDate(new Date(300)).getDataSet();
...@@ -155,44 +143,24 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase ...@@ -155,44 +143,24 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase
veryYoungDataSet = veryYoungDataSet =
dataSet("veryYoungDataSet").modificationDate(new Date(2000)).getDataSet(); dataSet("veryYoungDataSet").modificationDate(new Date(2000)).getDataSet();
share = new File(workingDirectory, "store/" + SHARE_ID);
share.mkdirs();
properties = new Properties(); properties = new Properties();
properties.setProperty(ExperimentBasedArchivingTask.STOREROOT_DIR_KEY, share.getParent());
properties.setProperty(ExperimentBasedArchivingTask.MONITORED_SHARE_KEY, SHARE_ID);
properties.setProperty(ExperimentBasedArchivingTask.MINIMUM_FREE_SPACE_KEY, "100"); properties.setProperty(ExperimentBasedArchivingTask.MINIMUM_FREE_SPACE_KEY, "100");
properties.setProperty(ExperimentBasedArchivingTask.EXCLUDED_DATA_SET_TYPES_KEY, "ABC, B"); properties.setProperty(ExperimentBasedArchivingTask.EXCLUDED_DATA_SET_TYPES_KEY, "ABC, B");
monitoredDir = new File(workingDirectory, "/some/dir");
monitoredDir.mkdirs();
properties.setProperty(ExperimentBasedArchivingTask.MONITORED_DIR,
monitoredDir.getAbsolutePath());
properties.setProperty(ExperimentBasedArchivingTask.FREE_SPACE_PROVIDER_PREFIX + "class", properties.setProperty(ExperimentBasedArchivingTask.FREE_SPACE_PROVIDER_PREFIX + "class",
MockFreeSpaceProvider.class.getName()); MockFreeSpaceProvider.class.getName());
} }
private DataSetBuilder dataSet(String code) private DataSetBuilder dataSet(String code)
{ {
return new DataSetBuilder().code(code).shareID(SHARE_ID).type("A") return new DataSetBuilder().code(code).type("A")
.location(LOCATION_PREFIX + code).status(DataSetArchivingStatus.AVAILABLE) .location(LOCATION_PREFIX + code).status(DataSetArchivingStatus.AVAILABLE)
.registrationDate(new Date(100)); .registrationDate(new Date(100));
} }
private void writeSomeDataTo(DataSet dataSet, int numberOfBytes)
{
File dataSetFolder =
new File(new File(share.getParentFile(), dataSet.getShareId()),
dataSet.getLocation());
dataSetFolder.mkdirs();
byte[] data = new byte[numberOfBytes];
for (int i = 0; i < numberOfBytes; i++)
{
data[i] = (byte) i;
}
try
{
FileUtils.writeByteArrayToFile(new File(dataSetFolder, "data"), data);
} catch (IOException ex)
{
throw CheckedExceptionTunnel.wrapIfNecessary(ex);
}
}
@AfterMethod @AfterMethod
public void afterMethod() public void afterMethod()
{ {
...@@ -211,6 +179,7 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase ...@@ -211,6 +179,7 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase
lockedDataSet.setSize(100L); lockedDataSet.setSize(100L);
oldDataSet.setSize(20L); oldDataSet.setSize(20L);
youngDataSet.setSize(10L); youngDataSet.setSize(10L);
prepareSizesAndTypes(lockedDataSet, oldDataSet, youngDataSet);
prepareArchivingDataSets(oldDataSet, youngDataSet); prepareArchivingDataSets(oldDataSet, youngDataSet);
task.setUp("", properties); task.setUp("", properties);
...@@ -249,22 +218,6 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase ...@@ -249,22 +218,6 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase
context.assertIsSatisfied(); context.assertIsSatisfied();
} }
@Test
public void testArchiveExperimentContainingDataSetsFromDifferentShare()
{
prepareFreeSpaceProvider(99L);
prepareListExperiments(e1);
prepareListDataSetsOf(e1, dataSetInAShareToBeIgnored, youngDataSet);
youngDataSet.setSize(10L);
prepareArchivingDataSets(youngDataSet);
task.setUp("", properties);
task.execute();
checkLog(logEntry(e1, youngDataSet));
context.assertIsSatisfied();
}
@Test @Test
public void testArchiveExperimentContainingDataSetsOfAnyType() public void testArchiveExperimentContainingDataSetsOfAnyType()
{ {
...@@ -272,8 +225,9 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase ...@@ -272,8 +225,9 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase
prepareFreeSpaceProvider(99L); prepareFreeSpaceProvider(99L);
prepareListExperiments(e1); prepareListExperiments(e1);
prepareListDataSetsOf(e1, dataSetOfIgnoredType, youngDataSet); prepareListDataSetsOf(e1, dataSetOfIgnoredType, youngDataSet);
dataSetOfIgnoredType.setSize(20L); prepareDefaultDataSetSize(20L);
youngDataSet.setSize(10L); youngDataSet.setSize(10L);
prepareSizesAndTypes(youngDataSet);
prepareArchivingDataSets(dataSetOfIgnoredType, youngDataSet); prepareArchivingDataSets(dataSetOfIgnoredType, youngDataSet);
task.setUp("", properties); task.setUp("", properties);
...@@ -290,6 +244,7 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase ...@@ -290,6 +244,7 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase
prepareListExperiments(e1); prepareListExperiments(e1);
prepareListDataSetsOf(e1, dataSetOfIgnoredType, youngDataSet); prepareListDataSetsOf(e1, dataSetOfIgnoredType, youngDataSet);
youngDataSet.setSize(10L); youngDataSet.setSize(10L);
prepareSizesAndTypes(youngDataSet);
prepareArchivingDataSets(youngDataSet); prepareArchivingDataSets(youngDataSet);
task.setUp("", properties); task.setUp("", properties);
...@@ -312,6 +267,7 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase ...@@ -312,6 +267,7 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase
middleOldDataSet.setSize(10L); middleOldDataSet.setSize(10L);
prepareArchivingDataSets(oldDataSet); prepareArchivingDataSets(oldDataSet);
prepareArchivingDataSets(middleOldDataSet); prepareArchivingDataSets(middleOldDataSet);
prepareSizesAndTypes(oldDataSet, middleOldDataSet);
task.setUp("", properties); task.setUp("", properties);
task.execute(); task.execute();
...@@ -320,23 +276,6 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase ...@@ -320,23 +276,6 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase
context.assertIsSatisfied(); context.assertIsSatisfied();
} }
@Test
public void testCalculatingDataSetSize()
{
prepareFreeSpaceProvider(99L);
prepareListExperiments(e1);
prepareListDataSetsOf(e1, youngDataSet);
writeSomeDataTo(youngDataSet, 700 * 1024);
prepareRetrievingShareIdFor(youngDataSet);
prepareArchivingDataSets(youngDataSet);
task.setUp("", properties);
task.execute();
checkLog(logEntry(e1, youngDataSet));
context.assertIsSatisfied();
}
@Test @Test
public void testArchiveExperimentsUntilThereIsEnoughFreeSpace() public void testArchiveExperimentsUntilThereIsEnoughFreeSpace()
{ {
...@@ -344,8 +283,9 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase ...@@ -344,8 +283,9 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase
prepareListExperiments(e1, e2); prepareListExperiments(e1, e2);
prepareListDataSetsOf(e1, veryYoungDataSet); prepareListDataSetsOf(e1, veryYoungDataSet);
prepareListDataSetsOf(e2, oldDataSet, youngDataSet); prepareListDataSetsOf(e2, oldDataSet, youngDataSet);
oldDataSet.setSize(500 * 1024L); oldDataSet.setSize(500L);
youngDataSet.setSize(700 * 1024L); youngDataSet.setSize(700L);
prepareSizesAndTypes(oldDataSet, youngDataSet);
prepareArchivingDataSets(oldDataSet, youngDataSet); prepareArchivingDataSets(oldDataSet, youngDataSet);
task.setUp("", properties); task.setUp("", properties);
...@@ -356,29 +296,29 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase ...@@ -356,29 +296,29 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase
} }
@Test @Test
public void testArchiveExperimentsWithMonitoredDirectoryUntilThereIsEnoughFreeSpace() public void testArchiveAllExperiments()
{ {
final File monitored = new File(workingDirectory, "/some/dir"); prepareFreeSpaceProvider(90L);
monitored.mkdirs();
prepareFreeSpaceProvider(monitored, 90L, 95L, 101L);
prepareListExperiments(e3, e1, e2); prepareListExperiments(e3, e1, e2);
prepareListDataSetsOf(e3, lockedDataSet, dataSetInAShareToBeIgnored); prepareListDataSetsOf(e3, lockedDataSet, dataSetWithNoModificationDate);
prepareListDataSetsOf(e1, veryYoungDataSet); prepareListDataSetsOf(e1, veryYoungDataSet);
prepareListDataSetsOf(e2, oldDataSet, youngDataSet); prepareListDataSetsOf(e2, oldDataSet, youngDataSet);
lockedDataSet.setSize(1000 * 1024L); lockedDataSet.setSize(1000L);
dataSetInAShareToBeIgnored.setSize(100 * 1024L); dataSetWithNoModificationDate.setSize(100L);
oldDataSet.setSize(500 * 1024L); oldDataSet.setSize(500L);
youngDataSet.setSize(700 * 1024L); youngDataSet.setSize(700L);
prepareArchivingDataSets(dataSetInAShareToBeIgnored); prepareArchivingDataSets(dataSetWithNoModificationDate);
prepareArchivingDataSets(oldDataSet, youngDataSet); prepareArchivingDataSets(oldDataSet, youngDataSet);
properties.remove(ExperimentBasedArchivingTask.MONITORED_SHARE_KEY); prepareArchivingDataSets(veryYoungDataSet);
properties.put(ExperimentBasedArchivingTask.MONITORED_DIR, monitored.getPath()); prepareDefaultDataSetSize(5000L);
prepareSizesAndTypes(dataSetWithNoModificationDate, oldDataSet, youngDataSet);
task.setUp("", properties); task.setUp("", properties);
task.execute(); task.execute();
checkLog(true, logEntry(e3, dataSetInAShareToBeIgnored), checkLog(true, logEntry(e3, dataSetWithNoModificationDate),
logEntry(e2, oldDataSet, youngDataSet)); logEntry(e2, oldDataSet, youngDataSet), logEntry(e1, veryYoungDataSet));
context.assertIsSatisfied(); context.assertIsSatisfied();
} }
...@@ -391,8 +331,10 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase ...@@ -391,8 +331,10 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase
prepareListDataSetsOf(e2, dataSetWithNoModificationDate); prepareListDataSetsOf(e2, dataSetWithNoModificationDate);
prepareArchivingDataSets(youngDataSet); prepareArchivingDataSets(youngDataSet);
prepareArchivingDataSets(dataSetWithNoModificationDate); prepareArchivingDataSets(dataSetWithNoModificationDate);
youngDataSet.setSize(700 * 1024L); youngDataSet.setSize(700L);
dataSetWithNoModificationDate.setSize(500 * 1024L); dataSetWithNoModificationDate.setSize(500L);
prepareSizesAndTypes(youngDataSet, dataSetWithNoModificationDate);
task.setUp("", properties); task.setUp("", properties);
task.execute(); task.execute();
...@@ -434,21 +376,19 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase ...@@ -434,21 +376,19 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase
private void prepareFreeSpaceProvider(final long freeSpace) private void prepareFreeSpaceProvider(final long freeSpace)
{ {
prepareFreeSpaceProvider(share, freeSpace); prepareFreeSpaceProvider(monitoredDir, freeSpace);
} }
private void prepareFreeSpaceProvider(final File dir, final long... freeSpace) private void prepareFreeSpaceProvider(final File dir, final long freeSpace)
{ {
context.checking(new Expectations() context.checking(new Expectations()
{ {
{ {
try try
{ {
for (long fs : freeSpace) File absoluteFile = new File(dir.getAbsolutePath());
{ one(freeSpaceProvider).freeSpaceKb(new HostAwareFile(absoluteFile));
one(freeSpaceProvider).freeSpaceKb(new HostAwareFile(dir)); will(returnValue(1024L * freeSpace));
will(returnValue(1024L * fs));
}
} catch (IOException ex) } catch (IOException ex)
{ {
throw CheckedExceptionTunnel.wrapIfNecessary(ex); throw CheckedExceptionTunnel.wrapIfNecessary(ex);
...@@ -485,26 +425,31 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase ...@@ -485,26 +425,31 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase
}); });
} }
private void prepareArchivingDataSets(ExternalData... dataSets) private void prepareDefaultDataSetSize(long defaultSize)
{ {
final List<String> dataSetCodes = getDataSetCodes(dataSets); properties.put(ExperimentBasedArchivingTask.DATA_SET_SIZE_PREFIX
context.checking(new Expectations() + ExperimentBasedArchivingTask.DEFAULT_DATA_SET_TYPE, "" + defaultSize);
{
{
one(service).archiveDataSets(dataSetCodes, true);
}
});
} }
private void prepareRetrievingShareIdFor(final DataSet dataSet) private void prepareSizesAndTypes(DataSet... dataSets)
{ {
for (DataSet dataSet : dataSets)
{
String dataSetSize = String.valueOf(dataSet.getSize());
String dataSetType = "DS_TYPE_" + dataSetSize;
dataSet.getDataSetType().setCode(dataSetType);
properties.put(ExperimentBasedArchivingTask.DATA_SET_SIZE_PREFIX + dataSetType,
dataSetSize);
}
}
private void prepareArchivingDataSets(ExternalData... dataSets)
{
final List<String> dataSetCodes = getDataSetCodes(dataSets);
context.checking(new Expectations() context.checking(new Expectations()
{ {
{ {
one(shareIdManager).lock(dataSet.getCode()); one(service).archiveDataSets(dataSetCodes, true);
one(shareIdManager).getShareId(dataSet.getCode());
will(returnValue(dataSet.getShareId()));
one(shareIdManager).releaseLock(dataSet.getCode());
} }
}); });
} }
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment