diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/ExperimentBasedArchivingTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/ExperimentBasedArchivingTask.java
new file mode 100644
index 0000000000000000000000000000000000000000..bdca4c074378239f51de9020991b85fd528776e3
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/ExperimentBasedArchivingTask.java
@@ -0,0 +1,343 @@
+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.plugins;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.log4j.Logger;
+
+import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.filesystem.HostAwareFile;
+import ch.systemsx.cisd.common.filesystem.IFreeSpaceProvider;
+import ch.systemsx.cisd.common.filesystem.SimpleFreeSpaceProvider;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.maintenance.IDataStoreLockingMaintenanceTask;
+import ch.systemsx.cisd.common.utilities.PropertyParametersUtil;
+import ch.systemsx.cisd.common.utilities.PropertyUtils;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IShareIdManager;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSet;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetArchivingStatus;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Experiment;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ExternalData;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Project;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.ProjectIdentifier;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.ProjectIdentifierFactory;
+
+/**
+ * Archiving maintenance task which archives all data sets of experiments starting with
+ * the oldest experiment if free disk space is below a threshold.
+ *
+ * @author Franz-Josef Elmer
+ */
+public class ExperimentBasedArchivingTask implements IDataStoreLockingMaintenanceTask
+{
+ private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
+ ExperimentBasedArchivingTask.class);
+
+ private static final Logger notificationLog = LogFactory.getLogger(LogCategory.NOTIFY,
+ ExperimentBasedArchivingTask.class);
+
+ static final String MINIMUM_FREE_SPACE_KEY = "minimum-free-space-in-MB";
+
+ static final String STOREROOT_DIR_KEY = "storeroot-dir";
+
+ static final String MONITORED_SHARE_KEY = "monitored-share";
+
+ static final String EXCLUDED_DATA_SET_TYPES_KEY = "excluded-data-set-types";
+
+ static final String MAX_NUMBER_OF_EXPERIMENTS_KEY = "max-number-of-experiments";
+
+ private static final EnumSet<DataSetArchivingStatus> ARCHIVE_STATES = EnumSet.of(
+ DataSetArchivingStatus.ARCHIVE_PENDING, DataSetArchivingStatus.ARCHIVED);
+
+ private final IEncapsulatedOpenBISService service;
+
+ private final IFreeSpaceProvider freeSpaceProvider;
+
+ private File storeRoot;
+
+ private File share;
+
+ private long minimumFreeSpace;
+
+ private String shareID;
+
+ private final IShareIdManager shareIdManager;
+
+ private Set<String> excludedDataSetTypes;
+
+ private int maxNumberOfExperiments;
+
+ public ExperimentBasedArchivingTask()
+ {
+ this(ServiceProvider.getOpenBISService(), new SimpleFreeSpaceProvider(), ServiceProvider
+ .getShareIdManager());
+ }
+
+ ExperimentBasedArchivingTask(IEncapsulatedOpenBISService service,
+ IFreeSpaceProvider freeSpaceProvider, IShareIdManager shareIdManager)
+ {
+ this.service = service;
+ this.freeSpaceProvider = freeSpaceProvider;
+ this.shareIdManager = shareIdManager;
+ }
+
+ public boolean requiresDataStoreLock()
+ {
+ return true;
+ }
+
+ public void setUp(String pluginName, Properties properties)
+ {
+ String storeRootFileName =
+ PropertyUtils.getMandatoryProperty(properties, STOREROOT_DIR_KEY);
+ storeRoot = new File(storeRootFileName);
+ if (storeRoot.isDirectory() == false)
+ {
+ throw new ConfigurationFailureException(
+ "Store root doesn't exists or isn't a directory: " + storeRoot);
+ }
+ shareID = PropertyUtils.getMandatoryProperty(properties, MONITORED_SHARE_KEY);
+ share = new File(storeRoot, shareID);
+ if (share.isDirectory() == false)
+ {
+ throw new ConfigurationFailureException("Share " + shareID
+ + " doesn't exists or isn't a directory.");
+ }
+ minimumFreeSpace =
+ FileUtils.ONE_MB * PropertyUtils.getLong(properties, MINIMUM_FREE_SPACE_KEY, 1024);
+ excludedDataSetTypes =
+ new HashSet<String>(Arrays.asList(PropertyParametersUtil.parseItemisedProperty(
+ properties.getProperty(EXCLUDED_DATA_SET_TYPES_KEY, ""),
+ EXCLUDED_DATA_SET_TYPES_KEY)));
+ maxNumberOfExperiments = PropertyUtils.getInt(properties, MAX_NUMBER_OF_EXPERIMENTS_KEY, 0);
+ }
+
+ public void execute()
+ {
+ long freeSpace = getFreeSpace();
+ if (freeSpace >= minimumFreeSpace)
+ {
+ return;
+ }
+ if (operationLog.isInfoEnabled())
+ {
+ operationLog.info("Free space is below threshold: " + freeSpace + " ("
+ + FileUtils.byteCountToDisplaySize(freeSpace) + ") < " + minimumFreeSpace
+ + " (" + FileUtils.byteCountToDisplaySize(minimumFreeSpace) + ")");
+ }
+ List<ExperimentDataSetsInfo> infos = new ArrayList<ExperimentDataSetsInfo>();
+ for (Project project : service.listProjects())
+ {
+ ProjectIdentifier projectIdentifier =
+ new ProjectIdentifierFactory(project.getIdentifier()).createIdentifier();
+ for (Experiment experiment : service.listExperiments(projectIdentifier))
+ {
+ List<ExternalData> dataSets =
+ service.listDataSetsByExperimentID(experiment.getId());
+ infos.add(new ExperimentDataSetsInfo(experiment.getIdentifier(), dataSets));
+ }
+ }
+ Collections.sort(infos, new ExperimentDataSetsInfoComparator());
+ StringBuilder archivingMessages = new StringBuilder();
+ if (maxNumberOfExperiments > 0)
+ {
+ for (int i = 0; i < Math.min(maxNumberOfExperiments, infos.size()); i++)
+ {
+ archive(infos.get(i), archivingMessages);
+ }
+ } else
+ {
+ for (int i = 0; i < infos.size() && freeSpace < minimumFreeSpace; i++)
+ {
+ ExperimentDataSetsInfo info = infos.get(i);
+ freeSpace += info.calculateSize();
+ archive(info, archivingMessages);
+ }
+ }
+ if (archivingMessages.length() > 0)
+ {
+ notificationLog.info("Archiving summary:" + archivingMessages);
+ }
+ }
+
+ private long getFreeSpace()
+ {
+ try
+ {
+ return 1024L * freeSpaceProvider.freeSpaceKb(new HostAwareFile(share));
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ private void archive(ExperimentDataSetsInfo info, StringBuilder archivingMessages)
+ {
+ List<DataSet> dataSets = info.getDataSetsToBeArchived();
+ if (dataSets.isEmpty())
+ {
+ return;
+ }
+ List<String> dataSetCodes = new ArrayList<String>();
+ for (DataSet dataSet : dataSets)
+ {
+ dataSetCodes.add(dataSet.getCode());
+ }
+ String message =
+ "Starting archiving " + dataSetCodes.size()
+ + " data sets (if not already archived) of experiment "
+ + info.getExperimentIdentifier() + ": " + dataSetCodes;
+ operationLog.info(message);
+ archivingMessages.append('\n').append(message);
+ service.archiveDataSets(dataSetCodes, false);
+ }
+
+ private final class ExperimentDataSetsInfo
+ {
+ private Date lastModificationDate;
+
+ private List<DataSet> dataSetsToBeArchived = new ArrayList<DataSet>();
+
+ private final String experimentIdentifier;
+
+ ExperimentDataSetsInfo(String experimentIdentifier, List<ExternalData> dataSets)
+ {
+ this.experimentIdentifier = experimentIdentifier;
+ boolean hasLockedDataSet = false;
+ for (ExternalData dataSet : dataSets)
+ {
+ if (dataSet instanceof DataSet == false)
+ {
+ continue;
+ }
+ DataSet realDataSet = (DataSet) dataSet;
+ if (shareID.equals(realDataSet.getShareId()) == false)
+ {
+ continue;
+ }
+ if (excludedDataSetTypes.contains(realDataSet.getDataSetType().getCode()))
+ {
+ continue;
+ }
+ DataSetArchivingStatus status = realDataSet.getStatus();
+ if (DataSetArchivingStatus.LOCKED.equals(status))
+ {
+ hasLockedDataSet = true;
+ }
+ if (ARCHIVE_STATES.contains(status))
+ {
+ continue;
+ }
+ dataSetsToBeArchived.add(realDataSet);
+ Date modificationDate = dataSet.getModificationDate();
+ if (modificationDate == null)
+ {
+ modificationDate = dataSet.getRegistrationDate();
+ }
+ if (lastModificationDate == null || lastModificationDate.before(modificationDate))
+ {
+ lastModificationDate = modificationDate;
+ }
+ }
+ if (hasLockedDataSet)
+ {
+ dataSetsToBeArchived.clear();
+ }
+ }
+
+ long calculateSize()
+ {
+ long sum = 0;
+ for (DataSet dataSet : dataSetsToBeArchived)
+ {
+ Long size = dataSet.getSize();
+ if (size == null)
+ {
+ String dataSetCode = dataSet.getCode();
+ shareIdManager.lock(dataSetCode);
+ try
+ {
+ File shareRoot =
+ new File(storeRoot, shareIdManager.getShareId(dataSetCode));
+ String location = dataSet.getLocation();
+ size = FileUtils.sizeOfDirectory(new File(shareRoot, location));
+ } finally
+ {
+ shareIdManager.releaseLock(dataSetCode);
+ }
+ }
+ sum += size;
+ }
+ return sum;
+ }
+
+ public String getExperimentIdentifier()
+ {
+ return experimentIdentifier;
+ }
+
+ public Date getLastModificationDate()
+ {
+ return lastModificationDate;
+ }
+
+ public List<DataSet> getDataSetsToBeArchived()
+ {
+ return dataSetsToBeArchived;
+ }
+ }
+
+ private final class ExperimentDataSetsInfoComparator implements Comparator<ExperimentDataSetsInfo>
+ {
+ public int compare(ExperimentDataSetsInfo i1, ExperimentDataSetsInfo i2)
+ {
+ Date d1 = i1.getLastModificationDate();
+ Date d2 = i2.getLastModificationDate();
+ if (d1 != null && d2 != null)
+ {
+ return d1.compareTo(d2);
+ }
+ if (d1 == null && d2 != null)
+ {
+ return 1;
+ }
+ if (d1 != null && d2 == null)
+ {
+ return -1;
+ }
+ return 0;
+ }
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/EncapsulatedOpenBISService.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/EncapsulatedOpenBISService.java
index ba6bc4bffcecdd370314fbb85d3310f1537bf84f..6ee9df81701b057d6810476f8d5128c8fab2884a 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/EncapsulatedOpenBISService.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/EncapsulatedOpenBISService.java
@@ -551,6 +551,11 @@ public final class EncapsulatedOpenBISService implements IEncapsulatedOpenBISSer
return service.searchForDataSets(session.getToken(), searchCriteria);
}
+ public List<Project> listProjects()
+ {
+ return service.listProjects(session.getToken());
+ }
+
public List<Experiment> listExperiments(ProjectIdentifier projectIdentifier)
{
return service.listExperiments(session.getToken(), projectIdentifier);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IEncapsulatedOpenBISService.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IEncapsulatedOpenBISService.java
index ad9da1b8bab45ffd90499b7ae833a0a0b82f8989..3c467b90a60de982a2f0b79e8ba1eb1cae3e34d6 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IEncapsulatedOpenBISService.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IEncapsulatedOpenBISService.java
@@ -458,6 +458,12 @@ public interface IEncapsulatedOpenBISService
@ManagedAuthentication
public List<ExternalData> searchForDataSets(SearchCriteria searchCriteria);
+ /**
+ * {@link IETLLIMSService#listProjects(String)}
+ */
+ @ManagedAuthentication
+ public List<Project> listProjects();
+
/**
* {@link IETLLIMSService#listExperiments(String, ProjectIdentifier)}
*/
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/ExperimentBasedArchivingTaskTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/ExperimentBasedArchivingTaskTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..03580a958df0441a61220c591cf3944d8b0dac83
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/ExperimentBasedArchivingTaskTest.java
@@ -0,0 +1,477 @@
+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.plugins;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.log4j.Level;
+import org.jmock.Expectations;
+import org.jmock.Mockery;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
+import ch.systemsx.cisd.base.tests.AbstractFileSystemTestCase;
+import ch.systemsx.cisd.common.filesystem.HostAwareFile;
+import ch.systemsx.cisd.common.filesystem.IFreeSpaceProvider;
+import ch.systemsx.cisd.common.logging.BufferedAppender;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IShareIdManager;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSet;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetArchivingStatus;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Experiment;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ExternalData;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Project;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.builders.DataSetBuilder;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.builders.ExperimentBuilder;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.ProjectIdentifier;
+
+/**
+ *
+ *
+ * @author Franz-Josef Elmer
+ */
+public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase
+{
+ private static final String LOG_ENTRY_PREFIX_TEMPLATE = "INFO %s.ExperimentBasedArchivingTask - ";
+
+ private static final String LOG_ENTRY_PREFIX =
+ String.format(LOG_ENTRY_PREFIX_TEMPLATE, LogCategory.OPERATION);
+
+ private static final String NOTIFY_LOG_ENTRY_PREFIX =
+ String.format(LOG_ENTRY_PREFIX_TEMPLATE, LogCategory.NOTIFY);
+
+ private static final String FREE_SPACE_LOG_ENTRY = LOG_ENTRY_PREFIX
+ + "Free space is below threshold: 103809024 (99 MB) < 104857600 (100 MB)";
+
+ private static final String LOCATION_PREFIX = "abc/";
+ private static final String SHARE_ID = "3";
+
+ private BufferedAppender logRecorder;
+ private Mockery context;
+ private IEncapsulatedOpenBISService service;
+ private IFreeSpaceProvider freeSpaceProvider;
+ private IShareIdManager shareIdManager;
+ private ExperimentBasedArchivingTask task;
+ private File share;
+ private Properties properties;
+ private Experiment e1;
+ private Experiment e2;
+ private Experiment e3;
+ private DataSet lockedDataSet;
+ private ExternalData notARealDataSet;
+ private DataSet dataSetInAShareToBeIgnored;
+ private DataSet dataSetOfIgnoredType;
+ private DataSet dataSetWithNoModificationDate;
+ private DataSet veryOldDataSet;
+ private DataSet oldDataSet;
+ private DataSet middleOldDataSet;
+ private DataSet youngDataSet;
+ private DataSet veryYoungDataSet;
+
+ @BeforeMethod
+ public void before()
+ {
+ logRecorder = new BufferedAppender("%-5p %c - %m%n", Level.INFO);
+ context = new Mockery();
+ service = context.mock(IEncapsulatedOpenBISService.class);
+ freeSpaceProvider = context.mock(IFreeSpaceProvider.class);
+ shareIdManager = context.mock(IShareIdManager.class);
+ task = new ExperimentBasedArchivingTask(service, freeSpaceProvider, shareIdManager);
+ assertEquals(true, task.requiresDataStoreLock());
+
+ e1 = new ExperimentBuilder().id(41).identifier("/S/P/E1").getExperiment();
+ e2 = new ExperimentBuilder().id(42).identifier("/S/P/E2").getExperiment();
+ e3 = new ExperimentBuilder().id(43).identifier("/S/P/E3").getExperiment();
+ notARealDataSet = new ExternalData();
+ lockedDataSet = dataSet("lockedDataSet").status(DataSetArchivingStatus.LOCKED).getDataSet();
+ dataSetInAShareToBeIgnored = dataSet("dataSetInAShareToBeIgnored").shareID("42").getDataSet();
+ dataSetOfIgnoredType = dataSet("dataSetOfIgnoredType").type("ABC").getDataSet();
+ dataSetWithNoModificationDate = dataSet("dataSetWithNoModificationDate").getDataSet();
+ veryOldDataSet = dataSet("veryOldDataSet").modificationDate(new Date(200)).getDataSet();
+ oldDataSet = dataSet("oldDataSet").modificationDate(new Date(300)).getDataSet();
+ middleOldDataSet = dataSet("middleOldDataSet").modificationDate(new Date(400)).getDataSet();
+ youngDataSet = dataSet("youngDataSet").modificationDate(new Date(1000)).getDataSet();
+ veryYoungDataSet = dataSet("veryYoungDataSet").modificationDate(new Date(2000)).getDataSet();
+
+ share = new File(workingDirectory, "store/" + SHARE_ID);
+ share.mkdirs();
+ properties = new Properties();
+ properties.setProperty(ExperimentBasedArchivingTask.STOREROOT_DIR_KEY, share.getParent());
+ properties.setProperty(ExperimentBasedArchivingTask.MONITORED_SHARE_KEY, SHARE_ID);
+ properties.setProperty(ExperimentBasedArchivingTask.MINIMUM_FREE_SPACE_KEY, "100");
+ properties.setProperty(ExperimentBasedArchivingTask.EXCLUDED_DATA_SET_TYPES_KEY, "ABC, B");
+ }
+
+ private DataSetBuilder dataSet(String code)
+ {
+ return new DataSetBuilder().code(code).shareID(SHARE_ID).type("A")
+ .location(LOCATION_PREFIX + code).status(DataSetArchivingStatus.AVAILABLE)
+ .registrationDate(new Date(100));
+ }
+
+ private void writeSomeDataTo(DataSet dataSet, int numberOfBytes)
+ {
+ File dataSetFolder =
+ new File(new File(share.getParentFile(), dataSet.getShareId()),
+ dataSet.getLocation());
+ dataSetFolder.mkdirs();
+ byte[] data = new byte[numberOfBytes];
+ for (int i = 0; i < numberOfBytes; i++)
+ {
+ data[i] = (byte) i;
+ }
+ try
+ {
+ FileUtils.writeByteArrayToFile(new File(dataSetFolder, "data"), data);
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ @AfterMethod
+ public void afterMethod()
+ {
+ logRecorder.reset();
+ // The following line of code should also be called at the end of each test method.
+ // Otherwise one do not known which test failed.
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testLockedDataSet()
+ {
+ prepareFreeSpaceProvider(99L);
+ prepareListExperiments(e1);
+ prepareListDataSetsOf(e1, oldDataSet, lockedDataSet, youngDataSet);
+
+ task.setUp("", properties);
+ task.execute();
+
+ checkLog();
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testArchiveExperimentContainingNotARealDataSet()
+ {
+ prepareFreeSpaceProvider(99L);
+ prepareListExperiments(e1);
+ prepareListDataSetsOf(e1, notARealDataSet);
+
+ task.setUp("", properties);
+ task.execute();
+
+ checkLog();
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testArchiveExperiments()
+ {
+ prepareFreeSpaceProvider(99L);
+ prepareListExperiments(e1, e2);
+ prepareListDataSetsOf(e1);
+ prepareListDataSetsOf(e2);
+
+ task.setUp("", properties);
+ task.execute();
+
+ checkLog();
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testArchiveExperimentContainingDataSetsFromDifferentShare()
+ {
+ prepareFreeSpaceProvider(99L);
+ prepareListExperiments(e1);
+ prepareListDataSetsOf(e1, dataSetInAShareToBeIgnored, youngDataSet);
+ youngDataSet.setSize(10L);
+ prepareArchivingDataSets(youngDataSet);
+
+ task.setUp("", properties);
+ task.execute();
+
+ checkLog(logEntry(e1, youngDataSet));
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testArchiveExperimentContainingDataSetsOfAnyType()
+ {
+ properties.remove(ExperimentBasedArchivingTask.EXCLUDED_DATA_SET_TYPES_KEY);
+ prepareFreeSpaceProvider(99L);
+ prepareListExperiments(e1);
+ prepareListDataSetsOf(e1, dataSetOfIgnoredType, youngDataSet);
+ dataSetOfIgnoredType.setSize(20L);
+ youngDataSet.setSize(10L);
+ prepareArchivingDataSets(dataSetOfIgnoredType, youngDataSet);
+
+ task.setUp("", properties);
+ task.execute();
+
+ checkLog(logEntry(e1, dataSetOfIgnoredType, youngDataSet));
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testArchiveExperimentContainingDataSetsOfTypeToBeIgnored()
+ {
+ prepareFreeSpaceProvider(99L);
+ prepareListExperiments(e1);
+ prepareListDataSetsOf(e1, dataSetOfIgnoredType, youngDataSet);
+ youngDataSet.setSize(10L);
+ prepareArchivingDataSets(youngDataSet);
+
+ task.setUp("", properties);
+ task.execute();
+
+ checkLog(logEntry(e1, youngDataSet));
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testCalculatingExperimentAgeWhereSomeDataSetsAreAlreadyArchived()
+ {
+ prepareFreeSpaceProvider(99L);
+ prepareListExperiments(e1, e2);
+ prepareListDataSetsOf(e1, oldDataSet, youngDataSet, veryYoungDataSet);
+ oldDataSet.setSize(10L);
+ youngDataSet.setStatus(DataSetArchivingStatus.ARCHIVE_PENDING);
+ veryYoungDataSet.setStatus(DataSetArchivingStatus.ARCHIVED);
+ prepareListDataSetsOf(e2, middleOldDataSet);
+ middleOldDataSet.setSize(10L);
+ prepareArchivingDataSets(oldDataSet);
+ prepareArchivingDataSets(middleOldDataSet);
+
+ task.setUp("", properties);
+ task.execute();
+
+ checkLog(logEntry(e1, oldDataSet), logEntry(e2, middleOldDataSet));
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testCalculatingDataSetSize()
+ {
+ prepareFreeSpaceProvider(99L);
+ prepareListExperiments(e1);
+ prepareListDataSetsOf(e1, youngDataSet);
+ writeSomeDataTo(youngDataSet, 700 * 1024);
+ prepareRetrievingShareIdFor(youngDataSet);
+ prepareArchivingDataSets(youngDataSet);
+
+ task.setUp("", properties);
+ task.execute();
+
+ checkLog(logEntry(e1, youngDataSet));
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testArchiveExperimentsUntilThereIsEnoughFreeSpace()
+ {
+ prepareFreeSpaceProvider(99L);
+ prepareListExperiments(e1, e2);
+ prepareListDataSetsOf(e1, veryYoungDataSet);
+ prepareListDataSetsOf(e2, oldDataSet, youngDataSet);
+ oldDataSet.setSize(500 * 1024L);
+ youngDataSet.setSize(700 * 1024L);
+ prepareArchivingDataSets(oldDataSet, youngDataSet);
+
+ task.setUp("", properties);
+ task.execute();
+
+ checkLog(logEntry(e2, oldDataSet, youngDataSet));
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testArchiveExperimentsUntilMaximumNumberOfExperimentsIsReached()
+ {
+ properties.setProperty(ExperimentBasedArchivingTask.MAX_NUMBER_OF_EXPERIMENTS_KEY, "2");
+ prepareFreeSpaceProvider(99L);
+ prepareListExperiments(e1, e2, e3);
+ prepareListDataSetsOf(e1, veryYoungDataSet);
+ prepareListDataSetsOf(e2, veryOldDataSet, youngDataSet);
+ prepareListDataSetsOf(e3, middleOldDataSet, oldDataSet);
+ prepareArchivingDataSets(veryOldDataSet, youngDataSet);
+ prepareArchivingDataSets(middleOldDataSet, oldDataSet);
+
+ task.setUp("", properties);
+ task.execute();
+
+ checkLog(logEntry(e3, middleOldDataSet, oldDataSet),
+ logEntry(e2, veryOldDataSet, youngDataSet));
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testArchiveExperimentsWithMaximumNumberIsLargerThanNumberOfExperiments()
+ {
+ properties.setProperty(ExperimentBasedArchivingTask.MAX_NUMBER_OF_EXPERIMENTS_KEY, "3");
+ prepareFreeSpaceProvider(99L);
+ prepareListExperiments(e1);
+ prepareListDataSetsOf(e1, youngDataSet);
+ prepareArchivingDataSets(youngDataSet);
+
+ task.setUp("", properties);
+ task.execute();
+
+ checkLog(logEntry(e1, youngDataSet));
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testArchiveExperimentsInCorrectOrderWhereTheOldestDataSetHasNoModificationDate()
+ {
+ properties.setProperty(ExperimentBasedArchivingTask.MAX_NUMBER_OF_EXPERIMENTS_KEY, "3");
+ prepareFreeSpaceProvider(99L);
+ prepareListExperiments(e1, e2);
+ prepareListDataSetsOf(e1, youngDataSet);
+ prepareListDataSetsOf(e2, dataSetWithNoModificationDate);
+ prepareArchivingDataSets(youngDataSet);
+ prepareArchivingDataSets(dataSetWithNoModificationDate);
+
+ task.setUp("", properties);
+ task.execute();
+
+ checkLog(logEntry(e2, dataSetWithNoModificationDate), logEntry(e1, youngDataSet));
+ context.assertIsSatisfied();
+ }
+
+ private void checkLog(String... archivingEntries)
+ {
+ StringBuilder operationLogBuilder = new StringBuilder(FREE_SPACE_LOG_ENTRY);
+ StringBuilder notifyMessageBuilder = new StringBuilder();
+ for (String entry : archivingEntries)
+ {
+ operationLogBuilder.append("\n").append(LOG_ENTRY_PREFIX).append(entry);
+ notifyMessageBuilder.append("\n").append(entry);
+ }
+ if (archivingEntries.length > 0)
+ {
+ operationLogBuilder.append("\n").append(NOTIFY_LOG_ENTRY_PREFIX);
+ operationLogBuilder.append("Archiving summary:").append(notifyMessageBuilder);
+ }
+ assertEquals(operationLogBuilder.toString(), logRecorder.getLogContent());
+ }
+
+ private String logEntry(Experiment experiment, DataSet... dataSets)
+ {
+ List<String> dataSetCodes = getDataSetCodes(dataSets);
+ return "Starting archiving " + dataSetCodes.size()
+ + " data sets (if not already archived) of experiment "
+ + experiment.getIdentifier() + ": " + dataSetCodes;
+ }
+
+ private void prepareFreeSpaceProvider(final long freeSpace)
+ {
+ context.checking(new Expectations()
+ {
+ {
+ try
+ {
+ one(freeSpaceProvider).freeSpaceKb(new HostAwareFile(share));
+ will(returnValue(1024L * freeSpace));
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+
+ }
+ });
+ }
+
+ private void prepareListExperiments(final Experiment... experiments)
+ {
+ context.checking(new Expectations()
+ {
+ {
+ one(service).listProjects();
+ Project project = new Project();
+ project.setIdentifier("/S/P");
+ will(returnValue(Arrays.asList(project)));
+
+ one(service).listExperiments(getProjectIdentifier(e1));
+ will(returnValue(Arrays.asList(experiments)));
+ }
+ });
+ }
+
+ private void prepareListDataSetsOf(final Experiment experiment, final ExternalData... dataSets)
+ {
+ context.checking(new Expectations()
+ {
+ {
+ one(service).listDataSetsByExperimentID(experiment.getId());
+ will(returnValue(Arrays.asList(dataSets)));
+ }
+ });
+ }
+
+ private void prepareArchivingDataSets(ExternalData... dataSets)
+ {
+ final List<String> dataSetCodes = getDataSetCodes(dataSets);
+ context.checking(new Expectations()
+ {
+ {
+ one(service).archiveDataSets(dataSetCodes, false);
+ }
+ });
+ }
+
+ private void prepareRetrievingShareIdFor(final DataSet dataSet)
+ {
+ context.checking(new Expectations()
+ {
+ {
+ one(shareIdManager).lock(dataSet.getCode());
+ one(shareIdManager).getShareId(dataSet.getCode());
+ will(returnValue(dataSet.getShareId()));
+ one(shareIdManager).releaseLock(dataSet.getCode());
+ }
+ });
+ }
+
+ private List<String> getDataSetCodes(ExternalData... dataSets)
+ {
+ final List<String> dataSetCodes = new ArrayList<String>();
+ for (ExternalData dataSet : dataSets)
+ {
+ dataSetCodes.add(dataSet.getCode());
+ }
+ return dataSetCodes;
+ }
+
+ private ProjectIdentifier getProjectIdentifier(Experiment e)
+ {
+ return new ProjectIdentifier(e.getProject().getSpace().getCode(), e.getProject().getCode());
+ }
+
+}