From d3fb92cb390223a46626ab03c4a61a933774148e Mon Sep 17 00:00:00 2001
From: felmer <felmer>
Date: Mon, 3 Nov 2014 12:28:09 +0000
Subject: [PATCH] SSDM-1010: SegmentedStoreUtils.freeSpace() introduced and
tested.
SVN: 32703
---
.../shared/utils/SegmentedStoreUtils.java | 113 ++++++++++++++++
.../shared/utils/SegmentedStoreUtilsTest.java | 124 ++++++++++++++++--
.../dto/SimpleDataSetInformationDTO.java | 12 ++
.../translator/SimpleDataSetHelper.java | 1 +
4 files changed, 242 insertions(+), 8 deletions(-)
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java
index 58ad240b680..a1ebf600348 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java
@@ -24,6 +24,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -34,6 +35,7 @@ import org.apache.commons.io.FileUtils;
import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
import ch.systemsx.cisd.base.exceptions.IOExceptionUnchecked;
import ch.systemsx.cisd.base.utilities.OSUtilities;
+import ch.systemsx.cisd.common.collection.CollectionUtils;
import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
import ch.systemsx.cisd.common.exceptions.UserFailureException;
@@ -53,6 +55,7 @@ import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
import ch.systemsx.cisd.openbis.dss.generic.shared.IShareIdManager;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IDatasetLocation;
+import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
/**
@@ -84,6 +87,15 @@ public class SegmentedStoreUtils
}
};
+ private static final Comparator<SimpleDataSetInformationDTO> MODIFICATION_TIMESTAMP_COMPARATOR = new Comparator<SimpleDataSetInformationDTO>()
+ {
+ @Override
+ public int compare(SimpleDataSetInformationDTO d1, SimpleDataSetInformationDTO d2)
+ {
+ return d1.getModificationTimestamp().compareTo(d2.getModificationTimestamp());
+ }
+ };
+
private static final FileFilter FILTER_ON_SHARES = new FileFilter()
{
@Override
@@ -204,6 +216,107 @@ public class SegmentedStoreUtils
(System.currentTimeMillis() - start) / 1000.0));
return shares;
}
+
+ /**
+ * Frees space in specified share for unarchived data sets. This method assumes that the size of
+ * all specified data sets are known by the {@link DatasetDescription} objects. Data sets with oldest
+ * modification date are removed first. The archiving status of these data sets are set back to ARCHIVED.
+ *
+ * @param dataSets The data sets which should be kept (if already in the specified share). In addition
+ * they specify the amount spaced to be freed.
+ */
+ public static void freeSpace(Share unarchivingScratchShare, IEncapsulatedOpenBISService service,
+ List<DatasetDescription> dataSets, IDataSetDirectoryProvider dataSetDirectoryProvider,
+ IShareIdManager shareIdManager, ISimpleLogger logger)
+ {
+ long requestedSpace = calculateTotalSizeOfDataSetsToKeep(dataSets);
+ long actualFreeSpace = unarchivingScratchShare.calculateFreeSpace();
+ if (actualFreeSpace > requestedSpace)
+ {
+ return;
+ }
+ List<SimpleDataSetInformationDTO> dataSetsInShare = listRemovableDataSets(unarchivingScratchShare, dataSets);
+ Collections.sort(dataSetsInShare, MODIFICATION_TIMESTAMP_COMPARATOR);
+ List<SimpleDataSetInformationDTO> dataSetsToRemoveFromShare =
+ listDataSetsToRemoveFromShare(dataSetsInShare, requestedSpace, actualFreeSpace,
+ unarchivingScratchShare, logger);
+ service.archiveDataSets(extractCodes(dataSetsToRemoveFromShare), false);
+ for (SimpleDataSetInformationDTO dataSet : dataSetsToRemoveFromShare)
+ {
+ deleteDataSet(dataSet, dataSetDirectoryProvider, shareIdManager, logger);
+ }
+ logger.log(LogLevel.INFO, "The following data sets have been successfully removed from share '"
+ + unarchivingScratchShare.getShareId() + "' and their archiving status has been successfully "
+ + "set back to ARCHIVED: " + CollectionUtils.abbreviate(extractCodes(dataSetsToRemoveFromShare), 10));
+ }
+
+ private static long calculateTotalSizeOfDataSetsToKeep(List<DatasetDescription> dataSets)
+ {
+ long size = 0;
+ for (DatasetDescription dataSet : dataSets)
+ {
+ Long dataSetSize = dataSet.getDataSetSize();
+ if (dataSetSize == null)
+ {
+ throw new IllegalArgumentException("Unknown size of data set '" + dataSet.getDataSetCode() + "'.");
+ }
+ size += dataSetSize;
+ }
+ return size;
+ }
+
+ private static List<SimpleDataSetInformationDTO> listRemovableDataSets(Share share,
+ List<DatasetDescription> dataSets)
+ {
+ Set<String> dataSetsToBeKept = new HashSet<String>();
+ for (DatasetDescription dataSet : dataSets)
+ {
+ dataSetsToBeKept.add(dataSet.getDataSetCode());
+ }
+ List<SimpleDataSetInformationDTO> dataSetsInShare = new ArrayList<SimpleDataSetInformationDTO>();
+ for (SimpleDataSetInformationDTO dataSetInShare : share.getDataSetsOrderedBySize())
+ {
+ if (dataSetsToBeKept.contains(dataSetInShare.getDataStoreCode()) == false)
+ {
+ dataSetsInShare.add(dataSetInShare);
+ }
+ }
+ return dataSetsInShare;
+ }
+
+ private static List<String> extractCodes(List<SimpleDataSetInformationDTO> dataSets)
+ {
+ List<String> codes = new ArrayList<String>();
+ for (SimpleDataSetInformationDTO dataSet : dataSets)
+ {
+ codes.add(dataSet.getDataSetCode());
+ }
+ return codes;
+ }
+
+ private static List<SimpleDataSetInformationDTO> listDataSetsToRemoveFromShare(
+ List<SimpleDataSetInformationDTO> dataSetsInShare,
+ long requestedSpace, long actualFreeSpace, Share share, ISimpleLogger logger)
+ {
+ long freeSpace = actualFreeSpace;
+ List<SimpleDataSetInformationDTO> dataSetsToRemoveFromShare = new ArrayList<SimpleDataSetInformationDTO>();
+ for (int i = 0, n = dataSetsInShare.size(); i < n && requestedSpace >= freeSpace; i++)
+ {
+ SimpleDataSetInformationDTO dataSetInShare = dataSetsInShare.get(i);
+ freeSpace += dataSetInShare.getDataSetSize();
+ dataSetsToRemoveFromShare.add(dataSetInShare);
+ }
+ logger.log(LogLevel.INFO, "Remove the following data sets from share '" + share.getShareId()
+ + "' and set their archiving status back to ARCHIVED: "
+ + CollectionUtils.abbreviate(extractCodes(dataSetsToRemoveFromShare), 10));
+ if (requestedSpace >= freeSpace)
+ {
+ throw new EnvironmentFailureException("After removing all removable data sets from share '"
+ + share.getShareId() + "' there is still only " + FileUtilities.byteCountToDisplaySize(freeSpace)
+ + " free space less than the requested " + FileUtilities.byteCountToDisplaySize(requestedSpace) + ".");
+ }
+ return dataSetsToRemoveFromShare;
+ }
static List<Share> getSharesWithDataSets(File storeRoot, String dataStoreCode,
boolean filterOutToBeIgnoredForShuffling, IFreeSpaceProvider freeSpaceProvider,
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtilsTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtilsTest.java
index d6ec4854e5d..d29d9411feb 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtilsTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtilsTest.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Date;
import java.util.List;
import org.apache.commons.io.FileUtils;
@@ -48,6 +49,7 @@ import ch.systemsx.cisd.openbis.dss.generic.shared.ProxyShareIdManager;
import ch.systemsx.cisd.openbis.generic.shared.Constants;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.PhysicalDataSet;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IDatasetLocation;
+import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
/**
@@ -84,6 +86,10 @@ public class SegmentedStoreUtilsTest extends AbstractFileSystemTestCase
private File store;
+ private int modificationTimestamp;
+
+ private File shareFolder;
+
@BeforeMethod
public void beforeMethod()
{
@@ -106,6 +112,7 @@ public class SegmentedStoreUtilsTest extends AbstractFileSystemTestCase
log = new MockLogger();
store = new File(workingDirectory, "store");
store.mkdirs();
+ shareFolder = new File(store, "1");
new File(store, "blabla").mkdirs();
new File(store, "error").mkdirs();
}
@@ -123,6 +130,85 @@ public class SegmentedStoreUtilsTest extends AbstractFileSystemTestCase
}
}
+ @Test
+ public void testFreeSpace()
+ {
+ SimpleDataSetInformationDTO ds1 = dataSet(1, 10 * FileUtils.ONE_KB);
+ SimpleDataSetInformationDTO ds2 = dataSet(2, 10 * FileUtils.ONE_KB);
+ SimpleDataSetInformationDTO ds3 = dataSet(3, 12 * FileUtils.ONE_KB);
+ SimpleDataSetInformationDTO ds4 = dataSet(4, 11 * FileUtils.ONE_KB);
+ SimpleDataSetInformationDTO ds5 = dataSet(5, 14 * FileUtils.ONE_KB);
+ Share share = new Share(shareFolder, 0, freeSpaceProvider);
+ share.addDataSet(ds5);
+ share.addDataSet(ds3);
+ share.addDataSet(ds1);
+ RecordingMatcher<HostAwareFile> recordingFileMatcher = prepareFreeSpace(12L);
+ prepareSetArchingStatus(ds1);
+ File file = prepareDeleteFromShare(ds1);
+ assertEquals(true, file.exists());
+
+ SegmentedStoreUtils.freeSpace(share, service, asDatasetDescriptions(ds2, ds4), dataSetDirectoryProvider,
+ shareIdManager, log);
+
+ assertEquals(false, file.exists());
+ assertEquals(shareFolder.getPath(), recordingFileMatcher.recordedObject().getPath());
+ assertEquals("INFO: Remove the following data sets from share '1' and set their archiving status "
+ + "back to ARCHIVED: [ds-1]\n"
+ + "INFO: Await for data set ds-1 to be unlocked.\n"
+ + "INFO: Start deleting data set ds-1 at " + shareFolder + "/abc/ds-1\n"
+ + "INFO: Data set ds-1 at " + shareFolder + "/abc/ds-1 has been successfully deleted.\n"
+ + "INFO: The following data sets have been successfully removed from share '1' "
+ + "and their archiving status has been successfully set back to ARCHIVED: [ds-1]\n", log.toString());
+ }
+
+ private File prepareDeleteFromShare(final SimpleDataSetInformationDTO dataSet)
+ {
+ final File file = new File(shareFolder, dataSet.getDataSetLocation());
+ context.checking(new Expectations()
+ {
+ {
+ one(shareIdManager).await(dataSet.getDataSetCode());
+ one(dataSetDirectoryProvider).getDataSetDirectory(dataSet);
+ will(returnValue(file));
+ }
+ });
+ return file;
+ }
+
+ private void prepareSetArchingStatus(final SimpleDataSetInformationDTO... dataSets)
+ {
+ context.checking(new Expectations()
+ {
+ {
+ List<String> dataSetCodes = new ArrayList<String>();
+ for (SimpleDataSetInformationDTO dataSet : dataSets)
+ {
+ dataSetCodes.add(dataSet.getDataSetCode());
+ }
+ one(service).archiveDataSets(dataSetCodes, false);
+ }
+ });
+ }
+
+ private RecordingMatcher<HostAwareFile> prepareFreeSpace(final long freeSpace)
+ {
+ final RecordingMatcher<HostAwareFile> recorder = new RecordingMatcher<HostAwareFile>();
+ context.checking(new Expectations()
+ {
+ {
+ try
+ {
+ one(freeSpaceProvider).freeSpaceKb(with(recorder));
+ will(returnValue(freeSpace));
+ } catch (IOException ex)
+ {
+ ex.printStackTrace();
+ }
+ }
+ });
+ return recorder;
+ }
+
@Test
public void testGetDataSetsPerShare()
{
@@ -482,13 +568,13 @@ public class SegmentedStoreUtilsTest extends AbstractFileSystemTestCase
share1.mkdirs();
FileUtilities.writeToFile(new File(share1, "share.properties"),
ShareFactory.IGNORED_FOR_SHUFFLING_PROP + " = true");
-
+
String share = SegmentedStoreUtils.findIncomingShare(incomingFolder, store, log);
-
+
assertEquals("1", share);
assertEquals("", log.toString());
}
-
+
@Test
public void testGetSharesFilterOutIgnoredForShuffling()
{
@@ -498,15 +584,15 @@ public class SegmentedStoreUtilsTest extends AbstractFileSystemTestCase
ShareFactory.IGNORED_FOR_SHUFFLING_PROP + " = true");
File share2 = new File(store, "2");
share2.mkdirs();
-
+
List<Share> shares =
SegmentedStoreUtils.getSharesWithDataSets(store, DATA_STORE_CODE, true,
freeSpaceProvider, service, log, timeProvider);
-
+
assertEquals("2", shares.get(0).getShareId());
assertEquals(1, shares.size());
}
-
+
@Test
public void testGetAllShares()
{
@@ -516,11 +602,11 @@ public class SegmentedStoreUtilsTest extends AbstractFileSystemTestCase
ShareFactory.IGNORED_FOR_SHUFFLING_PROP + " = true");
File share2 = new File(store, "2");
share2.mkdirs();
-
+
List<Share> shares =
SegmentedStoreUtils.getSharesWithDataSets(store, DATA_STORE_CODE, false,
freeSpaceProvider, service, log, timeProvider);
-
+
assertEquals("1", shares.get(0).getShareId());
assertEquals("2", shares.get(1).getShareId());
assertEquals(2, shares.size());
@@ -551,6 +637,27 @@ public class SegmentedStoreUtilsTest extends AbstractFileSystemTestCase
assertEquals(Arrays.asList(names).toString(), actualNames.toString());
}
+ private List<DatasetDescription> asDatasetDescriptions(SimpleDataSetInformationDTO... dataSets)
+ {
+ List<DatasetDescription> result = new ArrayList<DatasetDescription>();
+ for (SimpleDataSetInformationDTO dataSet : dataSets)
+ {
+ DatasetDescription datasetDescription = new DatasetDescription();
+ datasetDescription.setDataSetCode(dataSet.getDataSetCode());
+ datasetDescription.setDataSetSize(dataSet.getDataSetSize());
+ result.add(datasetDescription);
+ }
+ return result;
+ }
+
+ private SimpleDataSetInformationDTO dataSet(int id, long size)
+ {
+ File dsFile = new File(shareFolder, "abc/ds-" + id);
+ dsFile.mkdirs();
+ FileUtilities.writeToFile(new File(dsFile, "read.me"), id + " nice works!");
+ return dataSet(dsFile, DATA_STORE_CODE, size);
+ }
+
private SimpleDataSetInformationDTO dataSet(File dataSetFile, String dataStoreCode, Long size)
{
SimpleDataSetInformationDTO dataSet = new SimpleDataSetInformationDTO();
@@ -561,6 +668,7 @@ public class SegmentedStoreUtilsTest extends AbstractFileSystemTestCase
dataSet.setDataSetShareId(path.substring(0, indexOfFirstSeparator));
dataSet.setDataSetLocation(path.substring(indexOfFirstSeparator + 1));
dataSet.setDataSetSize(size);
+ dataSet.setModificationTimestamp(new Date(modificationTimestamp += 10000));
return dataSet;
}
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/dto/SimpleDataSetInformationDTO.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/dto/SimpleDataSetInformationDTO.java
index f100956f7aa..9a1aad195b4 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/dto/SimpleDataSetInformationDTO.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/dto/SimpleDataSetInformationDTO.java
@@ -42,6 +42,8 @@ public class SimpleDataSetInformationDTO implements Serializable, IDatasetLocati
private String dataSetLocation;
private Date registrationTimestamp;
+
+ private Date modificationTimestamp;
private Long dataSetSize;
@@ -136,6 +138,16 @@ public class SimpleDataSetInformationDTO implements Serializable, IDatasetLocati
this.registrationTimestamp = registrationTimestamp;
}
+ public Date getModificationTimestamp()
+ {
+ return modificationTimestamp;
+ }
+
+ public void setModificationTimestamp(Date modificationTimestamp)
+ {
+ this.modificationTimestamp = modificationTimestamp;
+ }
+
public int getSpeedHint()
{
return speedHint;
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/translator/SimpleDataSetHelper.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/translator/SimpleDataSetHelper.java
index 984218728e3..2e603cceee4 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/translator/SimpleDataSetHelper.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/translator/SimpleDataSetHelper.java
@@ -60,6 +60,7 @@ public class SimpleDataSetHelper
result.setDataStoreCode(data.getDataStore().getCode());
result.setDataSetCode(data.getCode());
result.setRegistrationTimestamp(data.getRegistrationDate());
+ result.setModificationTimestamp(data.getModificationDate());
result.setSpeedHint(data.getSpeedHint());
result.setDataSetShareId(data.getShareId());
result.setDataSetLocation(data.getLocation());
--
GitLab