diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java
index a1ebf600348ca7d454bf390c4ad57e8052aa6f7d..2b08fc7daf91d2ef6a837963ea878b81677036bb 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java
@@ -25,6 +25,7 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -49,8 +50,8 @@ import ch.systemsx.cisd.common.logging.ISimpleLogger;
import ch.systemsx.cisd.common.logging.LogLevel;
import ch.systemsx.cisd.common.utilities.ITimeProvider;
import ch.systemsx.cisd.common.utilities.SystemTimeProvider;
-import ch.systemsx.cisd.openbis.dss.generic.shared.IDataSetDirectoryProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.IChecksumProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IDataSetDirectoryProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
import ch.systemsx.cisd.openbis.dss.generic.shared.IShareIdManager;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
@@ -229,25 +230,56 @@ public class SegmentedStoreUtils
List<DatasetDescription> dataSets, IDataSetDirectoryProvider dataSetDirectoryProvider,
IShareIdManager shareIdManager, ISimpleLogger logger)
{
- long requestedSpace = calculateTotalSizeOfDataSetsToKeep(dataSets);
+ List<DatasetDescription> filteredDataSets = new ArrayList<DatasetDescription>(dataSets);
+ List<SimpleDataSetInformationDTO> filteredDataSetsInShare
+ = new ArrayList<SimpleDataSetInformationDTO>(unarchivingScratchShare.getDataSetsOrderedBySize());
+ removeCommonDataSets(filteredDataSets, filteredDataSetsInShare);
+ long requestedSpace = calculateTotalSizeOfDataSetsToKeep(filteredDataSets);
long actualFreeSpace = unarchivingScratchShare.calculateFreeSpace();
- if (actualFreeSpace > requestedSpace)
+ if (actualFreeSpace < requestedSpace)
+ {
+ Collections.sort(filteredDataSetsInShare, MODIFICATION_TIMESTAMP_COMPARATOR);
+ List<SimpleDataSetInformationDTO> dataSetsToRemoveFromShare =
+ listDataSetsToRemoveFromShare(filteredDataSetsInShare, requestedSpace, actualFreeSpace,
+ unarchivingScratchShare, logger);
+ logger.log(LogLevel.INFO, "Remove the following data sets from share '" + unarchivingScratchShare.getShareId()
+ + "' and set their archiving status back to ARCHIVED: "
+ + CollectionUtils.abbreviate(extractCodes(dataSetsToRemoveFromShare), 10));
+ service.archiveDataSets(extractCodes(dataSetsToRemoveFromShare), false);
+ for (SimpleDataSetInformationDTO dataSet : dataSetsToRemoveFromShare)
+ {
+ deleteDataSet(dataSet, dataSetDirectoryProvider, shareIdManager, logger);
+ }
+ logger.log(LogLevel.INFO, "The following data sets have been successfully removed from share '"
+ + unarchivingScratchShare.getShareId() + "' and their archiving status has been successfully "
+ + "set back to ARCHIVED: " + CollectionUtils.abbreviate(extractCodes(dataSetsToRemoveFromShare), 10));
+ actualFreeSpace = unarchivingScratchShare.calculateFreeSpace();
+ }
+ logger.log(LogLevel.INFO, "Free space on unarchiving scratch share '"
+ + unarchivingScratchShare.getShareId() + "': " + FileUtilities.byteCountToDisplaySize(actualFreeSpace)
+ + ", requested space for unarchiving " + filteredDataSets.size() + " data sets: "
+ + FileUtilities.byteCountToDisplaySize(requestedSpace));
+ }
+
+ private static void removeCommonDataSets(List<DatasetDescription> dataSets, List<SimpleDataSetInformationDTO> dataSetsInShare)
+ {
+ Set<String> extractCodes = new HashSet<String>(extractCodes(dataSetsInShare));
+ for (Iterator<DatasetDescription> iterator = dataSets.iterator(); iterator.hasNext();)
{
- return;
+ DatasetDescription dataSet = iterator.next();
+ if (extractCodes.remove(dataSet.getDataSetCode()))
+ {
+ iterator.remove();
+ }
}
- List<SimpleDataSetInformationDTO> dataSetsInShare = listRemovableDataSets(unarchivingScratchShare, dataSets);
- Collections.sort(dataSetsInShare, MODIFICATION_TIMESTAMP_COMPARATOR);
- List<SimpleDataSetInformationDTO> dataSetsToRemoveFromShare =
- listDataSetsToRemoveFromShare(dataSetsInShare, requestedSpace, actualFreeSpace,
- unarchivingScratchShare, logger);
- service.archiveDataSets(extractCodes(dataSetsToRemoveFromShare), false);
- for (SimpleDataSetInformationDTO dataSet : dataSetsToRemoveFromShare)
- {
- deleteDataSet(dataSet, dataSetDirectoryProvider, shareIdManager, logger);
+ for (Iterator<SimpleDataSetInformationDTO> iterator = dataSetsInShare.iterator(); iterator.hasNext();)
+ {
+ SimpleDataSetInformationDTO dataSet = iterator.next();
+ if (extractCodes.contains(dataSet.getDataSetCode()) == false)
+ {
+ iterator.remove();
+ }
}
- logger.log(LogLevel.INFO, "The following data sets have been successfully removed from share '"
- + unarchivingScratchShare.getShareId() + "' and their archiving status has been successfully "
- + "set back to ARCHIVED: " + CollectionUtils.abbreviate(extractCodes(dataSetsToRemoveFromShare), 10));
}
private static long calculateTotalSizeOfDataSetsToKeep(List<DatasetDescription> dataSets)
@@ -265,25 +297,6 @@ public class SegmentedStoreUtils
return size;
}
- private static List<SimpleDataSetInformationDTO> listRemovableDataSets(Share share,
- List<DatasetDescription> dataSets)
- {
- Set<String> dataSetsToBeKept = new HashSet<String>();
- for (DatasetDescription dataSet : dataSets)
- {
- dataSetsToBeKept.add(dataSet.getDataSetCode());
- }
- List<SimpleDataSetInformationDTO> dataSetsInShare = new ArrayList<SimpleDataSetInformationDTO>();
- for (SimpleDataSetInformationDTO dataSetInShare : share.getDataSetsOrderedBySize())
- {
- if (dataSetsToBeKept.contains(dataSetInShare.getDataStoreCode()) == false)
- {
- dataSetsInShare.add(dataSetInShare);
- }
- }
- return dataSetsInShare;
- }
-
private static List<String> extractCodes(List<SimpleDataSetInformationDTO> dataSets)
{
List<String> codes = new ArrayList<String>();
@@ -306,14 +319,11 @@ public class SegmentedStoreUtils
freeSpace += dataSetInShare.getDataSetSize();
dataSetsToRemoveFromShare.add(dataSetInShare);
}
- logger.log(LogLevel.INFO, "Remove the following data sets from share '" + share.getShareId()
- + "' and set their archiving status back to ARCHIVED: "
- + CollectionUtils.abbreviate(extractCodes(dataSetsToRemoveFromShare), 10));
if (requestedSpace >= freeSpace)
{
- throw new EnvironmentFailureException("After removing all removable data sets from share '"
- + share.getShareId() + "' there is still only " + FileUtilities.byteCountToDisplaySize(freeSpace)
- + " free space less than the requested " + FileUtilities.byteCountToDisplaySize(requestedSpace) + ".");
+ throw new EnvironmentFailureException("Even after removing all removable data sets from share '"
+ + share.getShareId() + "' there would be still only " + FileUtilities.byteCountToDisplaySize(freeSpace)
+ + " free space which is less than the requested " + FileUtilities.byteCountToDisplaySize(requestedSpace) + ".");
}
return dataSetsToRemoveFromShare;
}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtilsTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtilsTest.java
index d29d9411febd10f480a76b06b55f5c0e0922c71a..b8a6ce3b762e06ef3831895e37efaf7c106f53a0 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtilsTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtilsTest.java
@@ -27,6 +27,7 @@ import java.util.List;
import org.apache.commons.io.FileUtils;
import org.jmock.Expectations;
import org.jmock.Mockery;
+import org.jmock.Sequence;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@@ -131,7 +132,22 @@ public class SegmentedStoreUtilsTest extends AbstractFileSystemTestCase
}
@Test
- public void testFreeSpace()
+ public void testFreeSpaceNothingToDo()
+ {
+ SimpleDataSetInformationDTO ds1 = dataSet(1, 11 * FileUtils.ONE_KB);
+ Share share = new Share(shareFolder, 0, freeSpaceProvider);
+ RecordingMatcher<HostAwareFile> recordingFileMatcher = prepareFreeSpace(12L);
+
+ SegmentedStoreUtils.freeSpace(share, service, asDatasetDescriptions(ds1), dataSetDirectoryProvider,
+ shareIdManager, log);
+
+ assertEquals(shareFolder.getPath(), recordingFileMatcher.recordedObject().getPath());
+ assertEquals("INFO: Free space on unarchiving scratch share '1': 12.00 KB, "
+ + "requested space for unarchiving 1 data sets: 11.00 KB\n", log.toString());
+ }
+
+ @Test
+ public void testFreeSpaceRemovingOneDataSet()
{
SimpleDataSetInformationDTO ds1 = dataSet(1, 10 * FileUtils.ONE_KB);
SimpleDataSetInformationDTO ds2 = dataSet(2, 10 * FileUtils.ONE_KB);
@@ -142,73 +158,93 @@ public class SegmentedStoreUtilsTest extends AbstractFileSystemTestCase
share.addDataSet(ds5);
share.addDataSet(ds3);
share.addDataSet(ds1);
- RecordingMatcher<HostAwareFile> recordingFileMatcher = prepareFreeSpace(12L);
+ RecordingMatcher<HostAwareFile> recordingFileMatcher = prepareFreeSpace(12L, 22L);
prepareSetArchingStatus(ds1);
File file = prepareDeleteFromShare(ds1);
assertEquals(true, file.exists());
-
+
SegmentedStoreUtils.freeSpace(share, service, asDatasetDescriptions(ds2, ds4), dataSetDirectoryProvider,
shareIdManager, log);
-
+
assertEquals(false, file.exists());
- assertEquals(shareFolder.getPath(), recordingFileMatcher.recordedObject().getPath());
+ assertEquals(shareFolder.getPath(), recordingFileMatcher.getRecordedObjects().get(0).getPath());
+ assertEquals(shareFolder.getPath(), recordingFileMatcher.getRecordedObjects().get(1).getPath());
+ assertEquals(2, recordingFileMatcher.getRecordedObjects().size());
assertEquals("INFO: Remove the following data sets from share '1' and set their archiving status "
+ "back to ARCHIVED: [ds-1]\n"
+ "INFO: Await for data set ds-1 to be unlocked.\n"
+ "INFO: Start deleting data set ds-1 at " + shareFolder + "/abc/ds-1\n"
+ "INFO: Data set ds-1 at " + shareFolder + "/abc/ds-1 has been successfully deleted.\n"
+ "INFO: The following data sets have been successfully removed from share '1' "
- + "and their archiving status has been successfully set back to ARCHIVED: [ds-1]\n", log.toString());
- }
-
- private File prepareDeleteFromShare(final SimpleDataSetInformationDTO dataSet)
- {
- final File file = new File(shareFolder, dataSet.getDataSetLocation());
- context.checking(new Expectations()
- {
- {
- one(shareIdManager).await(dataSet.getDataSetCode());
- one(dataSetDirectoryProvider).getDataSetDirectory(dataSet);
- will(returnValue(file));
- }
- });
- return file;
+ + "and their archiving status has been successfully set back to ARCHIVED: [ds-1]\n"
+ + "INFO: Free space on unarchiving scratch share '1': 22.00 KB, requested space for "
+ + "unarchiving 2 data sets: 21.00 KB\n", log.toString());
}
-
- private void prepareSetArchingStatus(final SimpleDataSetInformationDTO... dataSets)
+
+ @Test
+ public void testFreeSpaceForThreeDataSetsOneAlreadyInShare()
{
- context.checking(new Expectations()
- {
- {
- List<String> dataSetCodes = new ArrayList<String>();
- for (SimpleDataSetInformationDTO dataSet : dataSets)
- {
- dataSetCodes.add(dataSet.getDataSetCode());
- }
- one(service).archiveDataSets(dataSetCodes, false);
- }
- });
+ SimpleDataSetInformationDTO ds1 = dataSet(1, 10 * FileUtils.ONE_KB);
+ SimpleDataSetInformationDTO ds2 = dataSet(2, 10 * FileUtils.ONE_KB);
+ SimpleDataSetInformationDTO ds3 = dataSet(3, 12 * FileUtils.ONE_KB);
+ SimpleDataSetInformationDTO ds4 = dataSet(4, 11 * FileUtils.ONE_KB);
+ SimpleDataSetInformationDTO ds5 = dataSet(5, 14 * FileUtils.ONE_KB);
+ Share share = new Share(shareFolder, 0, freeSpaceProvider);
+ share.addDataSet(ds5);
+ share.addDataSet(ds3);
+ share.addDataSet(ds1);
+ RecordingMatcher<HostAwareFile> recordingFileMatcher = prepareFreeSpace(12L, 24L);
+ prepareSetArchingStatus(ds3);
+ File file = prepareDeleteFromShare(ds3);
+ assertEquals(true, file.exists());
+
+ SegmentedStoreUtils.freeSpace(share, service, asDatasetDescriptions(ds1, ds2, ds4), dataSetDirectoryProvider,
+ shareIdManager, log);
+
+ assertEquals(false, file.exists());
+ assertEquals(shareFolder.getPath(), recordingFileMatcher.getRecordedObjects().get(0).getPath());
+ assertEquals(shareFolder.getPath(), recordingFileMatcher.getRecordedObjects().get(1).getPath());
+ assertEquals(2, recordingFileMatcher.getRecordedObjects().size());
+ assertEquals("INFO: Remove the following data sets from share '1' and set their archiving status "
+ + "back to ARCHIVED: [ds-3]\n"
+ + "INFO: Await for data set ds-3 to be unlocked.\n"
+ + "INFO: Start deleting data set ds-3 at " + shareFolder + "/abc/ds-3\n"
+ + "INFO: Data set ds-3 at " + shareFolder + "/abc/ds-3 has been successfully deleted.\n"
+ + "INFO: The following data sets have been successfully removed from share '1' "
+ + "and their archiving status has been successfully set back to ARCHIVED: [ds-3]\n"
+ + "INFO: Free space on unarchiving scratch share '1': 24.00 KB, requested space for "
+ + "unarchiving 2 data sets: 21.00 KB\n", log.toString());
}
-
- private RecordingMatcher<HostAwareFile> prepareFreeSpace(final long freeSpace)
+
+ @Test
+ public void testFreeSpaceRemovingDataSetsButStillNotEnoughFreeSpace()
{
- final RecordingMatcher<HostAwareFile> recorder = new RecordingMatcher<HostAwareFile>();
- context.checking(new Expectations()
- {
- {
- try
- {
- one(freeSpaceProvider).freeSpaceKb(with(recorder));
- will(returnValue(freeSpace));
- } catch (IOException ex)
- {
- ex.printStackTrace();
- }
- }
- });
- return recorder;
+ SimpleDataSetInformationDTO ds1 = dataSet(1, 10 * FileUtils.ONE_KB);
+ SimpleDataSetInformationDTO ds2 = dataSet(2, 10 * FileUtils.ONE_KB);
+ SimpleDataSetInformationDTO ds3 = dataSet(3, 12 * FileUtils.ONE_KB);
+ SimpleDataSetInformationDTO ds4 = dataSet(4, 11 * FileUtils.ONE_KB);
+ SimpleDataSetInformationDTO ds5 = dataSet(5, 14 * FileUtils.ONE_KB);
+ Share share = new Share(shareFolder, 0, freeSpaceProvider);
+ share.addDataSet(ds3);
+ share.addDataSet(ds2);
+ share.addDataSet(ds1);
+ RecordingMatcher<HostAwareFile> recordingFileMatcher = prepareFreeSpace(2L);
+
+ try
+ {
+ SegmentedStoreUtils.freeSpace(share, service, asDatasetDescriptions(ds4, ds5, ds1), dataSetDirectoryProvider,
+ shareIdManager, log);
+ fail("EnvironmentFailureException expected");
+ } catch (EnvironmentFailureException ex)
+ {
+ assertEquals("Even after removing all removable data sets from share '1' there would be "
+ + "still only 24.00 KB free space which is less than the requested 25.00 KB.", ex.getMessage());
+ }
+
+ assertEquals(shareFolder.getPath(), recordingFileMatcher.recordedObject().getPath());
+ assertEquals("", log.toString());
}
-
+
@Test
public void testGetDataSetsPerShare()
{
@@ -637,6 +673,59 @@ public class SegmentedStoreUtilsTest extends AbstractFileSystemTestCase
assertEquals(Arrays.asList(names).toString(), actualNames.toString());
}
+ private File prepareDeleteFromShare(final SimpleDataSetInformationDTO dataSet)
+ {
+ final File file = new File(shareFolder, dataSet.getDataSetLocation());
+ context.checking(new Expectations()
+ {
+ {
+ one(shareIdManager).await(dataSet.getDataSetCode());
+ one(dataSetDirectoryProvider).getDataSetDirectory(dataSet);
+ will(returnValue(file));
+ }
+ });
+ return file;
+ }
+
+ private void prepareSetArchingStatus(final SimpleDataSetInformationDTO... dataSets)
+ {
+ context.checking(new Expectations()
+ {
+ {
+ List<String> dataSetCodes = new ArrayList<String>();
+ for (SimpleDataSetInformationDTO dataSet : dataSets)
+ {
+ dataSetCodes.add(dataSet.getDataSetCode());
+ }
+ one(service).archiveDataSets(dataSetCodes, false);
+ }
+ });
+ }
+
+ private RecordingMatcher<HostAwareFile> prepareFreeSpace(final long... freeSpaceValues)
+ {
+ final RecordingMatcher<HostAwareFile> recorder = new RecordingMatcher<HostAwareFile>();
+ final Sequence sequence = context.sequence("free space");
+ context.checking(new Expectations()
+ {
+ {
+ try
+ {
+ for (long freeSpace : freeSpaceValues)
+ {
+ one(freeSpaceProvider).freeSpaceKb(with(recorder));
+ will(returnValue(freeSpace));
+ inSequence(sequence);
+ }
+ } catch (IOException ex)
+ {
+ ex.printStackTrace();
+ }
+ }
+ });
+ return recorder;
+ }
+
private List<DatasetDescription> asDatasetDescriptions(SimpleDataSetInformationDTO... dataSets)
{
List<DatasetDescription> result = new ArrayList<DatasetDescription>();