From 31a32b4479dd50d58a579589b49399ef0f36b456 Mon Sep 17 00:00:00 2001
From: felmer <felmer>
Date: Mon, 3 Nov 2014 15:18:55 +0000
Subject: [PATCH] SSDM-1010: New Share property introduced
'unarchiving-scratch-share' which is used in SegmentedStoreUtils.freeSpace()
and SegmentedStoreUtils.moveDataSetToAnotherShare(). Tests added to
SegmentedStoreUtilsTest.
SVN: 32708
---
.../shared/utils/SegmentedStoreUtils.java | 19 ++++
.../dss/generic/shared/utils/Share.java | 12 +++
.../generic/shared/utils/ShareFactory.java | 41 ++++---
.../shared/utils/SegmentedStoreUtilsTest.java | 101 +++++++++++++++++-
4 files changed, 156 insertions(+), 17 deletions(-)
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java
index 2b08fc7daf9..97082bff56f 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java
@@ -230,6 +230,13 @@ public class SegmentedStoreUtils
List<DatasetDescription> dataSets, IDataSetDirectoryProvider dataSetDirectoryProvider,
IShareIdManager shareIdManager, ISimpleLogger logger)
{
+ if (unarchivingScratchShare.isUnarchivingScratchShare() == false)
+ {
+ throw new EnvironmentFailureException("Share '" + unarchivingScratchShare.getShareId()
+ + "' isn't an unarchving scratch share. Such a share has the property "
+ + ShareFactory.UNARCHIVING_SCRATCH_SHARE_PROP + " of the file "
+ + ShareFactory.SHARE_PROPS_FILE + " set to 'true'.");
+ }
List<DatasetDescription> filteredDataSets = new ArrayList<DatasetDescription>(dataSets);
List<SimpleDataSetInformationDTO> filteredDataSetsInShare
= new ArrayList<SimpleDataSetInformationDTO>(unarchivingScratchShare.getDataSetsOrderedBySize());
@@ -403,6 +410,8 @@ public class SegmentedStoreUtils
{
return;
}
+ assertNoUnarchivingScratchShare(oldShare, logger);
+ assertNoUnarchivingScratchShare(share, logger);
File dataSetDirInNewShare = new File(share, relativePath);
dataSetDirInNewShare.mkdirs();
copyToShare(dataSetDirInStore, dataSetDirInNewShare, logger);
@@ -420,6 +429,16 @@ public class SegmentedStoreUtils
}
deleteDataSet(dataSetCode, dataSetDirInStore, shareIdManager, logger);
}
+
+ private static void assertNoUnarchivingScratchShare(File share, ISimpleLogger logger)
+ {
+ if (new ShareFactory().createShare(share, null, logger).isUnarchivingScratchShare())
+ {
+ throw new EnvironmentFailureException("Share '" + share.getName()
+ + "' is a scratch share for unarchiving purposes. "
+ + "No data sets can be moved from/to such a share.");
+ }
+ }
/**
* Deletes specified data set at specified location. This methods waits until any locks on the
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/Share.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/Share.java
index 272d6fc947a..5ad2bc679c7 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/Share.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/Share.java
@@ -89,6 +89,8 @@ public final class Share
private boolean withdrawShare;
+ private boolean unarchivingScratchShare;
+
private boolean ignoredForShuffling;
private Set<String> experimentIdentifiers;
@@ -229,6 +231,16 @@ public final class Share
this.withdrawShare = withdrawShare;
}
+ public boolean isUnarchivingScratchShare()
+ {
+ return unarchivingScratchShare;
+ }
+
+ public void setUnarchivingScratchShare(boolean unarchivingScratchShare)
+ {
+ this.unarchivingScratchShare = unarchivingScratchShare;
+ }
+
public boolean isIgnoredForShuffling()
{
return ignoredForShuffling;
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/ShareFactory.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/ShareFactory.java
index e605502d66e..97bd2df4995 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/ShareFactory.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/ShareFactory.java
@@ -48,7 +48,6 @@ public class ShareFactory
@Private
static final String SPEED_FILE = "speed";
- @Private
public static final String SHARE_PROPS_FILE = "share.properties";
@Private
@@ -60,6 +59,8 @@ public class ShareFactory
public static final String WITHDRAW_SHARE_PROP = "withdraw-share";
public static final String IGNORED_FOR_SHUFFLING_PROP = "ignored-for-shuffling";
+
+ public static final String UNARCHIVING_SCRATCH_SHARE_PROP = "unarchiving-scratch-share";
public static final String EXPERIMENTS_PROP = "experiments";
@@ -71,6 +72,8 @@ public class ShareFactory
private boolean ignoredForShuffling;
+ private boolean unarchivingScratchShare;
+
private Set<String> experimentIdentifiers = Collections.emptySet();
Share createShare(final SharesHolder sharesHolder, File shareRoot,
@@ -81,6 +84,7 @@ public class ShareFactory
Share share = new Share(sharesHolder, shareRoot, speed, freeSpaceProvider);
share.setShufflePriority(shufflePriority);
share.setWithdrawShare(withdrawShare);
+ share.setUnarchivingScratchShare(unarchivingScratchShare);
share.setIgnoredForShuffling(ignoredForShuffling);
share.setExperimentIdentifiers(experimentIdentifiers);
return share;
@@ -99,20 +103,7 @@ public class ShareFactory
File propsFile = new File(shareRoot, SHARE_PROPS_FILE);
if (propsFile.isFile())
{
- Properties props = new Properties();
- FileInputStream fis = null;
- try
- {
- fis = new FileInputStream(propsFile);
- props.load(fis);
- } catch (IOException ioex)
- {
- log.log(LogLevel.WARN, "Error while reading from " + propsFile.getAbsolutePath()
- + " : " + ioex.getMessage());
- } finally
- {
- IOUtils.closeQuietly(fis);
- }
+ Properties props = loadShareProperties(propsFile, log);
if (props.containsKey(SPEED_HINT_PROP))
{
@@ -148,6 +139,7 @@ public class ShareFactory
withdrawShare = PropertyUtils.getBoolean(props, WITHDRAW_SHARE_PROP, false);
ignoredForShuffling = PropertyUtils.getBoolean(props, IGNORED_FOR_SHUFFLING_PROP, false);
+ unarchivingScratchShare = PropertyUtils.getBoolean(props, UNARCHIVING_SCRATCH_SHARE_PROP, false);
experimentIdentifiers =
new HashSet<String>(Arrays.asList(PropertyParametersUtil.parseItemisedProperty(
props.getProperty(EXPERIMENTS_PROP, ""), EXPERIMENTS_PROP)));
@@ -155,6 +147,25 @@ public class ShareFactory
}
+ private Properties loadShareProperties(File propsFile, ISimpleLogger log)
+ {
+ Properties props = new Properties();
+ FileInputStream fis = null;
+ try
+ {
+ fis = new FileInputStream(propsFile);
+ props.load(fis);
+ } catch (IOException ioex)
+ {
+ log.log(LogLevel.WARN, "Error while reading from " + propsFile.getAbsolutePath()
+ + " : " + ioex.getMessage());
+ } finally
+ {
+ IOUtils.closeQuietly(fis);
+ }
+ return props;
+ }
+
private void readSpeedFile(File shareRoot, ISimpleLogger log)
{
File speedFile = new File(shareRoot, SPEED_FILE);
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtilsTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtilsTest.java
index b8a6ce3b762..6a7365c75db 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtilsTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtilsTest.java
@@ -131,16 +131,37 @@ public class SegmentedStoreUtilsTest extends AbstractFileSystemTestCase
}
}
+ @Test
+ public void testFreeSpaceForAShareWhichIsNotAnUnarchivingScratchShare()
+ {
+ SimpleDataSetInformationDTO ds1 = dataSet(1, 11 * FileUtils.ONE_KB);
+ Share share = new Share(shareFolder, 0, freeSpaceProvider);
+
+ try
+ {
+ SegmentedStoreUtils.freeSpace(share, service, asDatasetDescriptions(ds1), dataSetDirectoryProvider,
+ shareIdManager, log);
+ fail("EnvironmentFailureException expected.");
+ } catch (EnvironmentFailureException ex)
+ {
+ assertEquals("Share '1' isn't an unarchving scratch share. Such a share has the property "
+ + "unarchiving-scratch-share of the file share.properties set to 'true'.", ex.getMessage());
+ }
+
+ assertEquals("", log.toString());
+ }
+
@Test
public void testFreeSpaceNothingToDo()
{
SimpleDataSetInformationDTO ds1 = dataSet(1, 11 * FileUtils.ONE_KB);
Share share = new Share(shareFolder, 0, freeSpaceProvider);
+ share.setUnarchivingScratchShare(true);
RecordingMatcher<HostAwareFile> recordingFileMatcher = prepareFreeSpace(12L);
-
+
SegmentedStoreUtils.freeSpace(share, service, asDatasetDescriptions(ds1), dataSetDirectoryProvider,
shareIdManager, log);
-
+
assertEquals(shareFolder.getPath(), recordingFileMatcher.recordedObject().getPath());
assertEquals("INFO: Free space on unarchiving scratch share '1': 12.00 KB, "
+ "requested space for unarchiving 1 data sets: 11.00 KB\n", log.toString());
@@ -155,6 +176,7 @@ public class SegmentedStoreUtilsTest extends AbstractFileSystemTestCase
SimpleDataSetInformationDTO ds4 = dataSet(4, 11 * FileUtils.ONE_KB);
SimpleDataSetInformationDTO ds5 = dataSet(5, 14 * FileUtils.ONE_KB);
Share share = new Share(shareFolder, 0, freeSpaceProvider);
+ share.setUnarchivingScratchShare(true);
share.addDataSet(ds5);
share.addDataSet(ds3);
share.addDataSet(ds1);
@@ -190,6 +212,7 @@ public class SegmentedStoreUtilsTest extends AbstractFileSystemTestCase
SimpleDataSetInformationDTO ds4 = dataSet(4, 11 * FileUtils.ONE_KB);
SimpleDataSetInformationDTO ds5 = dataSet(5, 14 * FileUtils.ONE_KB);
Share share = new Share(shareFolder, 0, freeSpaceProvider);
+ share.setUnarchivingScratchShare(true);
share.addDataSet(ds5);
share.addDataSet(ds3);
share.addDataSet(ds1);
@@ -225,6 +248,7 @@ public class SegmentedStoreUtilsTest extends AbstractFileSystemTestCase
SimpleDataSetInformationDTO ds4 = dataSet(4, 11 * FileUtils.ONE_KB);
SimpleDataSetInformationDTO ds5 = dataSet(5, 14 * FileUtils.ONE_KB);
Share share = new Share(shareFolder, 0, freeSpaceProvider);
+ share.setUnarchivingScratchShare(true);
share.addDataSet(ds3);
share.addDataSet(ds2);
share.addDataSet(ds1);
@@ -523,6 +547,79 @@ public class SegmentedStoreUtilsTest extends AbstractFileSystemTestCase
fail();
}
+
+ @Test
+ public void testMoveDataSetToAnotherShareWhichIsAnUnarchivingScratchShare()
+ {
+ File share1 = new File(workingDirectory, "store/1");
+ File share1uuid01 = new File(share1, "uuid/01");
+ File dataSetDirInStore = new File(share1uuid01, "02/03/ds-1");
+ File original = new File(dataSetDirInStore, "original");
+ original.mkdirs();
+ final File helloFile = new File(original, "hello.txt");
+ FileUtilities.writeToFile(helloFile, "hello world");
+ File share2 = new File(workingDirectory, "store/2");
+ share2.mkdirs();
+ FileUtilities.writeToFile(new File(share2, ShareFactory.SHARE_PROPS_FILE),
+ ShareFactory.UNARCHIVING_SCRATCH_SHARE_PROP + "=true");
+ context.checking(new Expectations()
+ {
+ {
+ one(service).tryGetDataSet("ds-1");
+ will(returnValue(new PhysicalDataSet()));
+
+ one(shareIdManager).lock("ds-1");
+ one(shareIdManager).releaseLock("ds-1");
+ }
+ });
+
+ try
+ {
+ SegmentedStoreUtils.moveDataSetToAnotherShare(dataSetDirInStore, share2, service,
+ shareIdManager, checksumProvider, log);
+ fail("EnvironmentFailureException expected");
+ } catch (EnvironmentFailureException ex)
+ {
+ assertEquals("Share '2' is a scratch share for unarchiving purposes. "
+ + "No data sets can be moved from/to such a share.", ex.getMessage());
+ }
+ }
+
+ @Test
+ public void testMoveDataSetFromAnUnarchivingScratchShareToAnotherShare()
+ {
+ File share1 = new File(workingDirectory, "store/1");
+ File share1uuid01 = new File(share1, "uuid/01");
+ File dataSetDirInStore = new File(share1uuid01, "02/03/ds-1");
+ File original = new File(dataSetDirInStore, "original");
+ original.mkdirs();
+ FileUtilities.writeToFile(new File(share1, ShareFactory.SHARE_PROPS_FILE),
+ ShareFactory.UNARCHIVING_SCRATCH_SHARE_PROP + "=true");
+ final File helloFile = new File(original, "hello.txt");
+ FileUtilities.writeToFile(helloFile, "hello world");
+ File share2 = new File(workingDirectory, "store/2");
+ context.checking(new Expectations()
+ {
+ {
+ one(service).tryGetDataSet("ds-1");
+ will(returnValue(new PhysicalDataSet()));
+
+ one(shareIdManager).lock("ds-1");
+ one(shareIdManager).releaseLock("ds-1");
+ }
+ });
+
+ try
+ {
+ SegmentedStoreUtils.moveDataSetToAnotherShare(dataSetDirInStore, share2, service,
+ shareIdManager, checksumProvider, log);
+ fail("EnvironmentFailureException expected");
+ } catch (EnvironmentFailureException ex)
+ {
+ assertEquals("Share '1' is a scratch share for unarchiving purposes. "
+ + "No data sets can be moved from/to such a share.", ex.getMessage());
+ }
+ }
@Test
public void testCleanupOld()
--
GitLab