From 8ad5c15aae35897537220c1b0a5c6d847ff33c3b Mon Sep 17 00:00:00 2001
From: felmer <felmer>
Date: Mon, 14 Feb 2011 13:12:59 +0000
Subject: [PATCH] LMS-1995 renaming 'balance' -> 'shuffling' because only data
sets from incoming shares should be moved.
SVN: 19934
---
.../ch/systemsx/cisd/etlserver/ETLDaemon.java | 15 +++-
.../etlserver/api/v1/PutDataSetService.java | 2 +-
...cer.java => ISegmentedStoreShuffling.java} | 16 ++--
....java => SegmentedStoreShufflingTask.java} | 77 +++++++++++--------
...mpleBalancer.java => SimpleShuffling.java} | 20 ++---
.../shared/utils/SegmentedStoreUtils.java | 17 ++--
.../dss/generic/shared/utils/Share.java | 15 ++++
...a => SegmentedStoreShufflingTaskTest.java} | 70 ++++++++++++-----
...ncerTest.java => SimpleShufflingTest.java} | 13 ++--
9 files changed, 159 insertions(+), 86 deletions(-)
rename datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/{ISegmentedStoreBalancer.java => ISegmentedStoreShuffling.java} (67%)
rename datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/{SegmentedStoreBalancingTask.java => SegmentedStoreShufflingTask.java} (72%)
rename datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/{SimpleBalancer.java => SimpleShuffling.java} (89%)
rename datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/{SegmentedStoreBalancingTaskTest.java => SegmentedStoreShufflingTaskTest.java} (61%)
rename datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/{SimpleBalancerTest.java => SimpleShufflingTest.java} (94%)
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/ETLDaemon.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/ETLDaemon.java
index b6f7cb6d42b..4d0ad76b1f1 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/ETLDaemon.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/ETLDaemon.java
@@ -19,8 +19,11 @@ package ch.systemsx.cisd.etlserver;
import java.io.File;
import java.io.FileFilter;
import java.io.FilenameFilter;
+import java.util.Collections;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Properties;
+import java.util.Set;
import java.util.Timer;
import java.util.concurrent.TimeUnit;
@@ -95,6 +98,13 @@ public final class ETLDaemon
@Private
static IExitHandler exitHandler = SystemExit.SYSTEM_EXIT;
+ private static final Set<String> incomingShares = new LinkedHashSet<String>();
+
+ public static Set<String> getIdsOfIncomingShares()
+ {
+ return Collections.unmodifiableSet(incomingShares);
+ }
+
private static void printInitialLogMessage(final Parameters parameters)
{
operationLog.info("Data Store Server is starting up.");
@@ -210,7 +220,7 @@ public final class ETLDaemon
throw new ConfigurationFailureException(errorMessage);
}
}
-
+
private static void startupServer(final Parameters parameters)
{
final ThreadParameters[] threads = parameters.getThreads();
@@ -223,11 +233,12 @@ public final class ETLDaemon
final Properties mailProperties = Parameters.createMailProperties(properties);
final IMailClient mailClient = new MailClient(mailProperties);
File storeRootDir = DssPropertyParametersUtil.getStoreRootDir(parameters.getProperties());
- File[] shares = SegmentedStoreUtils.getImcomingShares(storeRootDir);
+ File[] shares = SegmentedStoreUtils.getShares(storeRootDir);
for (final ThreadParameters threadParameters : threads)
{
File incomingDataDirectory = threadParameters.getIncomingDataDirectory();
String shareId = SegmentedStoreUtils.findIncomingShare(incomingDataDirectory, shares);
+ incomingShares.add(shareId);
operationLog.info("[" + threadParameters.getThreadName() + "]: Data sets drop into '"
+ incomingDataDirectory + "' will stored in share " + shareId + ".");
createProcessingThread(parameters, threadParameters, shareId,
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/api/v1/PutDataSetService.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/api/v1/PutDataSetService.java
index 474c50f953d..d3bf9eabb9a 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/api/v1/PutDataSetService.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/api/v1/PutDataSetService.java
@@ -202,7 +202,7 @@ public class PutDataSetService
dataSetValidator = initializer.getDataSetValidator();
- File[] shares = SegmentedStoreUtils.getImcomingShares(storeDirectory);
+ File[] shares = SegmentedStoreUtils.getShares(storeDirectory);
if (shares.length == 0)
{
if (new File(storeDirectory, Constants.DEFAULT_SHARE_ID).mkdirs() == false)
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/ISegmentedStoreBalancer.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/ISegmentedStoreShuffling.java
similarity index 67%
rename from datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/ISegmentedStoreBalancer.java
rename to datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/ISegmentedStoreShuffling.java
index faaef689d9c..bff8e410ac3 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/ISegmentedStoreBalancer.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/ISegmentedStoreShuffling.java
@@ -23,18 +23,16 @@ import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
import ch.systemsx.cisd.openbis.dss.generic.shared.utils.Share;
/**
- * Strategy of balancing data sets between shares in segmented store.
- *
+ * Strategy of shuffling data sets from source shares to target shares. Source shares are incoming
+ * shares. Target shares are all shares.
+ *
* @author Franz-Josef Elmer
*/
-public interface ISegmentedStoreBalancer
+public interface ISegmentedStoreShuffling
{
/**
- * Balances sizes of specified shares by moving data sets between them using specified data set mover.
- *
- * @param service
- *
+ * Moves data sets from source shares to some target shares if necessary.
*/
- public void balanceStore(List<Share> shares, IEncapsulatedOpenBISService service,
- IDataSetMover dataSetMover, ISimpleLogger logger);
+ public void shuffleDataSets(List<Share> sourceShares, List<Share> targetShares,
+ IEncapsulatedOpenBISService service, IDataSetMover dataSetMover, ISimpleLogger logger);
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreBalancingTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreShufflingTask.java
similarity index 72%
rename from datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreBalancingTask.java
rename to datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreShufflingTask.java
index 8f0c48c8385..71fa7dd50fb 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreBalancingTask.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreShufflingTask.java
@@ -19,8 +19,10 @@ package ch.systemsx.cisd.etlserver.plugins;
import static ch.systemsx.cisd.common.logging.LogLevel.INFO;
import java.io.File;
+import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
+import java.util.Set;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.Logger;
@@ -40,6 +42,7 @@ import ch.systemsx.cisd.common.maintenance.IMaintenanceTask;
import ch.systemsx.cisd.common.utilities.ClassUtils;
import ch.systemsx.cisd.common.utilities.PropertyParametersUtil;
import ch.systemsx.cisd.common.utilities.PropertyUtils;
+import ch.systemsx.cisd.etlserver.ETLDaemon;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.utils.DssPropertyParametersUtil;
@@ -48,26 +51,28 @@ import ch.systemsx.cisd.openbis.dss.generic.shared.utils.Share;
import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
/**
- * Maintenance task which tries to balance a segmented store.
- *
+ * Maintenance task which shuffles data sets between shares of a segmented store. This task is
+ * supposed to prevent incoming shares from having not enough space.
+ *
* @author Franz-Josef Elmer
*/
-public class SegmentedStoreBalancingTask implements IMaintenanceTask
+public class SegmentedStoreShufflingTask implements IMaintenanceTask
{
- private static final ISegmentedStoreBalancer DUMMY_BALANCER = new ISegmentedStoreBalancer()
+ private static final ISegmentedStoreShuffling DUMMY_SHUFFLING = new ISegmentedStoreShuffling()
{
private static final int N = 3;
- public void balanceStore(List<Share> shares, IEncapsulatedOpenBISService service,
- IDataSetMover dataSetMover, ISimpleLogger logger)
+ public void shuffleDataSets(List<Share> sourceShares, List<Share> targetShares,
+ IEncapsulatedOpenBISService service, IDataSetMover dataSetMover,
+ ISimpleLogger logger)
{
logger.log(INFO, "Data Store Shares:");
- for (Share share : shares)
+ for (Share share : targetShares)
{
List<SimpleDataSetInformationDTO> dataSets = share.getDataSetsOrderedBySize();
logger.log(
INFO,
- " Share "
+ " " + (share.isIncoming() ? "Incoming" : "External") + " share "
+ share.getShareId()
+ " (free space: "
+ FileUtils.byteCountToDisplaySize(share.calculateFreeSpace())
@@ -95,11 +100,13 @@ public class SegmentedStoreBalancingTask implements IMaintenanceTask
}
};
- @Private static final String BALANCER_SECTION_NAME = "balancer";
+ @Private static final String SHUFFLING_SECTION_NAME = "shuffling";
@Private static final String CLASS_PROPERTY_NAME = "class";
private static final Logger operationLog =
- LogFactory.getLogger(LogCategory.OPERATION, SegmentedStoreBalancingTask.class);
+ LogFactory.getLogger(LogCategory.OPERATION, SegmentedStoreShufflingTask.class);
+
+ private final Set<String> incomingShares;
private final IEncapsulatedOpenBISService service;
private final IDataSetMover dataSetMover;
@@ -108,12 +115,12 @@ public class SegmentedStoreBalancingTask implements IMaintenanceTask
private File storeRoot;
private String dataStoreCode;
- @Private ISegmentedStoreBalancer balancer;
-
- public SegmentedStoreBalancingTask()
+ @Private ISegmentedStoreShuffling shuffling;
+
+ public SegmentedStoreShufflingTask()
{
- this(ServiceProvider.getOpenBISService(), new SimpleFreeSpaceProvider(),
- new IDataSetMover()
+ this(ETLDaemon.getIdsOfIncomingShares(), ServiceProvider.getOpenBISService(),
+ new SimpleFreeSpaceProvider(), new IDataSetMover()
{
public void moveDataSetToAnotherShare(File dataSetDirInStore, File share)
@@ -124,9 +131,10 @@ public class SegmentedStoreBalancingTask implements IMaintenanceTask
}, new Log4jSimpleLogger(operationLog));
}
- SegmentedStoreBalancingTask(final IEncapsulatedOpenBISService service,
+ SegmentedStoreShufflingTask(Set<String> incomingShares, IEncapsulatedOpenBISService service,
IFreeSpaceProvider freeSpaceProvider, IDataSetMover dataSetMover, ISimpleLogger logger)
{
+ this.incomingShares = incomingShares;
LogInitializer.init();
this.freeSpaceProvider = freeSpaceProvider;
this.service = service;
@@ -148,43 +156,52 @@ public class SegmentedStoreBalancingTask implements IMaintenanceTask
"Store root does not exists or is not a directory: "
+ storeRoot.getAbsolutePath());
}
- balancer = createBalancer(properties);
- operationLog.info("Plugin '" + pluginName + "' initialized: balancer: "
- + balancer.getClass().getName() + ", data store code: " + dataStoreCode
- + ", data store root: " + storeRoot.getAbsolutePath());
+ shuffling = createShuffling(properties);
+ operationLog.info("Plugin '" + pluginName + "' initialized: shuffling strategy: "
+ + shuffling.getClass().getName() + ", data store code: " + dataStoreCode
+ + ", data store root: " + storeRoot.getAbsolutePath() + ", incoming shares: "
+ + incomingShares);
}
- private ISegmentedStoreBalancer createBalancer(Properties properties)
+ private ISegmentedStoreShuffling createShuffling(Properties properties)
{
- Properties balancerProps =
+ Properties shufflingProps =
PropertyParametersUtil.extractSingleSectionProperties(properties,
- BALANCER_SECTION_NAME, false).getProperties();
- String className = balancerProps.getProperty(CLASS_PROPERTY_NAME);
+ SHUFFLING_SECTION_NAME, false).getProperties();
+ String className = shufflingProps.getProperty(CLASS_PROPERTY_NAME);
if (className == null)
{
- return DUMMY_BALANCER;
+ return DUMMY_SHUFFLING;
}
try
{
- return ClassUtils.create(ISegmentedStoreBalancer.class, className, balancerProps);
+ return ClassUtils.create(ISegmentedStoreShuffling.class, className, shufflingProps);
} catch (ConfigurationFailureException ex)
{
throw ex;
} catch (Exception ex)
{
- throw new ConfigurationFailureException("Cannot find balancer class '" + className
+ throw new ConfigurationFailureException("Cannot find shuffling class '" + className
+ "'", CheckedExceptionTunnel.unwrapIfNecessary(ex));
}
}
public void execute()
{
- operationLog.info("Starting segmented store balancing.");
+ operationLog.info("Starting segmented store shuffling.");
List<Share> shares =
SegmentedStoreUtils.getDataSetsPerShare(storeRoot, dataStoreCode,
freeSpaceProvider, service, operationLogger);
- balancer.balanceStore(shares, service, dataSetMover, operationLogger);
- operationLog.info("Segmented store balancing finished.");
+ List<Share> sourceShares = new ArrayList<Share>();
+ for (Share share : shares)
+ {
+ if (incomingShares.contains(share.getShareId()))
+ {
+ sourceShares.add(share);
+ }
+ }
+ shuffling.shuffleDataSets(sourceShares, shares, service, dataSetMover, operationLogger);
+ operationLog.info("Segmented store shuffling finished.");
}
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SimpleBalancer.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SimpleShuffling.java
similarity index 89%
rename from datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SimpleBalancer.java
rename to datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SimpleShuffling.java
index 91413247248..164aae82c99 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SimpleBalancer.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SimpleShuffling.java
@@ -37,12 +37,12 @@ import ch.systemsx.cisd.openbis.dss.generic.shared.utils.Share;
import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
/**
- * Simple balancer which moves data sets from full shares to the share with initial most free space
+ * Simple shuffling which moves data sets from full shares to the share with initial most free space
* until it is full.
*
* @author Franz-Josef Elmer
*/
-public class SimpleBalancer implements ISegmentedStoreBalancer
+public class SimpleShuffling implements ISegmentedStoreShuffling
{
@Private static final String MINIMUM_FREE_SPACE_KEY = "minimum-free-space-in-MB";
@@ -94,12 +94,12 @@ public class SimpleBalancer implements ISegmentedStoreBalancer
private final long minimumFreeSpace;
private final ITimeProvider timeProvider;
- public SimpleBalancer(Properties properties)
+ public SimpleShuffling(Properties properties)
{
this(properties, SystemTimeProvider.SYSTEM_TIME_PROVIDER);
}
- SimpleBalancer(Properties properties, ITimeProvider timeProvider)
+ SimpleShuffling(Properties properties, ITimeProvider timeProvider)
{
this.timeProvider = timeProvider;
minimumFreeSpace =
@@ -108,12 +108,12 @@ public class SimpleBalancer implements ISegmentedStoreBalancer
}
- public void balanceStore(List<Share> shares, IEncapsulatedOpenBISService service,
- IDataSetMover dataSetMover, ISimpleLogger logger)
+ public void shuffleDataSets(List<Share> sourceShares, List<Share> targetShares,
+ IEncapsulatedOpenBISService service, IDataSetMover dataSetMover, ISimpleLogger logger)
{
- List<ShareState> shareStates = getSortedShares(shares);
+ List<ShareState> shareStates = getSortedShares(targetShares);
ShareState shareWithMostFree = shareStates.get(shareStates.size() - 1);
- List<ShareState> fullShares = getFullShares(shareStates);
+ List<ShareState> fullShares = getFullShares(sourceShares);
for (ShareState fullShare : fullShares)
{
List<SimpleDataSetInformationDTO> dataSets =
@@ -172,10 +172,10 @@ public class SimpleBalancer implements ISegmentedStoreBalancer
+ ((timeProvider.getTimeInMilliseconds() - t0 + 500) / 1000) + " seconds.");
}
- private List<ShareState> getFullShares(List<ShareState> shareStates)
+ private List<ShareState> getFullShares(List<Share> sourceShares)
{
List<ShareState> fullShares = new ArrayList<ShareState>();
- for (ShareState shareState : shareStates)
+ for (ShareState shareState : getSortedShares(sourceShares))
{
if (shareState.getFreeSpace() < minimumFreeSpace)
{
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java
index 35a18a3a586..c280683802e 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java
@@ -54,9 +54,9 @@ public class SegmentedStoreUtils
{
private static final String RSYNC_EXEC = "rsync";
- private static final Pattern INCOMING_SHARE_ID_PATTERN = Pattern.compile("[0-9]+");
+ private static final Pattern SHARE_ID_PATTERN = Pattern.compile("[0-9]+");
- private static final FileFilter FILTER_ON_INCOMING_SHARES = new FileFilter()
+ private static final FileFilter FILTER_ON_SHARES = new FileFilter()
{
public boolean accept(File pathname)
{
@@ -65,16 +65,16 @@ public class SegmentedStoreUtils
return false;
}
String name = pathname.getName();
- return INCOMING_SHARE_ID_PATTERN.matcher(name).matches();
+ return SHARE_ID_PATTERN.matcher(name).matches();
}
};
/**
- * Lists all folders in specified store root directory which match incoming share pattern.
+ * Lists all folders in specified store root directory which match share pattern.
*/
- public static File[] getImcomingShares(File storeRootDir)
+ public static File[] getShares(File storeRootDir)
{
- File[] files = storeRootDir.listFiles(SegmentedStoreUtils.FILTER_ON_INCOMING_SHARES);
+ File[] files = storeRootDir.listFiles(SegmentedStoreUtils.FILTER_ON_SHARES);
Arrays.sort(files);
return files;
}
@@ -95,7 +95,7 @@ public class SegmentedStoreUtils
throw new ConfigurationFailureException(
"Store root does not exist or is not a folder: " + storeRoot);
}
- return findIncomingShare(incomingFolder, storeRoot.listFiles(FILTER_ON_INCOMING_SHARES));
+ return findIncomingShare(incomingFolder, storeRoot.listFiles(FILTER_ON_SHARES));
}
/**
@@ -129,6 +129,7 @@ public class SegmentedStoreUtils
return share;
}
}
+ testFile.delete();
throw new ConfigurationFailureException(
"Now share could be found for the following incoming folder: "
+ testFile.getParentFile().getAbsolutePath());
@@ -157,7 +158,7 @@ public class SegmentedStoreUtils
ISimpleLogger log, ITimeProvider timeProvider)
{
Map<String, Share> shares = new HashMap<String, Share>();
- for (File file : getImcomingShares(storeRoot))
+ for (File file : getShares(storeRoot))
{
Share share = new Share(file, freeSpaceProvider);
shares.put(share.getShareId(), share);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/Share.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/Share.java
index 7ced2d35549..57efcd00603 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/Share.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/Share.java
@@ -56,6 +56,8 @@ public final class Share
private final List<SimpleDataSetInformationDTO> dataSets =
new ArrayList<SimpleDataSetInformationDTO>();
+
+ private boolean incoming;
private long size;
@@ -74,6 +76,19 @@ public final class Share
return shareId;
}
+ public void setIncoming(boolean incoming)
+ {
+ this.incoming = incoming;
+ }
+
+ /**
+ * Returns <code>true</code> if this is an incoming share.
+ */
+ public boolean isIncoming()
+ {
+ return incoming;
+ }
+
/**
* Returns the root directory of this share.
*/
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreBalancingTaskTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreShufflingTaskTest.java
similarity index 61%
rename from datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreBalancingTaskTest.java
rename to datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreShufflingTaskTest.java
index 5416e4dd823..08d4f29f57b 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreBalancingTaskTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreShufflingTaskTest.java
@@ -16,11 +16,12 @@
package ch.systemsx.cisd.etlserver.plugins;
-import static ch.systemsx.cisd.etlserver.plugins.SegmentedStoreBalancingTask.BALANCER_SECTION_NAME;
-import static ch.systemsx.cisd.etlserver.plugins.SegmentedStoreBalancingTask.CLASS_PROPERTY_NAME;
+import static ch.systemsx.cisd.etlserver.plugins.SegmentedStoreShufflingTask.SHUFFLING_SECTION_NAME;
+import static ch.systemsx.cisd.etlserver.plugins.SegmentedStoreShufflingTask.CLASS_PROPERTY_NAME;
import java.io.File;
import java.util.Arrays;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Properties;
@@ -32,6 +33,7 @@ import org.testng.annotations.Test;
import ch.rinn.restrictions.Friend;
import ch.systemsx.cisd.base.tests.AbstractFileSystemTestCase;
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.common.filesystem.IFreeSpaceProvider;
import ch.systemsx.cisd.common.logging.ISimpleLogger;
import ch.systemsx.cisd.common.logging.LogLevel;
@@ -45,13 +47,16 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
*
* @author Franz-Josef Elmer
*/
-@Friend(toClasses=SegmentedStoreBalancingTask.class)
-public class SegmentedStoreBalancingTaskTest extends AbstractFileSystemTestCase
+@Friend(toClasses=SegmentedStoreShufflingTask.class)
+public class SegmentedStoreShufflingTaskTest extends AbstractFileSystemTestCase
{
- public static final class Balancer implements ISegmentedStoreBalancer
+ private static final String DATA_STORE_CODE = "data-store-1";
+
+ public static final class Balancer implements ISegmentedStoreShuffling
{
private final Properties properties;
- private List<Share> shares;
+ private List<Share> sourceShares;
+ private List<Share> targetShares;
private IEncapsulatedOpenBISService service;
private IDataSetMover dataSetMover;
private ISimpleLogger logger;
@@ -61,10 +66,12 @@ public class SegmentedStoreBalancingTaskTest extends AbstractFileSystemTestCase
this.properties = properties;
}
- public void balanceStore(List<Share> list, IEncapsulatedOpenBISService openBisService,
- IDataSetMover mover, ISimpleLogger simpleLogger)
+ public void shuffleDataSets(List<Share> sources, List<Share> targets,
+ IEncapsulatedOpenBISService openBisService, IDataSetMover mover,
+ ISimpleLogger simpleLogger)
{
- shares = list;
+ sourceShares = sources;
+ targetShares = targets;
service = openBisService;
dataSetMover = mover;
logger = simpleLogger;
@@ -76,7 +83,7 @@ public class SegmentedStoreBalancingTaskTest extends AbstractFileSystemTestCase
private IFreeSpaceProvider spaceProvider;
private IDataSetMover dataSetMover;
private ISimpleLogger logger;
- private SegmentedStoreBalancingTask balancerTask;
+ private SegmentedStoreShufflingTask balancerTask;
private File storeRoot;
@BeforeMethod
@@ -87,7 +94,10 @@ public class SegmentedStoreBalancingTaskTest extends AbstractFileSystemTestCase
spaceProvider = context.mock(IFreeSpaceProvider.class);
dataSetMover = context.mock(IDataSetMover.class);
logger = context.mock(ISimpleLogger.class);
- balancerTask = new SegmentedStoreBalancingTask(service, spaceProvider, dataSetMover, logger);
+ LinkedHashSet<String> incomingShareIds = new LinkedHashSet<String>(Arrays.asList("1"));
+ balancerTask =
+ new SegmentedStoreShufflingTask(incomingShareIds, service, spaceProvider,
+ dataSetMover, logger);
storeRoot = new File(workingDirectory, "store");
storeRoot.mkdirs();
}
@@ -104,27 +114,47 @@ public class SegmentedStoreBalancingTaskTest extends AbstractFileSystemTestCase
public void testExecute()
{
Properties properties = new Properties();
- properties.setProperty(DssPropertyParametersUtil.DSS_CODE_KEY, "data-store-1");
+ properties.setProperty(DssPropertyParametersUtil.DSS_CODE_KEY, DATA_STORE_CODE);
properties.setProperty(DssPropertyParametersUtil.STOREROOT_DIR_KEY, storeRoot.getPath());
- properties.setProperty(BALANCER_SECTION_NAME + "." + CLASS_PROPERTY_NAME,
- SegmentedStoreBalancingTaskTest.Balancer.class.getName());
+ properties.setProperty(SHUFFLING_SECTION_NAME + "." + CLASS_PROPERTY_NAME,
+ SegmentedStoreShufflingTaskTest.Balancer.class.getName());
balancerTask.setUp("mock-balancer", properties);
+ File share1 = new File(storeRoot, "1");
+ share1.mkdirs();
+ FileUtilities.writeToFile(new File(share1, "ds1"), "hello ds1");
+ File share2 = new File(storeRoot, "2");
+ share2.mkdirs();
+ FileUtilities.writeToFile(new File(share2, "ds2"), "hello ds2");
context.checking(new Expectations()
{
{
one(service).listDataSets();
- SimpleDataSetInformationDTO ds = new SimpleDataSetInformationDTO();
- ds.setDataStoreCode("other data store");
- will(returnValue(Arrays.asList(ds)));
+ SimpleDataSetInformationDTO ds1 = new SimpleDataSetInformationDTO();
+ ds1.setDataStoreCode(DATA_STORE_CODE);
+ ds1.setDataSetShareId("1");
+ ds1.setDataSetSize(10l);
+ ds1.setDataSetLocation("ds1");
+ SimpleDataSetInformationDTO ds2 = new SimpleDataSetInformationDTO();
+ ds2.setDataStoreCode(DATA_STORE_CODE);
+ ds2.setDataSetShareId("2");
+ ds2.setDataSetSize(20l);
+ ds2.setDataSetLocation("ds2");
+ SimpleDataSetInformationDTO ds3 = new SimpleDataSetInformationDTO();
+ ds3.setDataStoreCode("other data store");
+ will(returnValue(Arrays.asList(ds1, ds2, ds3)));
}
});
balancerTask.execute();
- Balancer balancer = (SegmentedStoreBalancingTaskTest.Balancer) balancerTask.balancer;
+ Balancer balancer = (SegmentedStoreShufflingTaskTest.Balancer) balancerTask.shuffling;
assertEquals("{class=" + balancer.getClass().getName() + "}",
balancer.properties.toString());
- assertEquals("[]", balancer.shares.toString());
+ assertEquals("1", balancer.sourceShares.get(0).getShareId());
+ assertEquals(1, balancer.sourceShares.size());
+ assertEquals("1", balancer.targetShares.get(0).getShareId());
+ assertEquals("2", balancer.targetShares.get(1).getShareId());
+ assertEquals(2, balancer.targetShares.size());
assertSame(service, balancer.service);
assertSame(dataSetMover, balancer.dataSetMover);
assertSame(logger, balancer.logger);
@@ -135,7 +165,7 @@ public class SegmentedStoreBalancingTaskTest extends AbstractFileSystemTestCase
public void testDefaultBalancer()
{
Properties properties = new Properties();
- properties.setProperty(DssPropertyParametersUtil.DSS_CODE_KEY, "data-store-1");
+ properties.setProperty(DssPropertyParametersUtil.DSS_CODE_KEY, DATA_STORE_CODE);
properties.setProperty(DssPropertyParametersUtil.STOREROOT_DIR_KEY, storeRoot.getPath());
balancerTask.setUp("mock-balancer", properties);
context.checking(new Expectations()
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/SimpleBalancerTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/SimpleShufflingTest.java
similarity index 94%
rename from datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/SimpleBalancerTest.java
rename to datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/SimpleShufflingTest.java
index 9d782b231d2..a740f523bab 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/SimpleBalancerTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/SimpleShufflingTest.java
@@ -52,8 +52,8 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
*
* @author Franz-Josef Elmer
*/
-@Friend(toClasses=SimpleBalancer.class)
-public class SimpleBalancerTest extends AbstractFileSystemTestCase
+@Friend(toClasses=SimpleShuffling.class)
+public class SimpleShufflingTest extends AbstractFileSystemTestCase
{
private static final String STORE_PATH = "01/02/03/";
@@ -98,7 +98,7 @@ public class SimpleBalancerTest extends AbstractFileSystemTestCase
private IEncapsulatedOpenBISService service;
private IDataSetMover dataSetMover;
private ISimpleLogger logger;
- private SimpleBalancer balancer;
+ private SimpleShuffling balancer;
private File store;
@BeforeMethod
@@ -118,8 +118,8 @@ public class SimpleBalancerTest extends AbstractFileSystemTestCase
}
});
Properties properties = new Properties();
- properties.setProperty(SimpleBalancer.MINIMUM_FREE_SPACE_KEY, "2");
- balancer = new SimpleBalancer(properties, timeProvider);
+ properties.setProperty(SimpleShuffling.MINIMUM_FREE_SPACE_KEY, "2");
+ balancer = new SimpleShuffling(properties, timeProvider);
store = new File(workingDirectory, "store");
}
@@ -163,7 +163,8 @@ public class SimpleBalancerTest extends AbstractFileSystemTestCase
}
});
- balancer.balanceStore(Arrays.asList(share1, share2, share3, share4), service, dataSetMover, logger);
+ balancer.shuffleDataSets(Arrays.asList(share1, share2),
+ Arrays.asList(share1, share2, share3, share4), service, dataSetMover, logger);
assertEquals(1, share1.getDataSetsOrderedBySize().size());
assertEquals(1, share2.getDataSetsOrderedBySize().size());
--
GitLab