From f9640aec587b68749ef2086136d2a613cc4e4e85 Mon Sep 17 00:00:00 2001
From: felmer <felmer>
Date: Mon, 4 Apr 2011 12:00:32 +0000
Subject: [PATCH] LMS-2125 TaskExecutor introduced. It will be used by
PostRegistrationMaintenanceTask as well as SegmentedtStoreShufflingTask. It
handle cleanup tasks.
SVN: 20639
---
.../plugins/ISegmentedStoreShuffling.java | 5 +
.../plugins/SegmentedStoreShufflingTask.java | 5 +
.../etlserver/plugins/SimpleShuffling.java | 103 ++++-------
.../plugins/SimpleShufflingShareFinder.java | 69 ++++++++
.../ArchivingPostRegistrationTask.java | 5 +-
.../postregistration/EagerShufflingTask.java | 63 +++----
.../postregistration/ICleanupTask.java | 6 +-
.../postregistration/NoCleanupTask.java | 4 +-
.../PostRegistrationMaintenanceTask.java | 109 +-----------
.../postregistration/TaskExecutor.java | 161 ++++++++++++++++++
.../shared/utils/SegmentedStoreUtils.java | 31 +++-
.../SegmentedStoreShufflingTaskTest.java | 9 +-
.../plugins/SimpleShufflingTest.java | 111 ++++++++----
.../EagerShufflingTaskTest.java | 4 +-
.../PostRegistrationMaintenanceTaskTest.java | 13 +-
15 files changed, 435 insertions(+), 263 deletions(-)
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SimpleShufflingShareFinder.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/TaskExecutor.java
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/ISegmentedStoreShuffling.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/ISegmentedStoreShuffling.java
index bff8e410ac3..a3ac7dd27bf 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/ISegmentedStoreShuffling.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/ISegmentedStoreShuffling.java
@@ -30,6 +30,11 @@ import ch.systemsx.cisd.openbis.dss.generic.shared.utils.Share;
*/
public interface ISegmentedStoreShuffling
{
+ /**
+ * Initialize this instance.
+ */
+ public void init(ISimpleLogger logger);
+
/**
* Moves data sets from source shares to some target shares if necessary.
*/
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreShufflingTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreShufflingTask.java
index dd09498bc21..e3424168f40 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreShufflingTask.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreShufflingTask.java
@@ -62,6 +62,10 @@ public class SegmentedStoreShufflingTask implements IDataStoreLockingMaintenance
{
private static final int N = 3;
+ public void init(ISimpleLogger logger)
+ {
+ }
+
public void shuffleDataSets(List<Share> sourceShares, List<Share> targetShares,
IEncapsulatedOpenBISService service, IDataSetMover dataSetMover,
ISimpleLogger logger)
@@ -162,6 +166,7 @@ public class SegmentedStoreShufflingTask implements IDataStoreLockingMaintenance
+ storeRoot.getAbsolutePath());
}
shuffling = createShuffling(properties);
+ shuffling.init(operationLogger);
operationLog.info("Plugin '" + pluginName + "' initialized: shuffling strategy: "
+ shuffling.getClass().getName() + ", data store code: " + dataStoreCode
+ ", data store root: " + storeRoot.getAbsolutePath() + ", incoming shares: "
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SimpleShuffling.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SimpleShuffling.java
index 97d054c0a4a..550f732bd37 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SimpleShuffling.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SimpleShuffling.java
@@ -18,7 +18,6 @@ package ch.systemsx.cisd.etlserver.plugins;
import static ch.systemsx.cisd.common.logging.LogLevel.INFO;
-import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@@ -28,11 +27,15 @@ import java.util.Properties;
import org.apache.commons.io.FileUtils;
import ch.rinn.restrictions.Private;
+import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
import ch.systemsx.cisd.common.logging.ISimpleLogger;
-import ch.systemsx.cisd.common.utilities.ITimeProvider;
+import ch.systemsx.cisd.common.logging.LogLevel;
import ch.systemsx.cisd.common.utilities.PropertyUtils;
-import ch.systemsx.cisd.common.utilities.SystemTimeProvider;
+import ch.systemsx.cisd.etlserver.postregistration.EagerShufflingTask;
+import ch.systemsx.cisd.etlserver.postregistration.IPostRegistrationTask;
+import ch.systemsx.cisd.etlserver.postregistration.TaskExecutor;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.utils.Share;
import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
@@ -47,20 +50,15 @@ public class SimpleShuffling implements ISegmentedStoreShuffling
@Private
static final String MINIMUM_FREE_SPACE_KEY = "minimum-free-space-in-MB";
- private static final class ShareState
+ private static final class ShareAndFreeSpace
{
private final Share share;
private long freeSpace;
- ShareState(Share share)
+ ShareAndFreeSpace(Share share)
{
this.share = share;
- recalculateFreeSpace();
- }
-
- private void recalculateFreeSpace()
- {
freeSpace = share.calculateFreeSpace();
}
@@ -73,50 +71,38 @@ public class SimpleShuffling implements ISegmentedStoreShuffling
{
return share;
}
-
- void removeDataSet(int dataSetIndex)
- {
- share.getDataSetsOrderedBySize().remove(dataSetIndex);
- recalculateFreeSpace();
- }
-
- void addDataSet(SimpleDataSetInformationDTO dataSet)
- {
- List<SimpleDataSetInformationDTO> dataSets = share.getDataSetsOrderedBySize();
- int index = Collections.binarySearch(dataSets, dataSet, Share.DATA_SET_SIZE_COMPARATOR);
- if (index < 0)
- {
- index = -index - 1;
- }
- dataSets.add(index, dataSet);
- recalculateFreeSpace();
- }
}
private final long minimumFreeSpace;
- private final ITimeProvider timeProvider;
+ private IPostRegistrationTask shufflingTask;
+
+ private TaskExecutor taskExecutor;
public SimpleShuffling(Properties properties)
{
- this(properties, SystemTimeProvider.SYSTEM_TIME_PROVIDER);
+ this(properties, new EagerShufflingTask(properties, ServiceProvider.getOpenBISService()));
}
- SimpleShuffling(Properties properties, ITimeProvider timeProvider)
+ SimpleShuffling(Properties properties, IPostRegistrationTask shufflingTask)
{
- this.timeProvider = timeProvider;
+ this.shufflingTask = shufflingTask;
minimumFreeSpace =
FileUtils.ONE_MB * PropertyUtils.getLong(properties, MINIMUM_FREE_SPACE_KEY, 1024);
+ taskExecutor = new TaskExecutor(properties);
+ }
+ public void init(ISimpleLogger logger)
+ {
+ taskExecutor.cleanup();
+ logger.log(LogLevel.INFO, "Simple shuffling strategy initialized");
}
public void shuffleDataSets(List<Share> sourceShares, List<Share> targetShares,
IEncapsulatedOpenBISService service, IDataSetMover dataSetMover, ISimpleLogger logger)
{
- List<ShareState> shareStates = getSortedShares(targetShares);
- ShareState shareWithMostFree = shareStates.get(shareStates.size() - 1);
- List<ShareState> fullShares = getFullShares(sourceShares);
- for (ShareState fullShare : fullShares)
+ List<ShareAndFreeSpace> fullShares = getFullShares(sourceShares);
+ for (ShareAndFreeSpace fullShare : fullShares)
{
List<SimpleDataSetInformationDTO> dataSets =
fullShare.getShare().getDataSetsOrderedBySize();
@@ -135,10 +121,13 @@ public class SimpleShuffling implements ISegmentedStoreShuffling
}
for (int i = 0; i < numberOfDataSetsToMove; i++)
{
- long dataSetSize = dataSets.get(i).getDataSetSize();
- if (shareWithMostFree.getFreeSpace() - dataSetSize > minimumFreeSpace)
+ SimpleDataSetInformationDTO dataSet = dataSets.get(i);
+ try
{
- copy(fullShare, 0, shareWithMostFree, dataSetMover, logger);
+ taskExecutor.execute(shufflingTask, "shuffling", dataSet.getDataSetCode());
+ } catch (Throwable ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
}
}
}
@@ -170,30 +159,10 @@ public class SimpleShuffling implements ISegmentedStoreShuffling
return freeSpaceAboveMinimum > 0 ? dataSets.size() : -1;
}
- private void copy(ShareState from, int dataSetIndex, ShareState to, IDataSetMover mover,
- ISimpleLogger logger)
- {
- Share fromShare = from.getShare();
- Share toShare = to.getShare();
- SimpleDataSetInformationDTO dataSet =
- fromShare.getDataSetsOrderedBySize().get(dataSetIndex);
- File dataSetDirInStore = new File(fromShare.getShare(), dataSet.getDataSetLocation());
- String commonMessage =
- "Copying data set " + dataSet.getDataSetCode() + " from share "
- + fromShare.getShareId() + " to share " + toShare.getShareId();
- logger.log(INFO, commonMessage + " ...");
- long t0 = timeProvider.getTimeInMilliseconds();
- mover.moveDataSetToAnotherShare(dataSetDirInStore, toShare.getShare(), logger);
- from.removeDataSet(dataSetIndex);
- to.addDataSet(dataSet);
- logger.log(INFO, commonMessage + " took "
- + ((timeProvider.getTimeInMilliseconds() - t0 + 500) / 1000) + " seconds.");
- }
-
- private List<ShareState> getFullShares(List<Share> sourceShares)
+ private List<ShareAndFreeSpace> getFullShares(List<Share> sourceShares)
{
- List<ShareState> fullShares = new ArrayList<ShareState>();
- for (ShareState shareState : getSortedShares(sourceShares))
+ List<ShareAndFreeSpace> fullShares = new ArrayList<ShareAndFreeSpace>();
+ for (ShareAndFreeSpace shareState : getSortedShares(sourceShares))
{
if (shareState.getFreeSpace() < minimumFreeSpace)
{
@@ -203,16 +172,16 @@ public class SimpleShuffling implements ISegmentedStoreShuffling
return fullShares;
}
- private List<ShareState> getSortedShares(List<Share> shares)
+ private List<ShareAndFreeSpace> getSortedShares(List<Share> shares)
{
- List<ShareState> shareStates = new ArrayList<ShareState>();
+ List<ShareAndFreeSpace> shareStates = new ArrayList<ShareAndFreeSpace>();
for (Share share : shares)
{
- shareStates.add(new ShareState(share));
+ shareStates.add(new ShareAndFreeSpace(share));
}
- Collections.sort(shareStates, new Comparator<ShareState>()
+ Collections.sort(shareStates, new Comparator<ShareAndFreeSpace>()
{
- public int compare(ShareState o1, ShareState o2)
+ public int compare(ShareAndFreeSpace o1, ShareAndFreeSpace o2)
{
long s1 = o1.getFreeSpace();
long s2 = o2.getFreeSpace();
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SimpleShufflingShareFinder.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SimpleShufflingShareFinder.java
new file mode 100644
index 00000000000..4c85b6ac0f9
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SimpleShufflingShareFinder.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.plugins;
+
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.commons.io.FileUtils;
+
+import ch.rinn.restrictions.Private;
+import ch.systemsx.cisd.common.utilities.PropertyUtils;
+import ch.systemsx.cisd.etlserver.postregistration.IShareFinder;
+import ch.systemsx.cisd.openbis.dss.generic.shared.utils.Share;
+import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
+
+/**
+ * Find the share with most free space above a specified minimum space.
+ *
+ * @author Franz-Josef Elmer
+ */
+public class SimpleShufflingShareFinder implements IShareFinder
+{
+ @Private
+ public static final String MINIMUM_FREE_SPACE_KEY = "minimum-free-space-in-MB";
+
+ private final long minimumFreeSpace;
+
+ public SimpleShufflingShareFinder(Properties properties)
+ {
+ minimumFreeSpace =
+ FileUtils.ONE_MB * PropertyUtils.getLong(properties, MINIMUM_FREE_SPACE_KEY, 1024);
+
+ }
+
+ public Share tryToFindShare(SimpleDataSetInformationDTO dataSet, List<Share> shares)
+ {
+ Share shareWithMostFree = null;
+ long maxFreeSpace = 0;
+ for (Share share : shares)
+ {
+ long freeSpace = share.calculateFreeSpace();
+ if (freeSpace > maxFreeSpace)
+ {
+ maxFreeSpace = freeSpace;
+ shareWithMostFree = share;
+ }
+ }
+ if (maxFreeSpace - dataSet.getDataSetSize() > minimumFreeSpace)
+ {
+ return shareWithMostFree;
+ }
+ return null;
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/ArchivingPostRegistrationTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/ArchivingPostRegistrationTask.java
index 4fa56b6fc22..c7cd4fe03f2 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/ArchivingPostRegistrationTask.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/ArchivingPostRegistrationTask.java
@@ -25,8 +25,10 @@ import java.util.Properties;
import org.apache.log4j.Logger;
+import ch.systemsx.cisd.common.logging.ISimpleLogger;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.logging.LogLevel;
import ch.systemsx.cisd.openbis.dss.generic.shared.ArchiverTaskContext;
import ch.systemsx.cisd.openbis.dss.generic.shared.IArchiverPlugin;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
@@ -148,7 +150,7 @@ public class ArchivingPostRegistrationTask extends AbstractPostRegistrationTask
this.dataSetCode = dataSetCode;
}
- public void cleanup()
+ public void cleanup(ISimpleLogger logger)
{
IEncapsulatedOpenBISService openBISService = ServiceProvider.getOpenBISService();
boolean statusUpdated = openBISService
@@ -168,6 +170,7 @@ public class ArchivingPostRegistrationTask extends AbstractPostRegistrationTask
new DeletedDataSet(dataSetCode, dataSet.getDataSetLocation(), 0);
List<DeletedDataSet> dataSetAsList = Collections.singletonList(deletedDataset);
archiver.deleteFromArchive(dataSetAsList);
+ logger.log(LogLevel.INFO, "Data set " + dataSetCode + " deleted from archive.");
}
}
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/EagerShufflingTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/EagerShufflingTask.java
index 752bbcfbefd..927e0f9da3c 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/EagerShufflingTask.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/EagerShufflingTask.java
@@ -52,7 +52,7 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
*/
public class EagerShufflingTask extends AbstractPostRegistrationTask
{
- @Private static final String SHARE_FINDER_KEY = "share-finder";
+ @Private public static final String SHARE_FINDER_KEY = "share-finder";
private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
EagerShufflingTask.class);
@@ -97,7 +97,7 @@ public class EagerShufflingTask extends AbstractPostRegistrationTask
new Log4jSimpleLogger(operationLog));
}
- EagerShufflingTask(Properties properties, Set<String> incomingShares,
+ @Private public EagerShufflingTask(Properties properties, Set<String> incomingShares,
IEncapsulatedOpenBISService service, IShareIdManager shareIdManager,
IFreeSpaceProvider freeSpaceProvider, IDataSetMover dataSetMover,
IConfigProvider configProvider, ISimpleLogger logger)
@@ -143,7 +143,6 @@ public class EagerShufflingTask extends AbstractPostRegistrationTask
Executor(String dataSetCode)
{
this.dataSetCode = dataSetCode;
-
}
public ICleanupTask createCleanupTask()
@@ -163,12 +162,7 @@ public class EagerShufflingTask extends AbstractPostRegistrationTask
+ ".");
return new NoCleanupTask();
}
- return new NoCleanupTask();
- // TODO, 2011-03-28, FJE: A better CleanupTask class is needed because
- // I'm not 100% sure that the data set might be delete in the new share even though
- // shuffling is almost finished.
- // The worst case is that the data set is deleted in both shares.
-// return new CleanupTask(dataSet, storeRoot, shareWithMostFreeOrNull.getShareId());
+ return new CleanupTask(dataSet, storeRoot, shareWithMostFreeOrNull.getShareId());
}
public void execute()
@@ -180,40 +174,33 @@ public class EagerShufflingTask extends AbstractPostRegistrationTask
new File(share, dataSet.getDataSetLocation()),
shareWithMostFreeOrNull.getShare(), logger);
logger.log(LogLevel.INFO, "Data set " + dataSetCode
- + " succesffully moved from share " + dataSet.getDataSetShareId() + " to "
+ + " successfully moved from share " + dataSet.getDataSetShareId() + " to "
+ shareWithMostFreeOrNull.getShareId() + ".");
}
}
}
-// private static final class CleanupTask implements ICleanupTask
-// {
-// private static final long serialVersionUID = 1L;
-//
-// private final SimpleDataSetInformationDTO dataSet;
-// private final File storeRoot;
-// private final String newShareId;
-//
-// CleanupTask(SimpleDataSetInformationDTO dataSet, File storeRoot, String newShareId)
-// {
-// this.dataSet = dataSet;
-// this.storeRoot = storeRoot;
-// this.newShareId = newShareId;
-// }
-//
-// public void cleanup()
-// {
-// IShareIdManager shareIdManager = ServiceProvider.getShareIdManager();
-// String currentShareId =
-// shareIdManager.getShareId(dataSet.getDataSetCode());
-// if (currentShareId.equals(newShareId) == false)
-// {
-// File dataSetFolder =
-// new File(new File(storeRoot, newShareId), dataSet.getDataSetLocation());
-// FileUtilities.deleteRecursively(dataSetFolder);
-// }
-// }
-// }
+ private static final class CleanupTask implements ICleanupTask
+ {
+ private static final long serialVersionUID = 1L;
+
+ private final SimpleDataSetInformationDTO dataSet;
+ private final File storeRoot;
+ private final String newShareId;
+
+ CleanupTask(SimpleDataSetInformationDTO dataSet, File storeRoot, String newShareId)
+ {
+ this.dataSet = dataSet;
+ this.storeRoot = storeRoot;
+ this.newShareId = newShareId;
+ }
+
+ public void cleanup(ISimpleLogger logger)
+ {
+ IShareIdManager shareIdManager = ServiceProvider.getShareIdManager();
+ SegmentedStoreUtils.cleanUp(dataSet, storeRoot, newShareId, shareIdManager, logger);
+ }
+ }
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/ICleanupTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/ICleanupTask.java
index 4e0b1bb48ad..bc068e87f79 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/ICleanupTask.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/ICleanupTask.java
@@ -18,6 +18,8 @@ package ch.systemsx.cisd.etlserver.postregistration;
import java.io.Serializable;
+import ch.systemsx.cisd.common.logging.ISimpleLogger;
+
/**
* Interface for cleanup tasks. Clean up tasks have to be {@link Serializable} because they are made
* persistent.
@@ -37,6 +39,8 @@ public interface ICleanupTask extends Serializable
* Clean up stuff an interrupted {@link IPostRegistrationTaskExecutor#execute()} has created.
* This method should not throw an exception if there is nothing to clean up or the clean up
* state is invalid.
+ *
+ * @param logger Logger which can be used to log what has been done.
*/
- public void cleanup();
+ public void cleanup(ISimpleLogger logger);
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/NoCleanupTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/NoCleanupTask.java
index 515962f7280..1fc5255bc1f 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/NoCleanupTask.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/NoCleanupTask.java
@@ -16,6 +16,8 @@
package ch.systemsx.cisd.etlserver.postregistration;
+import ch.systemsx.cisd.common.logging.ISimpleLogger;
+
/**
* An implementation of {@link ICleanupTask} which does nothing.
*
@@ -25,7 +27,7 @@ public class NoCleanupTask implements ICleanupTask
{
private static final long serialVersionUID = 1L;
- public void cleanup()
+ public void cleanup(ISimpleLogger logger)
{
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/PostRegistrationMaintenanceTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/PostRegistrationMaintenanceTask.java
index 5c673eef945..6e8cc3e54f6 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/PostRegistrationMaintenanceTask.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/PostRegistrationMaintenanceTask.java
@@ -17,8 +17,6 @@
package ch.systemsx.cisd.etlserver.postregistration;
import java.io.File;
-import java.io.FilenameFilter;
-import java.io.IOException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Collections;
@@ -31,15 +29,11 @@ import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.SerializationUtils;
import org.apache.commons.lang.time.DateUtils;
import org.apache.log4j.Logger;
import ch.rinn.restrictions.Private;
-import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
-import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
@@ -62,26 +56,12 @@ public class PostRegistrationMaintenanceTask implements IDataStoreLockingMainten
{
@Private static final String POST_REGISTRATION_TASKS_PROPERTY = "post-registration-tasks";
- @Private static final String CLEANUP_TASKS_FOLDER_PROPERTY = "cleanup-tasks-folder";
-
- private static final String DEFAULT_CLEANUP_TASKS_FOLDER = "clean-up-tasks";
-
@Private static final String IGNORE_DATA_SETS = "ignore-data-sets-before-date";
@Private static final String LAST_SEEN_DATA_SET_FILE_PROPERTY = "last-seen-data-set-file";
private static final String DEFAULT_LAST_SEEN_DATA_SET_FILE = "last-seen-data-set.txt";
- private static final String FILE_TYPE = ".ser";
-
- private static final FilenameFilter FILTER = new FilenameFilter()
- {
- public boolean accept(File dir, String name)
- {
- return name.endsWith(FILE_TYPE);
- }
- };
-
private static final Logger operationLog =
LogFactory.getLogger(LogCategory.OPERATION, PostRegistrationMaintenanceTask.class);
@@ -91,14 +71,14 @@ public class PostRegistrationMaintenanceTask implements IDataStoreLockingMainten
private Set<Entry<String, IPostRegistrationTask>> tasks;
- private File cleanupTasksFolder;
-
private File lastSeenDataSetFile;
private File newLastSeenDataSetFile;
private Date ignoreBeforeDate;
+ private TaskExecutor executor;
+
public boolean requiresDataStoreLock()
{
return needsLockOnDataStore;
@@ -126,15 +106,8 @@ public class PostRegistrationMaintenanceTask implements IDataStoreLockingMainten
map.put(sectionProperty.getKey(), task);
}
tasks = map.entrySet();
- cleanupTasksFolder =
- new File(properties.getProperty(CLEANUP_TASKS_FOLDER_PROPERTY,
- DEFAULT_CLEANUP_TASKS_FOLDER));
- if (cleanupTasksFolder.isFile())
- {
- throw new EnvironmentFailureException("Cleanup tasks folder is a file: "
- + cleanupTasksFolder.getAbsolutePath());
- }
- cleanupTasksFolder.mkdirs();
+ executor = new TaskExecutor(properties);
+ executor.cleanup();
String fileName =
properties.getProperty(LAST_SEEN_DATA_SET_FILE_PROPERTY,
DEFAULT_LAST_SEEN_DATA_SET_FILE);
@@ -159,7 +132,6 @@ public class PostRegistrationMaintenanceTask implements IDataStoreLockingMainten
public void execute()
{
- cleanup();
List<ExternalData> dataSets = getSortedUnseenDataSets();
for (int i = 0; i < dataSets.size(); i++)
{
@@ -172,26 +144,8 @@ public class PostRegistrationMaintenanceTask implements IDataStoreLockingMainten
for (Entry<String, IPostRegistrationTask> entry : tasks)
{
IPostRegistrationTask task = entry.getValue();
- ICleanupTask cleanupTask = null;
- File savedCleanupTask = null;
String taskName = entry.getKey();
- try
- {
- IPostRegistrationTaskExecutor executor = task.createExecutor(code);
- cleanupTask = executor.createCleanupTask();
- savedCleanupTask = save(code, taskName, cleanupTask);
- executor.execute();
- } catch (Throwable t)
- {
- cleanUpAndLog(t, cleanupTask, code, taskName);
- throw t;
- } finally
- {
- if (savedCleanupTask != null)
- {
- savedCleanupTask.delete();
- }
- }
+ executor.execute(task, taskName, code);
}
saveLastSeenDataSetId(dataSet.getId());
} catch (Throwable ex)
@@ -236,24 +190,6 @@ public class PostRegistrationMaintenanceTask implements IDataStoreLockingMainten
newLastSeenDataSetFile.renameTo(lastSeenDataSetFile);
}
- private void cleanUpAndLog(Throwable throwable, ICleanupTask cleanupTaskOrNull,
- String dataSetCode, String taskName)
- {
- operationLog.error("Post registration task '" + taskName + "' for data set " + dataSetCode
- + " failed.", throwable);
- if (cleanupTaskOrNull != null)
- {
- try
- {
- cleanupTaskOrNull.cleanup();
- } catch (Throwable t)
- {
- operationLog.error("Clean up of failed post registration task '" + taskName
- + "' for data set " + dataSetCode + " failed, too.", t);
- }
- }
- }
-
private void logPostponingMessage(List<ExternalData> dataSets, int i)
{
int numberOfDataSets = dataSets.size();
@@ -274,39 +210,4 @@ public class PostRegistrationMaintenanceTask implements IDataStoreLockingMainten
}
}
- private void cleanup()
- {
- File[] files = cleanupTasksFolder.listFiles(FILTER);
- for (File file : files)
- {
- cleanupTask(file);
- }
- }
-
- private void cleanupTask(File file)
- {
- try
- {
- byte[] bytes = FileUtils.readFileToByteArray(file);
- ((ICleanupTask) SerializationUtils.deserialize(bytes)).cleanup();
- } catch (Exception ex)
- {
- operationLog.error("Couldn't performed clean up task " + file, ex);
- }
- file.delete();
- }
-
- private File save(String code, String taskName, ICleanupTask cleanupTask)
- {
- try
- {
- File file = new File(cleanupTasksFolder, code + "_" + taskName + FILE_TYPE);
- FileUtils.writeByteArrayToFile(file, SerializationUtils.serialize(cleanupTask));
- return file;
- } catch (IOException ex)
- {
- throw CheckedExceptionTunnel.wrapIfNecessary(ex);
- }
- }
-
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/TaskExecutor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/TaskExecutor.java
new file mode 100644
index 00000000000..f109e2374aa
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/TaskExecutor.java
@@ -0,0 +1,161 @@
+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.postregistration;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.util.Properties;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.SerializationUtils;
+import org.apache.log4j.Logger;
+
+import ch.rinn.restrictions.Private;
+import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
+import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.common.logging.ISimpleLogger;
+import ch.systemsx.cisd.common.logging.Log4jSimpleLogger;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+
+/**
+ * Executing engine of {@link IPostRegistrationTask} instances. Manages persistent
+ * {@link ICleanupTask} instances.
+ *
+ * @author Franz-Josef Elmer
+ */
+public class TaskExecutor
+{
+ @Private static final String CLEANUP_TASKS_FOLDER_PROPERTY = "cleanup-tasks-folder";
+
+ private static final String DEFAULT_CLEANUP_TASKS_FOLDER = "clean-up-tasks";
+
+ private static final String FILE_TYPE = ".ser";
+
+ private static final FilenameFilter FILTER = new FilenameFilter()
+ {
+ public boolean accept(File dir, String name)
+ {
+ return name.endsWith(FILE_TYPE);
+ }
+ };
+
+ private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
+ TaskExecutor.class);
+
+ private final File cleanupTasksFolder;
+
+ /**
+ * Creates an instance for specified folder which will store persistent {@link ICleanupTask}
+ * instances.
+ */
+ public TaskExecutor(Properties properties)
+ {
+ cleanupTasksFolder =
+ new File(properties.getProperty(CLEANUP_TASKS_FOLDER_PROPERTY,
+ DEFAULT_CLEANUP_TASKS_FOLDER));
+ if (cleanupTasksFolder.isFile())
+ {
+ throw new EnvironmentFailureException("Cleanup tasks folder is a file: "
+ + cleanupTasksFolder.getAbsolutePath());
+ }
+ cleanupTasksFolder.mkdirs();
+ }
+
+ public void execute(IPostRegistrationTask task, String taskName, String dataSetCode)
+ throws Throwable
+ {
+ ICleanupTask cleanupTask = null;
+ File savedCleanupTask = null;
+ try
+ {
+ IPostRegistrationTaskExecutor executor = task.createExecutor(dataSetCode);
+ cleanupTask = executor.createCleanupTask();
+ savedCleanupTask = saveCleanupTask(dataSetCode, taskName, cleanupTask);
+ executor.execute();
+ } catch (Throwable t)
+ {
+ cleanUpAndLog(t, cleanupTask, dataSetCode, taskName);
+ throw t;
+ } finally
+ {
+ if (savedCleanupTask != null)
+ {
+ savedCleanupTask.delete();
+ }
+ }
+ }
+
+ private File saveCleanupTask(String code, String taskName, ICleanupTask cleanupTask)
+ {
+ try
+ {
+ File file = new File(cleanupTasksFolder, code + "_" + taskName + FILE_TYPE);
+ FileUtils.writeByteArrayToFile(file, SerializationUtils.serialize(cleanupTask));
+ return file;
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ private void cleanUpAndLog(Throwable throwable, ICleanupTask cleanupTaskOrNull,
+ String dataSetCode, String taskName)
+ {
+ operationLog.error("Task '" + taskName + "' for data set " + dataSetCode + " failed.",
+ throwable);
+ if (cleanupTaskOrNull != null)
+ {
+ try
+ {
+ cleanupTaskOrNull.cleanup(new Log4jSimpleLogger(operationLog));
+ } catch (Throwable t)
+ {
+ operationLog.error("Clean up of failed task '" + taskName + "' for data set "
+ + dataSetCode + " failed, too.", t);
+ }
+ }
+ }
+
+ /**
+ * Performs all cleanup task in cleanup-tasks folder.
+ */
+ public void cleanup()
+ {
+ Log4jSimpleLogger logger = new Log4jSimpleLogger(operationLog);
+ File[] files = cleanupTasksFolder.listFiles(FILTER);
+ for (File file : files)
+ {
+ cleanupTask(file, logger);
+ }
+ }
+
+ private void cleanupTask(File file, ISimpleLogger logger)
+ {
+ try
+ {
+ byte[] bytes = FileUtils.readFileToByteArray(file);
+ ((ICleanupTask) SerializationUtils.deserialize(bytes)).cleanup(logger);
+ } catch (Exception ex)
+ {
+ operationLog.error("Couldn't performed clean up task " + file, ex);
+ }
+ file.delete();
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java
index 9a5d414e8ae..e9f9c247d90 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java
@@ -222,8 +222,8 @@ public class SegmentedStoreUtils
* <li>Copying data set to new share.
* <li>Sanity check of successfully copied data set.
* <li>Changing share id in openBIS AS.
- * <li>Spawn an asynchronous task which deletes the data set at the old location if all locks on
- * the data set have been released.
+ * <li>Deletes the data set at the old location after all locks on the data set have been
+ * released.
* </ol>
*
* @param service to access openBIS AS.
@@ -249,13 +249,7 @@ public class SegmentedStoreUtils
String shareId = share.getName();
shareIdManager.setShareId(dataSetCode, shareId);
service.updateShareIdAndSize(dataSetCode, shareId, size);
- new Thread(new Runnable()
- {
- public void run()
- {
- deleteDataSet(dataSetCode, dataSetDirInStore, shareIdManager, logger);
- }
- }).start();
+ deleteDataSet(dataSetCode, dataSetDirInStore, shareIdManager, logger);
}
/**
@@ -282,6 +276,25 @@ public class SegmentedStoreUtils
}
}
+ public static void cleanUp(SimpleDataSetInformationDTO dataSet, File storeRoot,
+ String newShareId, IShareIdManager shareIdManager, ISimpleLogger logger)
+ {
+ String dataSetCode = dataSet.getDataSetCode();
+ String shareId = shareIdManager.getShareId(dataSetCode);
+ String oldShareId = dataSet.getDataSetShareId();
+ boolean currentIsOld = shareId.equals(oldShareId);
+ boolean currentIsNew = shareId.equals(newShareId);
+ if (currentIsOld == false && currentIsNew == false)
+ {
+ logger.log(LogLevel.WARN, "No clean up will be performed because data set "
+ + dataSetCode + " is neither in share " + oldShareId + " nor in share "
+ + newShareId + " but in share " + shareId + ".");
+ return;
+ }
+ File shareFolder = new File(storeRoot, currentIsOld ? newShareId : oldShareId);
+ deleteDataSet(dataSetCode, new File(shareFolder, dataSet.getDataSetLocation()), shareIdManager, logger);
+ }
+
private static void copyToShare(File file, File share)
{
RsyncCopier copier = new RsyncCopier(OSUtilities.findExecutable(RSYNC_EXEC));
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreShufflingTaskTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreShufflingTaskTest.java
index 08d4f29f57b..a35741dc677 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreShufflingTaskTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreShufflingTaskTest.java
@@ -16,8 +16,8 @@
package ch.systemsx.cisd.etlserver.plugins;
-import static ch.systemsx.cisd.etlserver.plugins.SegmentedStoreShufflingTask.SHUFFLING_SECTION_NAME;
import static ch.systemsx.cisd.etlserver.plugins.SegmentedStoreShufflingTask.CLASS_PROPERTY_NAME;
+import static ch.systemsx.cisd.etlserver.plugins.SegmentedStoreShufflingTask.SHUFFLING_SECTION_NAME;
import java.io.File;
import java.util.Arrays;
@@ -55,6 +55,7 @@ public class SegmentedStoreShufflingTaskTest extends AbstractFileSystemTestCase
public static final class Balancer implements ISegmentedStoreShuffling
{
private final Properties properties;
+ private boolean initialized;
private List<Share> sourceShares;
private List<Share> targetShares;
private IEncapsulatedOpenBISService service;
@@ -66,6 +67,11 @@ public class SegmentedStoreShufflingTaskTest extends AbstractFileSystemTestCase
this.properties = properties;
}
+ public void init(ISimpleLogger l)
+ {
+ initialized = true;
+ }
+
public void shuffleDataSets(List<Share> sources, List<Share> targets,
IEncapsulatedOpenBISService openBisService, IDataSetMover mover,
ISimpleLogger simpleLogger)
@@ -158,6 +164,7 @@ public class SegmentedStoreShufflingTaskTest extends AbstractFileSystemTestCase
assertSame(service, balancer.service);
assertSame(dataSetMover, balancer.dataSetMover);
assertSame(logger, balancer.logger);
+ assertEquals(true, balancer.initialized);
context.assertIsSatisfied();
}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/SimpleShufflingTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/SimpleShufflingTest.java
index f57347126bc..df6529fd8ee 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/SimpleShufflingTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/SimpleShufflingTest.java
@@ -21,9 +21,11 @@ import static org.apache.commons.io.FileUtils.ONE_MB;
import java.io.File;
import java.io.IOException;
+import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -38,12 +40,15 @@ import org.testng.annotations.Test;
import ch.rinn.restrictions.Friend;
import ch.systemsx.cisd.base.tests.AbstractFileSystemTestCase;
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.common.filesystem.HostAwareFile;
import ch.systemsx.cisd.common.filesystem.IFreeSpaceProvider;
import ch.systemsx.cisd.common.logging.ISimpleLogger;
import ch.systemsx.cisd.common.logging.LogLevel;
-import ch.systemsx.cisd.common.utilities.ITimeProvider;
+import ch.systemsx.cisd.etlserver.postregistration.EagerShufflingTask;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IConfigProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IShareIdManager;
import ch.systemsx.cisd.openbis.dss.generic.shared.utils.Share;
import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
@@ -55,6 +60,7 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
@Friend(toClasses=SimpleShuffling.class)
public class SimpleShufflingTest extends AbstractFileSystemTestCase
{
+ private static final String DSS_CODE = "dss1";
private static final String STORE_PATH = "01/02/03/";
private static final class MockSpaceProvider implements IFreeSpaceProvider
@@ -100,6 +106,9 @@ public class SimpleShufflingTest extends AbstractFileSystemTestCase
private ISimpleLogger logger;
private SimpleShuffling balancer;
private File store;
+ private EagerShufflingTask eagerShufflingTask;
+ private IShareIdManager shareIdManager;
+ private IConfigProvider configProvider;
@BeforeMethod
public void beforeMethod()
@@ -107,94 +116,130 @@ public class SimpleShufflingTest extends AbstractFileSystemTestCase
spaceProvider = new MockSpaceProvider();
context = new Mockery();
service = context.mock(IEncapsulatedOpenBISService.class);
+ shareIdManager = context.mock(IShareIdManager.class);
+ configProvider = context.mock(IConfigProvider.class);
dataSetMover = context.mock(IDataSetMover.class);
logger = context.mock(ISimpleLogger.class);
- final ITimeProvider timeProvider = context.mock(ITimeProvider.class);
+ Properties properties = new Properties();
+ properties.setProperty(SimpleShuffling.MINIMUM_FREE_SPACE_KEY, "2");
+ properties.setProperty(EagerShufflingTask.SHARE_FINDER_KEY + ".class", SimpleShufflingShareFinder.class.getName());
+ properties.setProperty(EagerShufflingTask.SHARE_FINDER_KEY + "." + SimpleShufflingShareFinder.MINIMUM_FREE_SPACE_KEY, "2");
+ store = new File(workingDirectory, "store");
+ store.mkdirs();
+ File ds1 = new File(store, "1/" + STORE_PATH + "ds1");
+ ds1.getParentFile().mkdirs();
+ FileUtilities.writeToFile(ds1, "hello ds1");
+ FileUtilities.writeToFile(new File(store, "1/" + STORE_PATH + "ds2"), "hello ds2");
+ FileUtilities.writeToFile(new File(store, "1/" + STORE_PATH + "ds3"), "hello ds3");
+ File share2 = new File(store, "2/" + STORE_PATH);
+ share2.mkdirs();
+ FileUtilities.writeToFile(new File(share2, "ds4"), "hello ds4");
+ new File(store, "3").mkdirs();
+ new File(store, "4").mkdirs();
context.checking(new Expectations()
{
{
- allowing(timeProvider).getTimeInMilliseconds();
- will(returnValue(0L));
+ allowing(configProvider).getDataStoreCode();
+ will(returnValue(DSS_CODE));
+
+ allowing(configProvider).getStoreRoot();
+ will(returnValue(store));
}
});
- Properties properties = new Properties();
- properties.setProperty(SimpleShuffling.MINIMUM_FREE_SPACE_KEY, "2");
- balancer = new SimpleShuffling(properties, timeProvider);
- store = new File(workingDirectory, "store");
+ eagerShufflingTask =
+ new EagerShufflingTask(properties, new HashSet<String>(Arrays.asList("1", "2")),
+ service, shareIdManager, spaceProvider, dataSetMover, configProvider,
+ logger);
+ balancer = new SimpleShuffling(properties, eagerShufflingTask);
}
@AfterMethod
- public void afterMethod()
+ public void afterMethod(Method method)
{
- // The following line of code should also be called at the end of each test method.
- // Otherwise one do not known which test failed.
- context.assertIsSatisfied();
+ try
+ {
+ context.assertIsSatisfied();
+ } catch (Throwable t)
+ {
+ // assert expectations were met, including the name of the failed method
+ throw new Error(method.getName() + "() : ", t);
+ }
}
@Test
public void test()
{
final Share share1 = new Share(new File(store, "1"), spaceProvider);
- share1.addDataSet(dataSet("ds1", 2000));
- share1.addDataSet(dataSet("ds2", ONE_MB));
- share1.addDataSet(dataSet("ds3", ONE_MB + 100));
+ final SimpleDataSetInformationDTO ds1 = dataSet("ds1", "1", 2000);
+ share1.addDataSet(ds1);
+ final SimpleDataSetInformationDTO ds2 = dataSet("ds2", "1", ONE_MB);
+ share1.addDataSet(ds2);
+ final SimpleDataSetInformationDTO ds3 = dataSet("ds3", "1", ONE_MB + 100);
+ share1.addDataSet(ds3);
+ spaceProvider.addFreeSpaceExpectationFor(share1, 100l);
+ spaceProvider.addFreeSpaceExpectationFor(share1, 100l);
spaceProvider.addFreeSpaceExpectationFor(share1, 100l);
spaceProvider.addFreeSpaceExpectationFor(share1, 100l);
- spaceProvider.addFreeSpaceExpectationFor(share1, 1100l);
- spaceProvider.addFreeSpaceExpectationFor(share1, 2100l);
final Share share2 = new Share(new File(store, "2"), spaceProvider);
- share2.addDataSet(dataSet("ds4", 2 * ONE_MB));
+ final SimpleDataSetInformationDTO ds4 = dataSet("ds4", "2", 2 * ONE_MB);
+ share2.addDataSet(ds4);
+ spaceProvider.addFreeSpaceExpectationFor(share2, 500l);
+ spaceProvider.addFreeSpaceExpectationFor(share2, 500l);
spaceProvider.addFreeSpaceExpectationFor(share2, 500l);
spaceProvider.addFreeSpaceExpectationFor(share2, 500l);
final Share share3 = new Share(new File(store, "3"), spaceProvider);
spaceProvider.addFreeSpaceExpectationFor(share3, 4 * 1024l);
spaceProvider.addFreeSpaceExpectationFor(share3, 3 * 1024l);
- spaceProvider.addFreeSpaceExpectationFor(share3, 1024l);
+ spaceProvider.addFreeSpaceExpectationFor(share3, 2 * 1024l);
final Share share4 = new Share(new File(store, "4"), spaceProvider);
spaceProvider.addFreeSpaceExpectationFor(share4, 3 * 1024l);
+ spaceProvider.addFreeSpaceExpectationFor(share4, 4 * 1024l);
+ spaceProvider.addFreeSpaceExpectationFor(share4, 1024l);
context.checking(new Expectations()
{
{
+ allowing(service).listDataSets();
+ will(returnValue(Arrays.asList(ds1, ds2, ds3, ds4)));
one(logger).log(LogLevel.INFO, "BEGIN Computing number of data sets to move for share 1");
one(logger).log(LogLevel.INFO, "\tSpace needed to free: 1994752 bytes (1948.00 kB, 1.90 MB)");
one(logger).log(LogLevel.INFO, "\tInspecting 3 data sets.");
one(logger).log(LogLevel.INFO, "END Computing number of data sets to move for share 1");
one(logger).log(LogLevel.INFO, "\t2 data sets to move, available space : 102500");
- one(logger).log(LogLevel.INFO, "Copying data set ds3 from share 1 to share 3 ...");
- one(dataSetMover).moveDataSetToAnotherShare(new File(share1.getShare(), STORE_PATH + "ds3"), share3.getShare(), logger);
- one(logger).log(LogLevel.INFO, "Copying data set ds3 from share 1 to share 3 took 0 seconds.");
+ one(shareIdManager).getShareId(ds3.getDataSetCode());
+ will(returnValue(ds3.getDataSetShareId()));
- one(logger).log(LogLevel.INFO, "Copying data set ds2 from share 1 to share 3 ...");
- one(dataSetMover).moveDataSetToAnotherShare(new File(share1.getShare(), STORE_PATH + "ds2"), share3.getShare(), logger);
- one(logger).log(LogLevel.INFO, "Copying data set ds2 from share 1 to share 3 took 0 seconds.");
+ one(dataSetMover).moveDataSetToAnotherShare(new File(share1.getShare(), STORE_PATH + "ds3"), share3.getShare(), logger);
+ one(logger).log(LogLevel.INFO, "Data set ds3 successfully moved from share 1 to 3.");
one(logger).log(LogLevel.INFO, "BEGIN Computing number of data sets to move for share 2");
one(logger).log(LogLevel.INFO, "\tSpace needed to free: 1585152 bytes (1548.00 kB, 1.51 MB)");
one(logger).log(LogLevel.INFO, "\tInspecting 1 data sets.");
one(logger).log(LogLevel.INFO, "END Computing number of data sets to move for share 2");
+ one(shareIdManager).getShareId(ds2.getDataSetCode());
+ will(returnValue(ds2.getDataSetShareId()));
+
+ one(dataSetMover).moveDataSetToAnotherShare(new File(share1.getShare(), STORE_PATH + "ds2"), share4.getShare(), logger);
+ one(logger).log(LogLevel.INFO, "Data set ds2 successfully moved from share 1 to 4.");
+
+ one(logger).log(LogLevel.WARN, "No share found for shuffling data set ds4.");
}
});
balancer.shuffleDataSets(Arrays.asList(share1, share2),
Arrays.asList(share1, share2, share3, share4), service, dataSetMover, logger);
- assertEquals(1, share1.getDataSetsOrderedBySize().size());
- assertEquals(1, share2.getDataSetsOrderedBySize().size());
- assertEquals("ds3", share3.getDataSetsOrderedBySize().get(0).getDataSetCode());
- assertEquals("ds2", share3.getDataSetsOrderedBySize().get(1).getDataSetCode());
- assertEquals(2, share3.getDataSetsOrderedBySize().size());
- assertEquals(0, share4.getDataSetsOrderedBySize().size());
spaceProvider.assertIsSatiesfied();
- context.assertIsSatisfied();
}
- private SimpleDataSetInformationDTO dataSet(String code, long size)
+ private SimpleDataSetInformationDTO dataSet(String code, String shareId, long size)
{
SimpleDataSetInformationDTO dataSet = new SimpleDataSetInformationDTO();
dataSet.setDataSetCode(code);
+ dataSet.setDataSetShareId(shareId);
dataSet.setDataSetLocation(STORE_PATH + code);
dataSet.setDataSetSize(size);
+ dataSet.setDataStoreCode(DSS_CODE);
return dataSet;
}
}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/postregistration/EagerShufflingTaskTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/postregistration/EagerShufflingTaskTest.java
index 3e438a9084b..b1b1b9fb9cd 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/postregistration/EagerShufflingTaskTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/postregistration/EagerShufflingTaskTest.java
@@ -132,7 +132,7 @@ public class EagerShufflingTaskTest extends AbstractFileSystemTestCase
executor.createCleanupTask();
executor.execute();
- assertEquals("Data set ds-1 succesffully moved from share 1 to 4.",
+ assertEquals("Data set ds-1 successfully moved from share 1 to 4.",
infoMessageMatcher.recordedObject());
assertHostAwareFile(hostAwareFileMatcher);
context.assertIsSatisfied();
@@ -160,7 +160,7 @@ public class EagerShufflingTaskTest extends AbstractFileSystemTestCase
executor.createCleanupTask();
executor.execute();
- assertEquals("Data set ds-1 succesffully moved from share 1 to 2.",
+ assertEquals("Data set ds-1 successfully moved from share 1 to 2.",
infoMessageMatcher.recordedObject());
assertHostAwareFile(hostAwareFileMatcher);
context.assertIsSatisfied();
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/postregistration/PostRegistrationMaintenanceTaskTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/postregistration/PostRegistrationMaintenanceTaskTest.java
index ee5c2432bd4..70496308477 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/postregistration/PostRegistrationMaintenanceTaskTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/postregistration/PostRegistrationMaintenanceTaskTest.java
@@ -16,7 +16,7 @@
package ch.systemsx.cisd.etlserver.postregistration;
-import static ch.systemsx.cisd.etlserver.postregistration.PostRegistrationMaintenanceTask.CLEANUP_TASKS_FOLDER_PROPERTY;
+import static ch.systemsx.cisd.etlserver.postregistration.TaskExecutor.CLEANUP_TASKS_FOLDER_PROPERTY;
import static ch.systemsx.cisd.etlserver.postregistration.PostRegistrationMaintenanceTask.LAST_SEEN_DATA_SET_FILE_PROPERTY;
import static ch.systemsx.cisd.etlserver.postregistration.PostRegistrationMaintenanceTask.POST_REGISTRATION_TASKS_PROPERTY;
@@ -48,6 +48,7 @@ import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
import ch.systemsx.cisd.base.tests.AbstractFileSystemTestCase;
import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.common.logging.BufferedAppender;
+import ch.systemsx.cisd.common.logging.ISimpleLogger;
import ch.systemsx.cisd.common.test.AssertionUtil;
import ch.systemsx.cisd.common.test.RecordingMatcher;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
@@ -60,7 +61,7 @@ import ch.systemsx.cisd.openbis.generic.shared.basic.dto.builders.DataSetBuilder
*
* @author Franz-Josef Elmer
*/
-@Friend(toClasses=PostRegistrationMaintenanceTask.class)
+@Friend(toClasses={PostRegistrationMaintenanceTask.class, TaskExecutor.class})
public class PostRegistrationMaintenanceTaskTest extends AbstractFileSystemTestCase
{
private static final String TASK_1_NAME = "task 1";
@@ -78,7 +79,7 @@ public class PostRegistrationMaintenanceTaskTest extends AbstractFileSystemTestC
this.name = name;
}
- public void cleanup()
+ public void cleanup(ISimpleLogger logger)
{
cleanupInvocations.add(name);
}
@@ -251,7 +252,7 @@ public class PostRegistrationMaintenanceTaskTest extends AbstractFileSystemTestC
maintenanceTask.setUp("post-registration", properties);
maintenanceTask.execute();
- AssertionUtil.assertContains("ERROR OPERATION.PostRegistrationMaintenanceTask - "
+ AssertionUtil.assertContains("ERROR OPERATION.TaskExecutor - "
+ "Couldn't performed clean up task " + cleanupFile2, logRecorder.getLogContent());
assertEquals(0, criteriaMatcher.recordedObject().getLastSeenDataSetId());
assertEquals(true, maintenanceTask.requiresDataStoreLock());
@@ -462,8 +463,8 @@ public class PostRegistrationMaintenanceTaskTest extends AbstractFileSystemTestC
assertEquals(false, maintenanceTask.requiresDataStoreLock());
maintenanceTask.execute();
- AssertionUtil.assertContains("ERROR OPERATION.PostRegistrationMaintenanceTask - "
- + "Post registration task '2' for data set ds-2 failed.",
+ AssertionUtil.assertContains("ERROR OPERATION.TaskExecutor - "
+ + "Task '2' for data set ds-2 failed.",
logRecorder.getLogContent());
AssertionUtil.assertContains("ERROR OPERATION.PostRegistrationMaintenanceTask - "
+ "Because post registration task failed for data set ds-2 "
--
GitLab