diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreShufflingTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreShufflingTask.java
index e3424168f408b66866777ac752bd8333afc0f4a8..d6be00f6a2bff2cfe6699b2dc4295ea3b662602a 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreShufflingTask.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreShufflingTask.java
@@ -20,6 +20,7 @@ import static ch.systemsx.cisd.common.logging.LogLevel.INFO;
import java.io.File;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Properties;
import java.util.Set;
@@ -201,7 +202,8 @@ public class SegmentedStoreShufflingTask implements IDataStoreLockingMaintenance
operationLog.info("Starting segmented store shuffling.");
List<Share> shares =
SegmentedStoreUtils.getDataSetsPerShare(storeRoot, dataStoreCode,
- freeSpaceProvider, service, operationLogger);
+ Collections.<String> emptySet(), freeSpaceProvider, service,
+ operationLogger);
List<Share> sourceShares = new ArrayList<Share>();
for (Share share : shares)
{
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/EagerShufflingTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/EagerShufflingTask.java
index 927e0f9da3c651a59822dfc0241d4c969e6960b4..de6edbb79d00796ea8da6da2597c952dfd1fdb4c 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/EagerShufflingTask.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/EagerShufflingTask.java
@@ -148,12 +148,8 @@ public class EagerShufflingTask extends AbstractPostRegistrationTask
public ICleanupTask createCleanupTask()
{
List<Share> shares =
- SegmentedStoreUtils.getDataSetsPerShare(storeRoot, dataStoreCode,
+ SegmentedStoreUtils.getDataSetsPerShare(storeRoot, dataStoreCode, incomingShares,
freeSpaceProvider, service, logger);
- for (Share share : shares)
- {
- share.setIncoming(incomingShares.contains(share.getShareId()));
- }
dataSet = findDataSet(shares, dataSetCode);
shareWithMostFreeOrNull = finder.tryToFindShare(dataSet, shares);
if (shareWithMostFreeOrNull == null)
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/IShareFinder.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/IShareFinder.java
index 970ac083cf73ce35313973455c44ebb1bac793fe..731a1d53b31073ae8fb0f47b956592d6834830e7 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/IShareFinder.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/IShareFinder.java
@@ -34,7 +34,7 @@ public interface IShareFinder
* Tries to find a share from the specified shares to whom the specified data set can be moved.
*
* @param dataSet with known size and old share ID.
- * @param shares All shares. Share instances know whether they are icoming or external.
+ * @param shares All shares. Share instances know whether they are incoming or external.
* @return <code>null</code> if no share could be found.
*/
public Share tryToFindShare(SimpleDataSetInformationDTO dataSet, List<Share> shares);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java
index ac8220d0cffe71fec06906df734a342295e23e35..362601a0b8ec86cdf8f54ef448e79a9827536c1f 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java
@@ -26,20 +26,38 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.Set;
+import org.apache.commons.io.FileUtils;
+
+import ch.rinn.restrictions.Private;
import ch.systemsx.cisd.common.collections.CollectionUtils;
import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.filesystem.BooleanStatus;
+import ch.systemsx.cisd.common.filesystem.IFreeSpaceProvider;
+import ch.systemsx.cisd.common.filesystem.SimpleFreeSpaceProvider;
+import ch.systemsx.cisd.common.logging.Log4jSimpleLogger;
+import ch.systemsx.cisd.common.utilities.ClassUtils;
+import ch.systemsx.cisd.common.utilities.PropertyParametersUtil;
+import ch.systemsx.cisd.etlserver.ETLDaemon;
+import ch.systemsx.cisd.etlserver.postregistration.IShareFinder;
import ch.systemsx.cisd.openbis.dss.generic.shared.ArchiverTaskContext;
import ch.systemsx.cisd.openbis.dss.generic.shared.IArchiverPlugin;
import ch.systemsx.cisd.openbis.dss.generic.shared.IDataSetDeleter;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IDataSetDirectoryProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IShareIdManager;
import ch.systemsx.cisd.openbis.dss.generic.shared.ProcessingStatus;
import ch.systemsx.cisd.openbis.dss.generic.shared.QueueingDataSetStatusUpdaterService;
import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetCodesWithStatus;
+import ch.systemsx.cisd.openbis.dss.generic.shared.utils.SegmentedStoreUtils;
+import ch.systemsx.cisd.openbis.dss.generic.shared.utils.Share;
+import ch.systemsx.cisd.openbis.generic.server.business.bo.SimpleDataSetHelper;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetArchivingStatus;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DeletedDataSet;
import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
+import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
/**
* The base class for archiving.
@@ -53,9 +71,15 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
private static final long serialVersionUID = 1L;
+ @Private public static final String SHARE_FINDER_KEY = "share-finder";
+
private final IStatusChecker archivePrerequisiteOrNull;
private final IStatusChecker unarchivePrerequisiteOrNull;
+
+ private transient IShareIdManager shareIdManager;
+
+ private transient IEncapsulatedOpenBISService service;
public AbstractArchiverProcessingPlugin(Properties properties, File storeRoot,
IStatusChecker archivePrerequisiteOrNull, IStatusChecker unarchivePrerequisiteOrNull)
@@ -64,7 +88,7 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
this.archivePrerequisiteOrNull = archivePrerequisiteOrNull;
this.unarchivePrerequisiteOrNull = unarchivePrerequisiteOrNull;
}
-
+
/**
* NOTE: this method is not allowed to throw exception as this will leave data sets in the
* openBIS database with an inconsistent status.
@@ -97,6 +121,18 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
{
operationLog.info("Archiving of the following datasets has been requested: "
+ CollectionUtils.abbreviate(datasets, 10));
+ for (DatasetDescription dataset : datasets)
+ {
+ if (dataset.getDataSetSize() == null)
+ {
+ String dataSetCode = dataset.getDatasetCode();
+ String shareId = getShareIdManager().getShareId(dataSetCode);
+ File shareFolder = new File(storeRoot, shareId);
+ String dataSetLocation = dataset.getDataSetLocation();
+ long size = FileUtils.sizeOfDirectory(new File(shareFolder, dataSetLocation));
+ getService().updateShareIdAndSize(dataSetCode, shareId, size);
+ }
+ }
DatasetProcessingStatuses statuses = safeArchive(datasets, context, removeFromDataStore);
@@ -212,14 +248,15 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
IDataSetDeleter dataSetDeleter = ServiceProvider.getDataStoreService().getDataSetDeleter();
dataSetDeleter.scheduleDeletionOfDataSets(datasets);
}
-
+
public ProcessingStatus unarchive(List<DatasetDescription> datasets,
- final ArchiverTaskContext context)
+ ArchiverTaskContext context)
{
operationLog.info("Unarchiving of the following datasets has been requested: "
+ CollectionUtils.abbreviate(datasets, 10));
-
- DatasetProcessingStatuses statuses = safeUnarchive(datasets, context);
+
+ DatasetProcessingStatuses statuses =
+ safeUnarchive(datasets, createUnarchivingContext(context));
asyncUpdateStatuses(statuses.getSuccessfulDatasetCodes(), AVAILABLE, true);
asyncUpdateStatuses(statuses.getFailedDatasetCodes(), ARCHIVED, true);
@@ -227,6 +264,30 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
return statuses.getProcessingStatus();
}
+ private ArchiverTaskContext createUnarchivingContext(ArchiverTaskContext context)
+ {
+ Properties props =
+ PropertyParametersUtil.extractSingleSectionProperties(properties, SHARE_FINDER_KEY,
+ false).getProperties();
+ if (props.isEmpty())
+ {
+ return context;
+ }
+
+ String dataStoreCode = ServiceProvider.getConfigProvider().getDataStoreCode();
+ Set<String> incomingShares = ETLDaemon.getIdsOfIncomingShares();
+ IFreeSpaceProvider freeSpaceProvider = new SimpleFreeSpaceProvider();
+ List<Share> shares =
+ SegmentedStoreUtils.getDataSetsPerShare(storeRoot, dataStoreCode, incomingShares,
+ freeSpaceProvider, getService(), new Log4jSimpleLogger(operationLog));
+ IShareFinder shareFinder =
+ ClassUtils.create(IShareFinder.class, props.getProperty("class"), props);
+ DataSetDirectoryProviderForUnarchiving directoryProvider =
+ new DataSetDirectoryProviderForUnarchiving(context.getDirectoryProvider(),
+ shareFinder, getService(), shares);
+ return new ArchiverTaskContext(directoryProvider);
+ }
+
/**
* a 'safe' method that never throws any exceptions.
*/
@@ -444,6 +505,82 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
return new GroupedDatasets(present, notPresent);
}
+
+ private IShareIdManager getShareIdManager()
+ {
+ if (shareIdManager == null)
+ {
+ shareIdManager = ServiceProvider.getShareIdManager();
+ }
+ return shareIdManager;
+ }
+
+ private IEncapsulatedOpenBISService getService()
+ {
+ if (service == null)
+ {
+ service = ServiceProvider.getOpenBISService();
+ }
+ return service;
+ }
+
+ /**
+ * Data set directory provider which might change share in accordance with a
+ * {@link IShareFinder}.
+ *
+ * @author Franz-Josef Elmer
+ */
+ private static final class DataSetDirectoryProviderForUnarchiving implements
+ IDataSetDirectoryProvider
+ {
+ private final IDataSetDirectoryProvider provider;
+
+ private final IShareFinder shareFinder;
+
+ private final IEncapsulatedOpenBISService service;
+
+ private final List<Share> shares;
+
+ DataSetDirectoryProviderForUnarchiving(IDataSetDirectoryProvider provider,
+ IShareFinder shareFinder, IEncapsulatedOpenBISService service, List<Share> shares)
+ {
+ this.provider = provider;
+ this.shareFinder = shareFinder;
+ this.service = service;
+ this.shares = shares;
+ }
+
+ public File getStoreRoot()
+ {
+ return provider.getStoreRoot();
+ }
+
+ public IShareIdManager getShareIdManager()
+ {
+ return provider.getShareIdManager();
+ }
+
+ public File getDataSetDirectory(DatasetDescription dataSet)
+ {
+ SimpleDataSetInformationDTO translatedDataSet = SimpleDataSetHelper.translate(dataSet);
+ String dataSetCode = dataSet.getDatasetCode();
+ IShareIdManager shareIdManager = getShareIdManager();
+ String shareId = shareIdManager.getShareId(dataSetCode);
+ translatedDataSet.setDataSetShareId(shareId);
+ Share share = shareFinder.tryToFindShare(translatedDataSet, shares);
+ if (share != null)
+ {
+ String newShareId = share.getShareId();
+ if (newShareId.equals(shareId) == false)
+ {
+ service.updateShareIdAndSize(dataSetCode, newShareId, dataSet.getDataSetSize());
+ shareIdManager.setShareId(dataSetCode, newShareId);
+ }
+ }
+ return provider.getDataSetDirectory(dataSet);
+ }
+
+ }
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManager.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManager.java
index 770dc5cedc4f7c57bb6d3997ad61c68a3d9c9034..467a966f3b87d524c088ae15d300cd42a7f310f2 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManager.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManager.java
@@ -160,9 +160,11 @@ public class DataSetFileOperationsManager
{
File destinationFolder = new File(destination, dataset.getDataSetLocation());
checkDestinationExists(destinationFolder);
+ File folder = originalData.getParentFile();
operationLog.info("Retrieve data set '" + dataset.getDatasetCode() + "' from '"
- + destinationFolder.getPath() + "' to '" + originalData.getParentFile());
- executor.retrieveDataSetFromDestination(originalData.getParentFile(), destinationFolder);
+ + destinationFolder.getPath() + "' to '" + folder);
+ folder.mkdirs();
+ executor.retrieveDataSetFromDestination(folder, destinationFolder);
return Status.OK;
} catch (ExceptionWithStatus ex)
{
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java
index 599921c0d988539f61283f72bba68b595349c233..3cc24255e2c342ac6655d8649bdc0f01a579c9a1 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java
@@ -26,6 +26,7 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.regex.Pattern;
import org.apache.commons.io.FileUtils;
@@ -141,17 +142,24 @@ public class SegmentedStoreUtils
* and updates the size of all data sets if necessary.
*
* @param dataStoreCode Code of the data store to which the root belongs.
+ * @param incomingShares Set of IDs of incoming shares. Will be used to mark {@link Share}
+ * object in the returned list.
* @param freeSpaceProvider Provider of free space used for all shares.
* @param service Access to openBIS API in order to get all data sets and to update data set
* size.
* @param log Logger for logging size calculations.
*/
public static List<Share> getDataSetsPerShare(File storeRoot, String dataStoreCode,
- IFreeSpaceProvider freeSpaceProvider, IEncapsulatedOpenBISService service,
- ISimpleLogger log)
+ Set<String> incomingShares, IFreeSpaceProvider freeSpaceProvider,
+ IEncapsulatedOpenBISService service, ISimpleLogger log)
{
- return getDataSetsPerShare(storeRoot, dataStoreCode, freeSpaceProvider, service, log,
+ List<Share> shares = getDataSetsPerShare(storeRoot, dataStoreCode, freeSpaceProvider, service, log,
SystemTimeProvider.SYSTEM_TIME_PROVIDER);
+ for (Share share : shares)
+ {
+ share.setIncoming(incomingShares.contains(share.getShareId()));
+ }
+ return shares;
}
static List<Share> getDataSetsPerShare(File storeRoot, String dataStoreCode,
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/SimpleDataSetHelper.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/SimpleDataSetHelper.java
index 5a057cd6350c4accd21f9918402332b9a0a4675e..3e6ca338fe6470c65371e298aec85684b01ac5e7 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/SimpleDataSetHelper.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/SimpleDataSetHelper.java
@@ -21,6 +21,7 @@ import java.util.HashSet;
import java.util.List;
import ch.systemsx.cisd.openbis.generic.shared.dto.DataPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
import ch.systemsx.cisd.openbis.generic.shared.dto.ExternalDataPE;
import ch.systemsx.cisd.openbis.generic.shared.dto.SamplePE;
import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
@@ -74,4 +75,19 @@ public class SimpleDataSetHelper
return result;
}
+
+ public static SimpleDataSetInformationDTO translate(DatasetDescription datasetDescription)
+ {
+ SimpleDataSetInformationDTO result = new SimpleDataSetInformationDTO();
+ result.setDatabaseInstanceCode(datasetDescription.getDatabaseInstanceCode());
+ result.setDataSetCode(datasetDescription.getDatasetCode());
+ result.setDataSetLocation(datasetDescription.getDataSetLocation());
+ result.setDataSetSize(datasetDescription.getDataSetSize());
+ result.setDataSetType(datasetDescription.getDatasetTypeCode());
+ result.setExperimentCode(datasetDescription.getExperimentCode());
+ result.setGroupCode(datasetDescription.getSpaceCode());
+ result.setProjectCode(datasetDescription.getProjectCode());
+ result.setSampleCode(datasetDescription.getSampleCode());
+ return result;
+ }
}