From 6fbada119a9cfc70d53ac26f74979fdfd94051bd Mon Sep 17 00:00:00 2001
From: jakubs <jakubs>
Date: Mon, 3 Nov 2014 15:27:29 +0000
Subject: [PATCH] SSDM-1081 implement basic unarchiving
SVN: 32709
---
.../dss/generic/server/DataStoreService.java | 10 +-
.../server/DataStoreServiceLogger.java | 13 +-
.../AbstractArchiverProcessingPlugin.java | 7 +-
.../IMultiDataSetFileOperationsManager.java | 3 +
.../archiver/IMultiDataSetPackageManager.java | 31 ++++
.../MultiDataSetFileOperationsManager.java | 31 +++-
.../archiver/MultiDataSetPackageManager.java | 77 +++++++++
.../archiver/MultiDatasetArchiver.java | 146 +++++++++++++++++-
...IMultiDataSetArchiverReadonlyQueryDAO.java | 2 +-
.../IMultiDatasetArchiverDBTransaction.java | 2 -
.../MultiDataSetArchiverDataSetDTO.java | 2 +-
.../MultiDatasetArchiverDBTransaction.java | 14 +-
.../MultiDatasetArchiverDataSourceUtil.java | 41 +++++
.../dss/generic/shared/IArchiverPlugin.java | 22 ++-
.../archiver/MultiDatasetArchiverTest.java | 45 ++++--
15 files changed, 384 insertions(+), 62 deletions(-)
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/IMultiDataSetPackageManager.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDataSetPackageManager.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/dataaccess/MultiDatasetArchiverDataSourceUtil.java
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/DataStoreService.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/DataStoreService.java
index abe8fcd4543..4c4921cd888 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/DataStoreService.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/DataStoreService.java
@@ -362,6 +362,13 @@ public class DataStoreService extends AbstractServiceWithLogger<IDataStoreServic
userEmailOrNull);
}
+ @Override
+ public List<String> getDataSetCodesForUnarchiving(String sessionToken, String userSessionToken, List<String> datasets, String userId)
+ {
+ sessionTokenManager.assertValidSessionToken(sessionToken);
+ return getArchiverPlugin().getDataSetCodesForUnarchiving(datasets);
+ }
+
@Override
public void archiveDatasets(String sessionToken, String userSessionToken,
List<DatasetDescription> datasets, String userId, String userEmailOrNull,
@@ -544,8 +551,7 @@ public class DataStoreService extends AbstractServiceWithLogger<IDataStoreServic
sessionTokenManager.assertValidSessionToken(sessionToken);
PluginTaskProvider<ISearchDomainService> provider = pluginTaskInfoProvider.getSearchDomainServiceProvider();
- DatastoreServiceDescription serviceDescription
- = findSearchDomainService(provider, preferredSequenceDatabaseOrNull);
+ DatastoreServiceDescription serviceDescription = findSearchDomainService(provider, preferredSequenceDatabaseOrNull);
if (serviceDescription != null)
{
ISearchDomainService service = provider.getPluginInstance(serviceDescription.getKey());
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/DataStoreServiceLogger.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/DataStoreServiceLogger.java
index 4d92847e6a2..fc6dd9a69b2 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/DataStoreServiceLogger.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/DataStoreServiceLogger.java
@@ -164,6 +164,13 @@ class DataStoreServiceLogger implements IDataStoreService, IInitializable
datasets.size(), userId, userEmailOrNull, removeFromDataStore);
}
+ @Override
+ public List<String> getDataSetCodesForUnarchiving(String sessionToken, String userSessionToken, List<String> datasets, String userId)
+ {
+ log("getDataSetCodesForUnarchiving", "NO_OF_DATASETS(%s)", datasets.size());
+ return null;
+ }
+
@Override
public LinkModel retrieveLinkFromDataSet(String sessionToken, String serviceKey,
DatasetDescription dataSet)
@@ -198,11 +205,11 @@ class DataStoreServiceLogger implements IDataStoreService, IInitializable
}
@Override
- public List<SearchDomainSearchResult> searchForDataSetsWithSequences(String sessionToken,
- String preferredSequenceDatabaseOrNull, String sequenceSnippet,
+ public List<SearchDomainSearchResult> searchForDataSetsWithSequences(String sessionToken,
+ String preferredSequenceDatabaseOrNull, String sequenceSnippet,
Map<String, String> optionalParametersOrNull)
{
- log("searchForDataSetsWithSequences", "SEQUENCE_DATABASE(%s) SEQUENCE_SNIPPET(%s)",
+ log("searchForDataSetsWithSequences", "SEQUENCE_DATABASE(%s) SEQUENCE_SNIPPET(%s)",
preferredSequenceDatabaseOrNull, sequenceSnippet);
return null;
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java
index 085532b1eb5..57b3b504399 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java
@@ -636,7 +636,7 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
}
}
- private IShareIdManager getShareIdManager()
+ protected IShareIdManager getShareIdManager()
{
if (shareIdManager == null)
{
@@ -748,4 +748,9 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
return maximumBatchSizeInBytes;
}
+ @Override
+ public List<String> getDataSetCodesForUnarchiving(List<String> dataSetCodes)
+ {
+ return dataSetCodes;
+ }
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/IMultiDataSetFileOperationsManager.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/IMultiDataSetFileOperationsManager.java
index b97c7ee1534..f42e6743161 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/IMultiDataSetFileOperationsManager.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/IMultiDataSetFileOperationsManager.java
@@ -37,6 +37,9 @@ public interface IMultiDataSetFileOperationsManager
Status deleteContainerFromStage(String containerPath);
+ /**
+ * Get's the content of archived content in final destination.
+ */
IHierarchicalContent getContainerAsHierarchicalContent(String containerPath);
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/IMultiDataSetPackageManager.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/IMultiDataSetPackageManager.java
new file mode 100644
index 00000000000..f9e64ad3e11
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/IMultiDataSetPackageManager.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2014 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.archiver;
+
+import java.io.File;
+import java.util.HashMap;
+
+import ch.systemsx.cisd.common.exceptions.Status;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.IPackageManager;
+
+/**
+ * @author Jakub Straszewski
+ */
+public interface IMultiDataSetPackageManager extends IPackageManager
+{
+ Status extractMultiDataSets(File packageFile, HashMap<String, File> dataSetCodeToDirectory);
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDataSetFileOperationsManager.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDataSetFileOperationsManager.java
index cc6e1baac43..f86b35a62b3 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDataSetFileOperationsManager.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDataSetFileOperationsManager.java
@@ -19,6 +19,7 @@ package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.archiver;
import java.io.File;
import java.io.Serializable;
import java.util.Collection;
+import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
@@ -39,10 +40,8 @@ import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchical
import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContentNode;
import ch.systemsx.cisd.openbis.dss.archiveverifier.batch.VerificationError;
import ch.systemsx.cisd.openbis.dss.generic.server.AbstractDataSetPackager;
-import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.IPackageManager;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.IPathCopierFactory;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.ISshCommandExecutorFactory;
-import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.TarPackageManager;
import ch.systemsx.cisd.openbis.dss.generic.shared.IShareIdManager;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
@@ -73,7 +72,7 @@ public class MultiDataSetFileOperationsManager extends AbstractDataSetFileOperat
private final boolean withSharding;
- protected IPackageManager packageManager;
+ protected IMultiDataSetPackageManager packageManager;
// TODO: some features existing in rsync archiver:
// - ignore existing
@@ -81,7 +80,7 @@ public class MultiDataSetFileOperationsManager extends AbstractDataSetFileOperat
public MultiDataSetFileOperationsManager(Properties properties, IPathCopierFactory pathCopierFactory,
ISshCommandExecutorFactory sshCommandExecutorFactory)
{
- this.packageManager = new TarPackageManager(properties);
+ this.packageManager = new MultiDataSetPackageManager(properties);
this.withSharding = PropertyUtils.getBoolean(properties, WITH_SHARDING_KEY, false);
@@ -142,6 +141,22 @@ public class MultiDataSetFileOperationsManager extends AbstractDataSetFileOperat
return success ? Status.OK : Status.createError("Couldn't delete archive container '" + containerPath);
}
+ public Status restoreDataSetsFromContainerInFinalDestination(String containerPath, String unarchivingShareId,
+ List<DatasetDescription> dataSetDescriptions)
+ {
+ HashMap<String, File> dataSetToLocation = new HashMap<String, File>();
+ for (DatasetDescription datasetDescription : dataSetDescriptions)
+ {
+ File location = getDirectoryProvider().getDataSetDirectory(unarchivingShareId, datasetDescription.getDataSetLocation());
+ dataSetToLocation.put(datasetDescription.getDataSetCode(), location);
+ }
+
+ File stageArchiveContainerFile = new File(getFinalArchive().getDestination(), containerPath);
+ packageManager.extractMultiDataSets(stageArchiveContainerFile, dataSetToLocation);
+
+ return Status.OK;
+ }
+
@Override
public Status createContainerInStage(String containerPath, List<DatasetDescription> datasetDescriptions)
{
@@ -158,16 +173,16 @@ public class MultiDataSetFileOperationsManager extends AbstractDataSetFileOperat
shareIdManager.lock(dataSet.getCode());
operationLog.info("Archive dataset " + dataSet.getCode() + " in " + containerPath);
}
-
+
boolean result = createFolderIfNotExists(stageArchive, stageArchiveContainerFile.getParentFile());
-
+
// TODO: react somehow?
if (result)
{
operationLog.warn("File already exists in archive " + stageArchiveContainerFile.getParentFile());
}
-
- packageManager.create(stageArchiveContainerFile, dataSets); // packagemanager
+
+ packageManager.create(stageArchiveContainerFile, dataSets);
} catch (Exception ex)
{
status = Status.createError(ex.toString());
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDataSetPackageManager.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDataSetPackageManager.java
new file mode 100644
index 00000000000..7f3035b13ab
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDataSetPackageManager.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright 2014 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.archiver;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Properties;
+
+import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
+import ch.systemsx.cisd.common.exceptions.Status;
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
+import ch.systemsx.cisd.common.filesystem.tar.Untar;
+import ch.systemsx.cisd.openbis.dss.generic.server.AbstractDataSetPackager;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.TarPackageManager;
+
+/**
+ * @author Jakub Straszewski
+ */
+public class MultiDataSetPackageManager extends TarPackageManager implements IMultiDataSetPackageManager
+{
+
+ public MultiDataSetPackageManager(Properties properties)
+ {
+ super(properties);
+ }
+
+ @Override
+ public Status extractMultiDataSets(File packageFile, HashMap<String, File> dataSetCodeToDirectory)
+ {
+ Untar untar = null;
+ try
+ {
+ untar = new Untar(packageFile);
+ untar.extract(dataSetCodeToDirectory);
+
+ for (File location : dataSetCodeToDirectory.values())
+ {
+ File metadataFile = new File(location, AbstractDataSetPackager.META_DATA_FILE_NAME);
+ if (metadataFile.exists() && metadataFile.isFile())
+ {
+ FileUtilities.delete(metadataFile);
+ }
+ }
+ return Status.OK;
+ } catch (Exception ex)
+ {
+ return Status.createError(ex.toString());
+ } finally
+ {
+ if (untar != null)
+ {
+ try
+ {
+ untar.close();
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+ }
+ }
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDatasetArchiver.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDatasetArchiver.java
index 71d5d8119eb..11fcd67a9b0 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDatasetArchiver.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDatasetArchiver.java
@@ -18,6 +18,7 @@ package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.archiver;
import java.io.File;
import java.io.Serializable;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
@@ -35,12 +36,16 @@ import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.AbstractArch
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.RsyncArchiveCopierFactory;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.RsyncArchiver;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.SshCommandExecutorFactory;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.archiver.dataaccess.IMultiDataSetArchiverReadonlyQueryDAO;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.archiver.dataaccess.IMultiDatasetArchiverDBTransaction;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.archiver.dataaccess.MultiDataSetArchiverContainerDTO;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.archiver.dataaccess.MultiDataSetArchiverDataSetDTO;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.archiver.dataaccess.MultiDatasetArchiverDBTransaction;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.archiver.dataaccess.MultiDatasetArchiverDataSourceUtil;
import ch.systemsx.cisd.openbis.dss.generic.shared.ArchiverTaskContext;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IDatasetLocation;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.PhysicalDataSet;
import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
/**
@@ -85,12 +90,19 @@ public class MultiDatasetArchiver extends AbstractArchiverProcessingPlugin
private IMultiDatasetArchiverDBTransaction transaction;
+ private IMultiDataSetArchiverReadonlyQueryDAO readonlyQuery;
+
+ public static final String UNARCHIVING_SHARE_ID = "unarchiving-share-id";
+
+ private final String unarchivingShareId;
+
public MultiDatasetArchiver(Properties properties, File storeRoot)
{
super(properties, storeRoot, null, null);
this.minimumContainerSize = PropertyUtils.getLong(properties, MINIMUM_CONTAINER_SIZE_IN_BYTES, DEFAULT_MINIMUM_CONTAINER_SIZE_IN_BYTES);
this.maximumContainerSize = PropertyUtils.getLong(properties, MAXIMUM_CONTAINER_SIZE_IN_BYTES, DEFAULT_MAXIMUM_CONTAINER_SIZE_IN_BYTES);
this.fileOperationsFactory = new FileOperationsManagerFactory(properties);
+ this.unarchivingShareId = PropertyUtils.getMandatoryProperty(properties, UNARCHIVING_SHARE_ID);
}
@Override
@@ -310,13 +322,124 @@ public class MultiDatasetArchiver extends AbstractArchiverProcessingPlugin
}
@Override
- protected DatasetProcessingStatuses doUnarchive(List<DatasetDescription> datasets, ArchiverTaskContext context)
+ public List<String> getDataSetCodesForUnarchiving(List<String> dataSetCodes)
+ {
+ assertAllDataSetsInTheSameContainer(dataSetCodes);
+ return getCodesOfAllDataSetsInContainer(dataSetCodes);
+ }
+
+ @Override
+ protected DatasetProcessingStatuses doUnarchive(List<DatasetDescription> parameterDataSets, ArchiverTaskContext context)
+ {
+ List<String> dataSetCodes = translateToDataSetCodes(parameterDataSets);
+ long containerId = assertAllDataSetsInTheSameContainer(dataSetCodes);
+ List<PhysicalDataSet> dataSets = translateToPhysicalDataSets(dataSetCodes);
+ assertNoAvailableDatasets(dataSets);
+
+ for (PhysicalDataSet physicalDataSet : dataSets)
+ {
+ String dataSetCode = physicalDataSet.getCode();
+
+ String shareId = getShareIdManager().getShareId(dataSetCode);
+ if (shareId.equals(unarchivingShareId) == false)
+ {
+ getService().updateShareIdAndSize(dataSetCode, unarchivingShareId, physicalDataSet.getSize());
+ getShareIdManager().setShareId(dataSetCode, unarchivingShareId);
+ }
+
+ }
+
+ MultiDataSetArchiverContainerDTO container = getReadonlyQuery().getContainerForId(containerId);
+
+ ((MultiDataSetFileOperationsManager) getFileOperations()).restoreDataSetsFromContainerInFinalDestination(
+ container.getPath(),
+
+ unarchivingShareId, parameterDataSets);
+
+ DatasetProcessingStatuses result = new DatasetProcessingStatuses();
+ result.addResult(parameterDataSets, Status.OK, Operation.UNARCHIVE);
+ return result;
+ }
+
+ private void assertNoAvailableDatasets(List<PhysicalDataSet> dataSets)
+ {
+ for (PhysicalDataSet physicalDataSet : dataSets)
+ {
+ if (physicalDataSet.isAvailable())
+ {
+ throw new IllegalArgumentException("Dataset '" + physicalDataSet.getCode() + "'specified for unarchiving is available");
+ }
+ }
+ }
+
+ private List<String> getCodesOfAllDataSetsInContainer(List<String> dataSetCodes)
+ {
+
+ MultiDataSetArchiverDataSetDTO dataset = getReadonlyQuery().getDataSetForCode(dataSetCodes.get(0));
+ Long containerId = dataset.getContainerId();
+ List<MultiDataSetArchiverDataSetDTO> dbDataSets = getReadonlyQuery().listDataSetsForContainerId(containerId);
+
+ List<String> enhancedDataSetCodes = new LinkedList<String>();
+ for (MultiDataSetArchiverDataSetDTO dbDataSet : dbDataSets)
+ {
+ enhancedDataSetCodes.add(dbDataSet.getCode());
+ }
+ return enhancedDataSetCodes;
+ }
+
+ private List<PhysicalDataSet> translateToPhysicalDataSets(List<String> dataSetCodes)
+ {
+ List<PhysicalDataSet> result = new LinkedList<PhysicalDataSet>();
+ for (AbstractExternalData dataSet : getService().listDataSetsByCode(dataSetCodes))
+ {
+ if (dataSet.tryGetAsDataSet() != null)
+ {
+ result.add(dataSet.tryGetAsDataSet());
+ }
+ else
+ {
+ throw new IllegalStateException("All data sets in container are expected to be physical datasets, but data set '" + dataSet.getCode()
+ + "' is not ");
+ }
+ }
+ return result;
+ }
+
+ private List<String> translateToDataSetCodes(List<? extends IDatasetLocation> dataSets)
+ {
+ LinkedList<String> result = new LinkedList<String>();
+ for (IDatasetLocation dataSet : dataSets)
+ {
+ result.add(dataSet.getDataSetCode());
+ }
+ return result;
+ }
+
+ /**
+ * @return ID of container that groups all listed data sets
+ * @throws exception if not all data sets are in the same container
+ */
+ private long assertAllDataSetsInTheSameContainer(List<String> dataSetCodes)
{
- if (datasets.size() > 0)
+ HashSet<Long> containerIds = new HashSet<Long>();
+ long containerId = -1;
+ for (String code : dataSetCodes)
+ {
+ MultiDataSetArchiverDataSetDTO dataSet = getTransaction().getDataSetForCode(code);
+ if (dataSet == null)
+ {
+ throw new IllegalArgumentException("Dataset " + code
+ + " was selected for unarchiving, but is not present in the archive");
+ }
+ containerIds.add(dataSet.getContainerId());
+ containerId = dataSet.getContainerId();
+ }
+ if (containerIds.size() > 1)
{
- throw new NotImplementedException("Unarchiving is not yet implemented for multi dataset archiver");
+ throw new IllegalArgumentException("Datasets selected for unarchiving do not all belong to one container, but to " + containerIds.size()
+ + " different containers");
}
- return new DatasetProcessingStatuses();
+ return containerId;
}
@Override
@@ -353,7 +476,8 @@ public class MultiDatasetArchiver extends AbstractArchiverProcessingPlugin
return dataSetInArchiveDB != null;
}
- @Private IMultiDataSetFileOperationsManager getFileOperations()
+ @Private
+ IMultiDataSetFileOperationsManager getFileOperations()
{
if (fileOperations == null)
{
@@ -362,7 +486,8 @@ public class MultiDatasetArchiver extends AbstractArchiverProcessingPlugin
return fileOperations;
}
- @Private IMultiDatasetArchiverDBTransaction getTransaction()
+ @Private
+ IMultiDatasetArchiverDBTransaction getTransaction()
{
if (transaction == null)
{
@@ -370,4 +495,13 @@ public class MultiDatasetArchiver extends AbstractArchiverProcessingPlugin
}
return transaction;
}
+
+ IMultiDataSetArchiverReadonlyQueryDAO getReadonlyQuery()
+ {
+ if (readonlyQuery == null)
+ {
+ readonlyQuery = MultiDatasetArchiverDataSourceUtil.getReadonlyQueryDAO();
+ }
+ return readonlyQuery;
+ }
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/dataaccess/IMultiDataSetArchiverReadonlyQueryDAO.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/dataaccess/IMultiDataSetArchiverReadonlyQueryDAO.java
index 31c0c393ab5..12eee012fb5 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/dataaccess/IMultiDataSetArchiverReadonlyQueryDAO.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/dataaccess/IMultiDataSetArchiverReadonlyQueryDAO.java
@@ -31,7 +31,7 @@ public interface IMultiDataSetArchiverReadonlyQueryDAO extends BaseQuery
* SELECT CONTAINER
*/
final static String SELECT_CONTAINER =
- " SELECT id, path"
+ " SELECT id, path "
+ "FROM containers ";
@Select(sql = SELECT_CONTAINER + "WHERE id = ?{1}")
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/dataaccess/IMultiDatasetArchiverDBTransaction.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/dataaccess/IMultiDatasetArchiverDBTransaction.java
index 4d72ee67b2a..bf86efbe9a7 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/dataaccess/IMultiDatasetArchiverDBTransaction.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/dataaccess/IMultiDatasetArchiverDBTransaction.java
@@ -21,8 +21,6 @@ import java.util.List;
import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
/**
- *
- *
* @author Franz-Josef Elmer
*/
public interface IMultiDatasetArchiverDBTransaction
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/dataaccess/MultiDataSetArchiverDataSetDTO.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/dataaccess/MultiDataSetArchiverDataSetDTO.java
index 1243583172e..e77121c331e 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/dataaccess/MultiDataSetArchiverDataSetDTO.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/dataaccess/MultiDataSetArchiverDataSetDTO.java
@@ -28,7 +28,7 @@ public class MultiDataSetArchiverDataSetDTO
private String code;
- @ResultColumn("CNTR_ID")
+ @ResultColumn("CTNR_ID")
private long containerId;
@ResultColumn("SIZE_IN_BYTES")
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/dataaccess/MultiDatasetArchiverDBTransaction.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/dataaccess/MultiDatasetArchiverDBTransaction.java
index bcdd19235ca..191e77f3796 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/dataaccess/MultiDatasetArchiverDBTransaction.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/dataaccess/MultiDatasetArchiverDBTransaction.java
@@ -18,11 +18,6 @@ package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.archiver.da
import java.util.List;
-import javax.sql.DataSource;
-
-import net.lemnik.eodsql.QueryTool;
-
-import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
/**
@@ -31,18 +26,11 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
public class MultiDatasetArchiverDBTransaction implements IMultiDatasetArchiverDBTransaction
{
- private static DataSource dataSource = ServiceProvider.getDataSourceProvider().getDataSource("multi-dataset-archiver-db");
-
private IMultiDataSetArchiverQueryDAO transaction;
public MultiDatasetArchiverDBTransaction()
{
- this.transaction = getTransactionalQuery();
- }
-
- private static IMultiDataSetArchiverQueryDAO getTransactionalQuery()
- {
- return QueryTool.getQuery(dataSource, IMultiDataSetArchiverQueryDAO.class);
+ this.transaction = MultiDatasetArchiverDataSourceUtil.getTransactionalQuery();
}
@Override
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/dataaccess/MultiDatasetArchiverDataSourceUtil.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/dataaccess/MultiDatasetArchiverDataSourceUtil.java
new file mode 100644
index 00000000000..2dcf48d0c8c
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/dataaccess/MultiDatasetArchiverDataSourceUtil.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2014 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.archiver.dataaccess;
+
+import javax.sql.DataSource;
+
+import net.lemnik.eodsql.QueryTool;
+
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
+
+/**
+ * @author Jakub Straszewski
+ */
+public class MultiDatasetArchiverDataSourceUtil
+{
+ private static DataSource dataSource = ServiceProvider.getDataSourceProvider().getDataSource("multi-dataset-archiver-db");
+
+ static IMultiDataSetArchiverQueryDAO getTransactionalQuery()
+ {
+ return QueryTool.getQuery(dataSource, IMultiDataSetArchiverQueryDAO.class);
+ }
+
+ public static IMultiDataSetArchiverReadonlyQueryDAO getReadonlyQueryDAO()
+ {
+ return QueryTool.getQuery(dataSource, IMultiDataSetArchiverReadonlyQueryDAO.class);
+ }
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IArchiverPlugin.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IArchiverPlugin.java
index d7aad7b503b..4aac4c281aa 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IArchiverPlugin.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IArchiverPlugin.java
@@ -27,15 +27,15 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
*
* @author Piotr Buczek
* @author Kaloyan Enimanev
+ * @author Jakub Straszewski
*/
public interface IArchiverPlugin extends Serializable
{
/**
* Asynchronously processes archiving of the specified datasets.
*
- * @returns {@link ProcessingStatus} of the finished processing with statuses of processing for
- * all scheduled data sets or null if processing succeeded for all datasets and no
- * additional information is provided.
+ * @returns {@link ProcessingStatus} of the finished processing with statuses of processing for all scheduled data sets or null if processing
+ * succeeded for all datasets and no additional information is provided.
*/
ProcessingStatus archive(List<DatasetDescription> datasets, ArchiverTaskContext context,
boolean removeFromDataStore);
@@ -43,17 +43,23 @@ public interface IArchiverPlugin extends Serializable
/**
* Asynchronously processes unarchiving of the specified datasets.
*
- * @returns {@link ProcessingStatus} of the finished processing with statuses of processing for
- * all scheduled data sets or null if processing succeeded for all datasets and no
- * additional information is provided.
+ * @returns {@link ProcessingStatus} of the finished processing with statuses of processing for all scheduled data sets or null if processing
+ * succeeded for all datasets and no additional information is provided.
*/
ProcessingStatus unarchive(List<DatasetDescription> datasets, ArchiverTaskContext context);
+ /**
+ * Enhances the list of data set codes, so that it contains all datasets that should be unarchived together in one batch.
+ *
+ * @returns the list that should be a superset {@code dataSetCodes}
+ */
+ List<String> getDataSetCodesForUnarchiving(List<String> dataSetCodes);
+
/**
* Delete data sets from the archive.
*
- * @returns {@link ProcessingStatus} containing the deletion statuses for all data sets or null
- * if processing succeeded for all datasets and no additional information is provided.
+ * @returns {@link ProcessingStatus} containing the deletion statuses for all data sets or null if processing succeeded for all datasets and no
+ * additional information is provided.
*/
ProcessingStatus deleteFromArchive(List<DatasetLocation> datasets);
}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDatasetArchiverTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDatasetArchiverTest.java
index 208b1c9d2e7..5cdbc941cf5 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDatasetArchiverTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDatasetArchiverTest.java
@@ -20,12 +20,12 @@ import static ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.archi
import static ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.archiver.MultiDataSetFileOperationsManager.STAGING_DESTINATION_KEY;
import static ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.archiver.MultiDatasetArchiver.MAXIMUM_CONTAINER_SIZE_IN_BYTES;
import static ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.archiver.MultiDatasetArchiver.MINIMUM_CONTAINER_SIZE_IN_BYTES;
+import static ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.archiver.MultiDatasetArchiver.UNARCHIVING_SHARE_ID;
import java.io.BufferedReader;
import java.io.File;
import java.io.InputStream;
import java.io.InputStreamReader;
-import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -33,8 +33,6 @@ import java.util.Comparator;
import java.util.List;
import java.util.Properties;
-import javassist.bytecode.analysis.MultiArrayType;
-
import org.apache.log4j.Level;
import org.jmock.Expectations;
import org.jmock.Mockery;
@@ -54,6 +52,7 @@ import ch.systemsx.cisd.common.time.TimingParameters;
import ch.systemsx.cisd.openbis.common.io.hierarchical_content.TarBasedHierarchicalContent;
import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContentNode;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.MockDataSetDirectoryProvider;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.archiver.dataaccess.IMultiDataSetArchiverReadonlyQueryDAO;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.archiver.dataaccess.IMultiDatasetArchiverDBTransaction;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.archiver.dataaccess.MultiDataSetArchiverContainerDTO;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.archiver.dataaccess.MultiDataSetArchiverDataSetDTO;
@@ -208,14 +207,16 @@ public class MultiDatasetArchiverTest extends AbstractFileSystemTestCase
private IMultiDataSetFileOperationsManager fileManager;
+ private IMultiDataSetArchiverReadonlyQueryDAO readonlyDAO;
+
public MockMultiDatasetArchiver(Properties properties, File storeRoot,
IEncapsulatedOpenBISService openBISService, IShareIdManager shareIdManager,
IDataSetStatusUpdater statusUpdater, IMultiDatasetArchiverDBTransaction transaction,
- IMultiDataSetFileOperationsManager fileManager)
+ IMultiDataSetFileOperationsManager fileManager, IMultiDataSetArchiverReadonlyQueryDAO readonlyDAO)
{
super(properties, storeRoot);
this.transaction = transaction;
- this.fileManager = fileManager;
+ this.readonlyDAO = readonlyDAO;
setService(openBISService);
setShareIdManager(shareIdManager);
setStatusUpdater(statusUpdater);
@@ -232,6 +233,12 @@ public class MultiDatasetArchiverTest extends AbstractFileSystemTestCase
{
return transaction;
}
+
+ @Override
+ IMultiDataSetArchiverReadonlyQueryDAO getReadonlyQuery()
+ {
+ return readonlyDAO;
+ }
}
private static final String EXPERIMENT_IDENTIFIER = "/S/P/E";
@@ -242,6 +249,8 @@ public class MultiDatasetArchiverTest extends AbstractFileSystemTestCase
private IMultiDatasetArchiverDBTransaction transaction;
+ private IMultiDataSetArchiverReadonlyQueryDAO readonlyDAO;
+
private IDataSetDirectoryProvider directoryProvider;
private IHierarchicalContentProvider hierarchicalContentProvider;
@@ -311,9 +320,11 @@ public class MultiDatasetArchiverTest extends AbstractFileSystemTestCase
properties = new Properties();
properties.setProperty(STAGING_DESTINATION_KEY, staging.getAbsolutePath());
properties.setProperty(FINAL_DESTINATION_KEY, archive.getAbsolutePath());
+ properties.setProperty(UNARCHIVING_SHARE_ID, "2");
directoryProvider = new MockDataSetDirectoryProvider(store, share.getName(), shareIdManager);
archiverContext = new ArchiverTaskContext(directoryProvider, hierarchicalContentProvider);
experiment = new ExperimentBuilder().identifier(EXPERIMENT_IDENTIFIER).type("MET").getExperiment();
+ readonlyDAO = context.mock(IMultiDataSetArchiverReadonlyQueryDAO.class);
context.checking(new Expectations()
{
{
@@ -341,28 +352,28 @@ public class MultiDatasetArchiverTest extends AbstractFileSystemTestCase
// To following line of code should also be called at the end of each test method.
// Otherwise one does not known which test failed.
context.assertIsSatisfied();
-
+
ServiceProviderTestWrapper.restoreApplicationContext();
}
-
+
@Test
public void testArchiveDataSetsWhichAreTooSmall()
{
prepareUpdateShareIdAndSize(ds1, 10);
prepareUpdateShareIdAndSize(ds2, 20);
properties.setProperty(MINIMUM_CONTAINER_SIZE_IN_BYTES, "35");
-
+
MultiDatasetArchiver archiver = createArchiver(null);
ProcessingStatus status = archiver.archive(Arrays.asList(ds1, ds2), archiverContext, false);
-
+
assertEquals("[ERROR: \"Set of data sets specified for archiving is too small (30 bytes) "
- + "to be archived with multi dataset archiver because minimum size is 35 bytes.\"]",
+ + "to be archived with multi dataset archiver because minimum size is 35 bytes.\"]",
status.getErrorStatuses().toString());
assertEquals("[ds1, ds2]: AVAILABLE false\n", statusUpdater.toString());
assertEquals("Containers:\nData sets:\ncomitted: false, rolledBack: true, closed: true", transaction.toString());
context.assertIsSatisfied();
}
-
+
@Test
public void testArchiveDataSetsWhichAreTooBig()
{
@@ -370,12 +381,12 @@ public class MultiDatasetArchiverTest extends AbstractFileSystemTestCase
prepareUpdateShareIdAndSize(ds2, 20);
properties.setProperty(MINIMUM_CONTAINER_SIZE_IN_BYTES, "25");
properties.setProperty(MAXIMUM_CONTAINER_SIZE_IN_BYTES, "27");
-
+
MultiDatasetArchiver archiver = createArchiver(null);
ProcessingStatus status = archiver.archive(Arrays.asList(ds1, ds2), archiverContext, false);
-
+
assertEquals("[ERROR: \"Set of data sets specified for archiving is too big (30 bytes) "
- + "to be archived with multi dataset archiver because maximum size is 27 bytes.\"]",
+ + "to be archived with multi dataset archiver because maximum size is 27 bytes.\"]",
status.getErrorStatuses().toString());
assertEquals("[ds1, ds2]: AVAILABLE false\n", statusUpdater.toString());
assertEquals("Containers:\nData sets:\ncomitted: false, rolledBack: true, closed: true", transaction.toString());
@@ -651,8 +662,8 @@ public class MultiDatasetArchiverTest extends AbstractFileSystemTestCase
context.checking(new Expectations()
{
{
- one(dataSetDeleter).scheduleDeletionOfDataSets(Arrays.asList(dataSets),
- TimingParameters.DEFAULT_MAXIMUM_RETRY_COUNT,
+ one(dataSetDeleter).scheduleDeletionOfDataSets(Arrays.asList(dataSets),
+ TimingParameters.DEFAULT_MAXIMUM_RETRY_COUNT,
TimingParameters.DEFAULT_INTERVAL_TO_WAIT_AFTER_FAILURE_SECONDS);
}
});
@@ -704,7 +715,7 @@ public class MultiDatasetArchiverTest extends AbstractFileSystemTestCase
private MultiDatasetArchiver createArchiver(IMultiDataSetFileOperationsManager fileManagerOrNull)
{
return new MockMultiDatasetArchiver(properties, store, openBISService, shareIdManager, statusUpdater,
- transaction, fileManagerOrNull);
+ transaction, fileManagerOrNull, readonlyDAO);
}
private DatasetDescription dataSet(final String code, String content)
--
GitLab