From e9a61c078461da22a72e418ba28a11554cf3c274 Mon Sep 17 00:00:00 2001
From: felmer <felmer>
Date: Tue, 28 Oct 2014 13:36:21 +0000
Subject: [PATCH] SSDM-1081: Some refactorings: Introducing
AbstractPackageManager, extracting method
MultiDatasetArchiver.checkArchivedDataSets(). Making
MultiDataSetFileOperationsManager.createContainerInStage() more robust
related to locking/unlocking data sets.
SVN: 32679
---
.../standard/AbstractPackageManager.java | 97 +++++++++++++++++++
.../plugins/standard/TarPackageManager.java | 73 +-------------
.../plugins/standard/ZipPackageManager.java | 70 +------------
.../archiver/ArchiveDestinationFactory.java | 4 +-
.../MultiDataSetFileOperationsManager.java | 41 ++++----
.../archiver/MultiDatasetArchiver.java | 63 ++++++------
6 files changed, 160 insertions(+), 188 deletions(-)
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractPackageManager.java
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractPackageManager.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractPackageManager.java
new file mode 100644
index 00000000000..a0ca986416b
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractPackageManager.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright 2014 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.List;
+import java.util.Properties;
+
+import ch.systemsx.cisd.common.time.TimingParameters;
+import ch.systemsx.cisd.openbis.dss.generic.server.AbstractDataSetPackager;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IDataSetDirectoryProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.utils.DataSetExistenceChecker;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
+
+/**
+ *
+ *
+ * @author Franz-Josef Elmer
+ */
+abstract class AbstractPackageManager implements IPackageManager
+{
+ private transient IHierarchicalContentProvider contentProvider;
+
+ private transient IDataSetDirectoryProvider directoryProvider;
+
+ @Override
+ public void create(File packageFile, AbstractExternalData dataSet)
+ {
+ create(packageFile, Collections.singletonList(dataSet), false);
+ }
+
+ @Override
+ public void create(File packageFile, List<AbstractExternalData> dataSets)
+ {
+ create(packageFile, dataSets, true);
+ }
+
+ private void create(File packageFile, List<AbstractExternalData> dataSets, boolean withPathPrefix)
+ {
+ AbstractDataSetPackager packager = null;
+
+ try
+ {
+ DataSetExistenceChecker existenceChecker =
+ new DataSetExistenceChecker(getDirectoryProvider(), TimingParameters.create(new Properties()));
+ packager = createPackager(packageFile, existenceChecker);
+
+ for (AbstractExternalData dataSet : dataSets)
+ {
+ packager.addDataSetTo(withPathPrefix ? dataSet.getCode() + "/" : "", dataSet);
+ }
+ } finally
+ {
+ if (packager != null)
+ {
+ packager.close();
+ }
+ }
+ }
+
+ protected abstract AbstractDataSetPackager createPackager(File packageFile, DataSetExistenceChecker existenceChecker);
+
+
+ protected IHierarchicalContentProvider getContentProvider()
+ {
+ if (contentProvider == null)
+ {
+ contentProvider = ServiceProvider.getHierarchicalContentProvider();
+ }
+ return contentProvider;
+ }
+
+ private IDataSetDirectoryProvider getDirectoryProvider()
+ {
+ if (directoryProvider == null)
+ {
+ directoryProvider = ServiceProvider.getDataStoreService().getDataSetDirectoryProvider();
+ }
+ return directoryProvider;
+ }}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/TarPackageManager.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/TarPackageManager.java
index 552423bdcae..1cd608065cd 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/TarPackageManager.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/TarPackageManager.java
@@ -27,28 +27,19 @@ import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.common.filesystem.tar.Untar;
import ch.systemsx.cisd.common.properties.PropertyUtils;
-import ch.systemsx.cisd.common.time.TimingParameters;
import ch.systemsx.cisd.openbis.common.io.hierarchical_content.TarBasedHierarchicalContent;
import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContent;
import ch.systemsx.cisd.openbis.dss.archiveverifier.batch.VerificationError;
import ch.systemsx.cisd.openbis.dss.generic.server.AbstractDataSetPackager;
import ch.systemsx.cisd.openbis.dss.generic.server.TarDataSetPackager;
-import ch.systemsx.cisd.openbis.dss.generic.shared.IDataSetDirectoryProvider;
-import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
-import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.utils.DataSetExistenceChecker;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
/**
* @author pkupczyk
*/
-public class TarPackageManager implements IPackageManager
+public class TarPackageManager extends AbstractPackageManager
{
- private transient IHierarchicalContentProvider contentProvider;
-
- private transient IDataSetDirectoryProvider directoryProvider;
-
private final File tempFolder;
public TarPackageManager(Properties properties)
@@ -63,49 +54,11 @@ public class TarPackageManager implements IPackageManager
}
@Override
- public void create(File packageFile, AbstractExternalData dataSet)
- {
- TarDataSetPackager packager = null;
-
- try
- {
- DataSetExistenceChecker existenceChecker =
- new DataSetExistenceChecker(getDirectoryProvider(), TimingParameters.create(new Properties()));
- packager = new TarDataSetPackager(packageFile, getContentProvider(), existenceChecker);
- packager.addDataSetTo("", dataSet);
- } finally
- {
- if (packager != null)
- {
- packager.close();
- }
- }
- }
-
- @Override
- public void create(File packageFile, List<AbstractExternalData> dataSets)
+ protected AbstractDataSetPackager createPackager(File packageFile, DataSetExistenceChecker existenceChecker)
{
- TarDataSetPackager packager = null;
-
- try
- {
- DataSetExistenceChecker existenceChecker =
- new DataSetExistenceChecker(getDirectoryProvider(), TimingParameters.create(new Properties()));
- packager = new TarDataSetPackager(packageFile, getContentProvider(), existenceChecker);
-
- for (AbstractExternalData dataSet : dataSets)
- {
- packager.addDataSetTo(dataSet.getCode() + "/", dataSet);
- }
- } finally
- {
- if (packager != null)
- {
- packager.close();
- }
- }
+ return new TarDataSetPackager(packageFile, getContentProvider(), existenceChecker);
}
-
+
@Override
public List<VerificationError> verify(File packageFile)
{
@@ -152,22 +105,4 @@ public class TarPackageManager implements IPackageManager
return new TarBasedHierarchicalContent(packageFile, tempFolder);
}
- private IHierarchicalContentProvider getContentProvider()
- {
- if (contentProvider == null)
- {
- contentProvider = ServiceProvider.getHierarchicalContentProvider();
- }
- return contentProvider;
- }
-
- private IDataSetDirectoryProvider getDirectoryProvider()
- {
- if (directoryProvider == null)
- {
- directoryProvider = ServiceProvider.getDataStoreService().getDataSetDirectoryProvider();
- }
- return directoryProvider;
- }
-
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/ZipPackageManager.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/ZipPackageManager.java
index 0d111c5ed5e..af559522f42 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/ZipPackageManager.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/ZipPackageManager.java
@@ -32,18 +32,13 @@ import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.properties.PropertyUtils;
-import ch.systemsx.cisd.common.time.TimingParameters;
import ch.systemsx.cisd.openbis.common.io.hierarchical_content.ZipBasedHierarchicalContent;
import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContent;
import ch.systemsx.cisd.openbis.dss.archiveverifier.batch.VerificationError;
import ch.systemsx.cisd.openbis.dss.archiveverifier.verifier.ZipFileIntegrityVerifier;
import ch.systemsx.cisd.openbis.dss.generic.server.AbstractDataSetPackager;
import ch.systemsx.cisd.openbis.dss.generic.server.ZipDataSetPackager;
-import ch.systemsx.cisd.openbis.dss.generic.shared.IDataSetDirectoryProvider;
-import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
-import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.utils.DataSetExistenceChecker;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
import de.schlichtherle.io.rof.SimpleReadOnlyFile;
import de.schlichtherle.util.zip.BasicZipFile;
@@ -52,17 +47,13 @@ import de.schlichtherle.util.zip.ZipEntry;
/**
* @author pkupczyk
*/
-public class ZipPackageManager implements IPackageManager
+public class ZipPackageManager extends AbstractPackageManager
{
static final String COMPRESS_KEY = "compressing";
private boolean compress;
- private transient IHierarchicalContentProvider contentProvider;
-
- private transient IDataSetDirectoryProvider directoryProvider;
-
public ZipPackageManager(Properties properties)
{
compress = PropertyUtils.getBoolean(properties, COMPRESS_KEY, true);
@@ -75,46 +66,9 @@ public class ZipPackageManager implements IPackageManager
}
@Override
- public void create(File packageFile, AbstractExternalData dataSet)
+ protected AbstractDataSetPackager createPackager(File packageFile, DataSetExistenceChecker existenceChecker)
{
- ZipDataSetPackager packager = null;
-
- try
- {
- DataSetExistenceChecker existenceChecker =
- new DataSetExistenceChecker(getDirectoryProvider(), TimingParameters.create(new Properties()));
- packager = new ZipDataSetPackager(packageFile, compress, getContentProvider(), existenceChecker);
- packager.addDataSetTo("", dataSet);
- } finally
- {
- if (packager != null)
- {
- packager.close();
- }
- }
- }
-
- @Override
- public void create(File packageFile, List<AbstractExternalData> dataSets)
- {
- ZipDataSetPackager packager = null;
-
- try
- {
- DataSetExistenceChecker existenceChecker =
- new DataSetExistenceChecker(getDirectoryProvider(), TimingParameters.create(new Properties()));
- packager = new ZipDataSetPackager(packageFile, compress, getContentProvider(), existenceChecker);
- for (AbstractExternalData dataSet : dataSets)
- {
- packager.addDataSetTo(dataSet.getCode(), dataSet);
- }
- } finally
- {
- if (packager != null)
- {
- packager.close();
- }
- }
+ return new ZipDataSetPackager(packageFile, compress, getContentProvider(), existenceChecker);
}
@Override
@@ -190,22 +144,4 @@ public class ZipPackageManager implements IPackageManager
return new ZipBasedHierarchicalContent(packageFile);
}
- private IHierarchicalContentProvider getContentProvider()
- {
- if (contentProvider == null)
- {
- contentProvider = ServiceProvider.getHierarchicalContentProvider();
- }
- return contentProvider;
- }
-
- private IDataSetDirectoryProvider getDirectoryProvider()
- {
- if (directoryProvider == null)
- {
- directoryProvider = ServiceProvider.getDataStoreService().getDataSetDirectoryProvider();
- }
- return directoryProvider;
- }
-
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/ArchiveDestinationFactory.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/ArchiveDestinationFactory.java
index 9e51b57fd22..66022748023 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/ArchiveDestinationFactory.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/ArchiveDestinationFactory.java
@@ -80,7 +80,7 @@ public class ArchiveDestinationFactory implements Serializable
return new ArchiveDestination(destination, executor, isHosted, timeoutInMillis);
}
- protected static RemoteDataSetFileOperationsExecutor createRemoteDataSetFileOperationsExecutor(Properties properties,
+ private static RemoteDataSetFileOperationsExecutor createRemoteDataSetFileOperationsExecutor(Properties properties,
IPathCopierFactory pathCopierFactory, ISshCommandExecutorFactory sshCommandExecutorFactory, HostAwareFile hostAwareFile,
long timeoutInMillis)
{
@@ -103,7 +103,7 @@ public class ArchiveDestinationFactory implements Serializable
return result;
}
- protected static LocalDataSetFileOperationsExcecutor createLocalDataSetFileOperationsExecutor(Properties properties,
+ private static LocalDataSetFileOperationsExcecutor createLocalDataSetFileOperationsExecutor(Properties properties,
IPathCopierFactory pathCopierFactory,
HostAwareFile hostAwareFile, long timeoutInMillis)
{
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDataSetFileOperationsManager.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDataSetFileOperationsManager.java
index 308fd95ba46..be2fbad4bda 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDataSetFileOperationsManager.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDataSetFileOperationsManager.java
@@ -143,31 +143,28 @@ public class MultiDataSetFileOperationsManager extends AbstractDataSetFileOperat
public Status createContainerInStage(String containerPath, List<DatasetDescription> datasetDescriptions)
{
- List<AbstractExternalData> dataSets = new LinkedList<AbstractExternalData>();
-
- IShareIdManager shareIdManager = getDirectoryProvider().getShareIdManager();
-
- for (DatasetDescription datasetDescription : datasetDescriptions)
- {
- AbstractExternalData dataSet = getDataSetWithAllMetaData(datasetDescription);
- dataSets.add(dataSet);
- shareIdManager.lock(dataSet.getCode());
- operationLog.info("Archive dataset " + dataSet.getCode() + " in " + containerPath);
- }
-
File stageArchiveContainerFile = new File(getStageArchive().getDestination(), containerPath);
-
- boolean result = createFolderIfNotExists(stageArchive, stageArchiveContainerFile.getParentFile());
-
- // TODO: react somehow?
- if (result)
- {
- operationLog.warn("File already exists in archive " + stageArchiveContainerFile.getParentFile());
- }
-
+ IShareIdManager shareIdManager = getDirectoryProvider().getShareIdManager();
Status status = Status.OK;
try
{
+ List<AbstractExternalData> dataSets = new LinkedList<AbstractExternalData>();
+ for (DatasetDescription datasetDescription : datasetDescriptions)
+ {
+ AbstractExternalData dataSet = getDataSetWithAllMetaData(datasetDescription);
+ dataSets.add(dataSet);
+ shareIdManager.lock(dataSet.getCode());
+ operationLog.info("Archive dataset " + dataSet.getCode() + " in " + containerPath);
+ }
+
+ boolean result = createFolderIfNotExists(stageArchive, stageArchiveContainerFile.getParentFile());
+
+ // TODO: react somehow?
+ if (result)
+ {
+ operationLog.warn("File already exists in archive " + stageArchiveContainerFile.getParentFile());
+ }
+
packageManager.create(stageArchiveContainerFile, dataSets); // packagemanager
} catch (Exception ex)
{
@@ -188,7 +185,7 @@ public class MultiDataSetFileOperationsManager extends AbstractDataSetFileOperat
}
}
- operationLog.info("Data sets '" + "dataSetCode" + "' archived: " + containerPath);
+ operationLog.info("Data sets archived: " + containerPath);
} catch (Exception ex)
{
operationLog.error("Couldn't create package file: " + containerPath, ex);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDatasetArchiver.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDatasetArchiver.java
index 0e1729e5a95..635a25ca2e7 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDatasetArchiver.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDatasetArchiver.java
@@ -177,34 +177,7 @@ public class MultiDatasetArchiver extends AbstractArchiverProcessingPlugin
archivedContent = getFileOperations().getContainerAsHierarchicalContent(containerPath);
- for (DatasetDescription dataset : dataSets)
- {
- String dataSetCode = dataset.getDataSetCode();
- IHierarchicalContent content = null;
- try
- {
- content = context.getHierarchicalContentProvider().asContentWithoutModifyingAccessTimestamp(dataSetCode);
-
- IHierarchicalContentNode root = content.getRootNode();
- IHierarchicalContentNode archiveDataSetRoot = archivedContent.getNode(dataset.getDataSetCode());
-
- status =
- RsyncArchiver.checkHierarchySizeAndChecksums(root, dataSetCode, archiveDataSetRoot,
- RsyncArchiver.ChecksumVerificationCondition.IF_AVAILABLE);
-
- if (status.isError())
- {
- throw new Exception(status.tryGetErrorMessage());
- }
- } finally
- {
- if (content != null)
- {
- content.close();
- }
- }
- statuses.addResult(dataSetCode, status, Operation.ARCHIVE);
- }
+ checkArchivedDataSets(archivedContent, dataSets, context, statuses);
} catch (Exception ex)
{
getFileOperations().deleteContainerFromFinalDestination(containerPath);
@@ -225,6 +198,40 @@ public class MultiDatasetArchiver extends AbstractArchiverProcessingPlugin
}
+ private void checkArchivedDataSets(IHierarchicalContent archivedContent, List<DatasetDescription> dataSets,
+ ArchiverTaskContext context, DatasetProcessingStatuses statuses)
+ {
+ Status status;
+ for (DatasetDescription dataset : dataSets)
+ {
+ String dataSetCode = dataset.getDataSetCode();
+ IHierarchicalContent content = null;
+ try
+ {
+ content = context.getHierarchicalContentProvider().asContentWithoutModifyingAccessTimestamp(dataSetCode);
+
+ IHierarchicalContentNode root = content.getRootNode();
+ IHierarchicalContentNode archiveDataSetRoot = archivedContent.getNode(dataset.getDataSetCode());
+
+ status =
+ RsyncArchiver.checkHierarchySizeAndChecksums(root, dataSetCode, archiveDataSetRoot,
+ RsyncArchiver.ChecksumVerificationCondition.IF_AVAILABLE);
+
+ if (status.isError())
+ {
+ throw new RuntimeException(status.tryGetErrorMessage());
+ }
+ } finally
+ {
+ if (content != null)
+ {
+ content.close();
+ }
+ }
+ statuses.addResult(dataSetCode, status, Operation.ARCHIVE);
+ }
+ }
+
private long getDataSetsSize(List<DatasetDescription> ds)
{
long result = 0;
--
GitLab