From c0b2d1876599d31d2b2e2c1733eaae2c1ddbe51e Mon Sep 17 00:00:00 2001
From: jakubs <jakubs>
Date: Mon, 27 Oct 2014 22:23:24 +0000
Subject: [PATCH] SSDM-1081 refactoring in DataSetFileOperations Extract part
of the logic to the abstract parent class Group part of variables together in
a container class
SVN: 32669
---
.../DataSetFileOperationsManager.java | 134 +++-----------
.../AbstractDataSetFileOperationsManager.java | 173 ++++++++++++++++++
2 files changed, 200 insertions(+), 107 deletions(-)
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/AbstractDataSetFileOperationsManager.java
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManager.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManager.java
index 79ef2e643f4..b7dc25926fe 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManager.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManager.java
@@ -22,24 +22,15 @@ import java.util.Properties;
import org.apache.commons.lang.time.DateUtils;
import org.apache.log4j.Logger;
-import ch.rinn.restrictions.Private;
import ch.systemsx.cisd.common.exceptions.ExceptionWithStatus;
import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.filesystem.BooleanStatus;
-import ch.systemsx.cisd.common.filesystem.FileOperations;
-import ch.systemsx.cisd.common.filesystem.FileUtilities;
-import ch.systemsx.cisd.common.filesystem.HostAwareFile;
-import ch.systemsx.cisd.common.filesystem.IPathCopier;
-import ch.systemsx.cisd.common.filesystem.highwatermark.HostAwareFileWithHighwaterMark;
-import ch.systemsx.cisd.common.filesystem.ssh.ISshCommandExecutor;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.properties.PropertyUtils;
import ch.systemsx.cisd.openbis.common.io.hierarchical_content.DefaultFileBasedHierarchicalContentFactory;
import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContent;
-import ch.systemsx.cisd.openbis.dss.generic.server.IDataSetFileOperationsExecutor;
-import ch.systemsx.cisd.openbis.dss.generic.server.LocalDataSetFileOperationsExcecutor;
-import ch.systemsx.cisd.openbis.dss.generic.server.RemoteDataSetFileOperationsExecutor;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.archiver.AbstractDataSetFileOperationsManager;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IDatasetLocation;
import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
@@ -48,7 +39,7 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
*
* @author Piotr Buczek
*/
-public class DataSetFileOperationsManager implements IDataSetFileOperationsManager
+public class DataSetFileOperationsManager extends AbstractDataSetFileOperationsManager implements IDataSetFileOperationsManager
{
private static interface IDeleteAction
{
@@ -60,92 +51,20 @@ public class DataSetFileOperationsManager implements IDataSetFileOperationsManag
private final static Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
DataSetFileOperationsManager.class);
- @Private
- static final String DESTINATION_KEY = "destination";
-
- @Private
- static final String TIMEOUT_KEY = "timeout";
-
- @Private
- static final String RSYNC_PASSWORD_FILE_KEY = "rsync-password-file";
-
- @Private
- static final String CHECK_EXISTENCE_FAILED = "couldn't check existence";
-
- @Private
- static final String DESTINATION_DOES_NOT_EXIST = "destination doesn't exist";
-
- @Private
- static final String RSYNC_EXEC = "rsync";
-
- @Private
- static final String SSH_EXEC = "ssh";
-
- @Private
- static final String GFIND_EXEC = "find";
-
- @Private
- static final long DEFAULT_TIMEOUT_SECONDS = 15;
-
- static final String FOLDER_OF_AS_DELETED_MARKED_DATA_SETS = "DELETED";
-
- private final IDataSetFileOperationsExecutor executor;
-
- private final String destination;
-
- private final long timeoutInMillis;
-
- private final boolean isHosted;
+ private final ArchiveDestination archiveDestinationManager;
public DataSetFileOperationsManager(Properties properties,
IPathCopierFactory pathCopierFactory,
ISshCommandExecutorFactory sshCommandExecutorFactory)
{
- String hostFile = PropertyUtils.getMandatoryProperty(properties, DESTINATION_KEY);
- HostAwareFile hostAwareFile = HostAwareFileWithHighwaterMark.create(hostFile, -1);
- String hostOrNull = hostAwareFile.tryGetHost();
-
- this.isHosted = hostOrNull != null;
-
- this.destination = hostAwareFile.getPath();
long timeoutInSeconds =
PropertyUtils.getLong(properties, TIMEOUT_KEY, DEFAULT_TIMEOUT_SECONDS);
- this.timeoutInMillis = timeoutInSeconds * DateUtils.MILLIS_PER_SECOND;
+ long timeoutInMillis = timeoutInSeconds * DateUtils.MILLIS_PER_SECOND;
- if (hostOrNull == null)
- {
- File sshExecutable = null; // don't use ssh locally
- File rsyncExecutable = Copier.getExecutable(properties, RSYNC_EXEC);
- IPathCopier copier =
- pathCopierFactory.create(rsyncExecutable, sshExecutable, timeoutInMillis);
- copier.check();
- String rsyncModule = hostAwareFile.tryGetRsyncModule();
- String rsyncPasswordFile = properties.getProperty(RSYNC_PASSWORD_FILE_KEY);
- this.executor =
- new LocalDataSetFileOperationsExcecutor(
- FileOperations.getMonitoredInstanceForCurrentThread(), copier,
- rsyncModule, rsyncPasswordFile);
- } else
- {
- File sshExecutable = Copier.getExecutable(properties, SSH_EXEC);
- File rsyncExecutable = Copier.getExecutable(properties, RSYNC_EXEC);
- File gfindExecutable = Copier.getExecutable(properties, GFIND_EXEC);
-
- IPathCopier copier =
- pathCopierFactory.create(rsyncExecutable, sshExecutable, timeoutInMillis);
- copier.check();
- String rsyncModule = hostAwareFile.tryGetRsyncModule();
- String rsyncPasswordFile = properties.getProperty(RSYNC_PASSWORD_FILE_KEY);
- FileUtilities.checkPathCopier(copier, hostOrNull, null, rsyncModule, rsyncPasswordFile,
- timeoutInMillis);
- ISshCommandExecutor sshCommandExecutor =
- sshCommandExecutorFactory.create(sshExecutable, hostOrNull);
- this.executor =
- new RemoteDataSetFileOperationsExecutor(sshCommandExecutor, copier,
- gfindExecutable, hostOrNull, rsyncModule, rsyncPasswordFile,
- timeoutInMillis);
+ String hostFile = PropertyUtils.getMandatoryProperty(properties, DESTINATION_KEY);
- }
+ this.archiveDestinationManager =
+ createArchiveDestinationManager(properties, pathCopierFactory, sshCommandExecutorFactory, hostFile, timeoutInMillis);
}
/**
@@ -157,19 +76,20 @@ public class DataSetFileOperationsManager implements IDataSetFileOperationsManag
{
try
{
- File destinationFolder = new File(destination, dataset.getDataSetLocation());
+ File destinationFolder = new File(archiveDestinationManager.getDestination(), dataset.getDataSetLocation());
if (createFolderIfNotExists(destinationFolder.getParentFile())
|| destinationExists(destinationFolder).isSuccess() == false)
{
operationLog.info("Copy dataset '" + dataset.getDataSetCode() + "' from '"
+ originalData.getPath() + "' to '" + destinationFolder.getParentFile());
- executor.copyDataSetToDestination(originalData, destinationFolder.getParentFile());
+ archiveDestinationManager.getExecutor().copyDataSetToDestination(originalData, destinationFolder.getParentFile());
} else
{
operationLog.info("Update dataset '" + dataset.getDataSetCode() + "' from '"
+ originalData.getPath() + "' to '" + destinationFolder.getParentFile());
- executor.syncDataSetWithDestination(originalData, destinationFolder.getParentFile());
+ archiveDestinationManager.getExecutor().syncDataSetWithDestination(originalData, destinationFolder.getParentFile());
}
+
return Status.OK;
} catch (ExceptionWithStatus ex)
{
@@ -186,13 +106,13 @@ public class DataSetFileOperationsManager implements IDataSetFileOperationsManag
{
try
{
- File destinationFolder = new File(destination, dataset.getDataSetLocation());
+ File destinationFolder = new File(archiveDestinationManager.getDestination(), dataset.getDataSetLocation());
checkDestinationExists(destinationFolder);
File folder = originalData.getParentFile();
operationLog.info("Retrieve data set '" + dataset.getDataSetCode() + "' from '"
+ destinationFolder.getPath() + "' to '" + folder);
folder.mkdirs();
- executor.retrieveDataSetFromDestination(folder, destinationFolder);
+ archiveDestinationManager.getExecutor().retrieveDataSetFromDestination(folder, destinationFolder);
return Status.OK;
} catch (ExceptionWithStatus ex)
{
@@ -218,7 +138,7 @@ public class DataSetFileOperationsManager implements IDataSetFileOperationsManag
@Override
public void delete(File dataSetFolder, String dataSetCode)
{
- executor.deleteFolder(dataSetFolder);
+ archiveDestinationManager.getExecutor().deleteFolder(dataSetFolder);
}
});
}
@@ -238,19 +158,19 @@ public class DataSetFileOperationsManager implements IDataSetFileOperationsManag
public void delete(File dataSetFolder, String dataSetCode)
{
File deletedFolder =
- new File(destination, FOLDER_OF_AS_DELETED_MARKED_DATA_SETS);
- executor.createFolder(deletedFolder);
+ new File(archiveDestinationManager.getDestination(), FOLDER_OF_AS_DELETED_MARKED_DATA_SETS);
+ archiveDestinationManager.getExecutor().createFolder(deletedFolder);
File markerFile = new File(deletedFolder, dataSetCode);
- executor.createMarkerFile(markerFile);
+ archiveDestinationManager.getExecutor().createMarkerFile(markerFile);
}
});
}
-
+
private Status delete(IDatasetLocation dataset, IDeleteAction action)
{
try
{
- File destinationFolder = new File(destination, dataset.getDataSetLocation());
+ File destinationFolder = new File(archiveDestinationManager.getDestination(), dataset.getDataSetLocation());
BooleanStatus destinationExists = destinationExists(destinationFolder);
if (destinationExists.isSuccess())
{
@@ -277,8 +197,8 @@ public class DataSetFileOperationsManager implements IDataSetFileOperationsManag
{
try
{
- File destinationFolder = new File(destination, dataset.getDataSetLocation());
- BooleanStatus resultStatus = executor.checkSame(originalData, destinationFolder);
+ File destinationFolder = new File(archiveDestinationManager.getDestination(), dataset.getDataSetLocation());
+ BooleanStatus resultStatus = archiveDestinationManager.getExecutor().checkSame(originalData, destinationFolder);
String message = resultStatus.tryGetMessage();
if (message != null) // if there is a message something went wrong
{
@@ -300,8 +220,8 @@ public class DataSetFileOperationsManager implements IDataSetFileOperationsManag
{
try
{
- File destinationFolder = new File(destination, dataset.getDataSetLocation());
- BooleanStatus resultStatus = executor.exists(destinationFolder);
+ File destinationFolder = new File(archiveDestinationManager.getDestination(), dataset.getDataSetLocation());
+ BooleanStatus resultStatus = archiveDestinationManager.getExecutor().exists(destinationFolder);
String message = resultStatus.tryGetMessage();
if (message != null) // if there is a message something went wrong
{
@@ -329,7 +249,7 @@ public class DataSetFileOperationsManager implements IDataSetFileOperationsManag
BooleanStatus destinationExists = destinationExists(destinationFolder);
if (destinationExists.isSuccess() == false)
{
- executor.createFolder(destinationFolder);
+ archiveDestinationManager.getExecutor().createFolder(destinationFolder);
return true;
}
return false;
@@ -337,7 +257,7 @@ public class DataSetFileOperationsManager implements IDataSetFileOperationsManag
private BooleanStatus destinationExists(File destinationFolder)
{
- BooleanStatus destinationExists = executor.exists(destinationFolder);
+ BooleanStatus destinationExists = archiveDestinationManager.getExecutor().exists(destinationFolder);
if (destinationExists.isError())
{
operationLog.error("Could not check existence of '" + destinationFolder + "': "
@@ -350,14 +270,14 @@ public class DataSetFileOperationsManager implements IDataSetFileOperationsManag
@Override
public boolean isHosted()
{
- return isHosted;
+ return archiveDestinationManager.isHosted();
}
@Override
public IHierarchicalContent getAsHierarchicalContent(DatasetDescription dataset)
{
return new DefaultFileBasedHierarchicalContentFactory()
- .asHierarchicalContent(new File(destination, dataset.getDataSetLocation()), null);
+ .asHierarchicalContent(new File(archiveDestinationManager.getDestination(), dataset.getDataSetLocation()), null);
}
}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/AbstractDataSetFileOperationsManager.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/AbstractDataSetFileOperationsManager.java
new file mode 100644
index 00000000000..d83ebf646f1
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/AbstractDataSetFileOperationsManager.java
@@ -0,0 +1,173 @@
+/*
+ * Copyright 2014 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.archiver;
+
+import java.io.File;
+import java.util.Properties;
+
+import ch.rinn.restrictions.Private;
+import ch.systemsx.cisd.common.filesystem.FileOperations;
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
+import ch.systemsx.cisd.common.filesystem.HostAwareFile;
+import ch.systemsx.cisd.common.filesystem.IPathCopier;
+import ch.systemsx.cisd.common.filesystem.highwatermark.HostAwareFileWithHighwaterMark;
+import ch.systemsx.cisd.common.filesystem.ssh.ISshCommandExecutor;
+import ch.systemsx.cisd.openbis.dss.generic.server.IDataSetFileOperationsExecutor;
+import ch.systemsx.cisd.openbis.dss.generic.server.LocalDataSetFileOperationsExcecutor;
+import ch.systemsx.cisd.openbis.dss.generic.server.RemoteDataSetFileOperationsExecutor;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.Copier;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.IPathCopierFactory;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.ISshCommandExecutorFactory;
+
+/**
+ * @author Jakub Straszewski
+ */
+public abstract class AbstractDataSetFileOperationsManager
+{
+
+ @Private
+ protected static final String DESTINATION_KEY = "destination";
+
+ @Private
+ protected static final String TIMEOUT_KEY = "timeout";
+
+ @Private
+ protected static final String RSYNC_PASSWORD_FILE_KEY = "rsync-password-file";
+
+ @Private
+ protected static final String CHECK_EXISTENCE_FAILED = "couldn't check existence";
+
+ @Private
+ protected static final String DESTINATION_DOES_NOT_EXIST = "destination doesn't exist";
+
+ @Private
+ protected static final String RSYNC_EXEC = "rsync";
+
+ @Private
+ protected static final String SSH_EXEC = "ssh";
+
+ @Private
+ protected static final String GFIND_EXEC = "find";
+
+ @Private
+ protected static final long DEFAULT_TIMEOUT_SECONDS = 15;
+
+ @Private
+ protected static final String FOLDER_OF_AS_DELETED_MARKED_DATA_SETS = "DELETED";
+
+ public static class ArchiveDestination
+ {
+ private final String destination;
+
+ private final IDataSetFileOperationsExecutor executor;
+
+ private final boolean isHosted;
+
+ private final long timeoutInMillis;
+
+ private ArchiveDestination(String destination, IDataSetFileOperationsExecutor executor, boolean isHosted, long timeoutInMillis)
+ {
+ this.destination = destination;
+ this.executor = executor;
+ this.isHosted = isHosted;
+ this.timeoutInMillis = timeoutInMillis;
+ }
+
+ public String getDestination()
+ {
+ return destination;
+ }
+
+ public IDataSetFileOperationsExecutor getExecutor()
+ {
+ return executor;
+ }
+
+ public boolean isHosted()
+ {
+ return isHosted;
+ }
+
+ public long getTimeoutInMillis()
+ {
+ return timeoutInMillis;
+ }
+
+ }
+
+ protected static ArchiveDestination createArchiveDestinationManager(Properties properties,
+ IPathCopierFactory pathCopierFactory,
+ ISshCommandExecutorFactory sshCommandExecutorFactory, String archiveDestinationHost, long timeoutInMillis)
+ {
+ HostAwareFile hostAwareFile = HostAwareFileWithHighwaterMark.create(archiveDestinationHost, -1);
+ boolean isHosted = hostAwareFile.tryGetHost() != null;
+ String destination = hostAwareFile.getPath();
+ IDataSetFileOperationsExecutor executor;
+
+ if (false == isHosted)
+ {
+ executor = createLocalDataSetFileOperationsExecutor(properties, pathCopierFactory, hostAwareFile, timeoutInMillis);
+ } else
+ {
+ executor =
+ createRemoteDataSetFileOperationsExecutor(properties, pathCopierFactory, sshCommandExecutorFactory, hostAwareFile,
+ timeoutInMillis);
+ }
+ return new ArchiveDestination(destination, executor, isHosted, timeoutInMillis);
+ }
+
+ protected static RemoteDataSetFileOperationsExecutor createRemoteDataSetFileOperationsExecutor(Properties properties,
+ IPathCopierFactory pathCopierFactory, ISshCommandExecutorFactory sshCommandExecutorFactory, HostAwareFile hostAwareFile,
+ long timeoutInMillis)
+ {
+ String hostOrNull = hostAwareFile.tryGetHost();
+ File sshExecutable = Copier.getExecutable(properties, SSH_EXEC);
+ File rsyncExecutable = Copier.getExecutable(properties, RSYNC_EXEC);
+ File gfindExecutable = Copier.getExecutable(properties, GFIND_EXEC);
+
+ IPathCopier copier =
+ pathCopierFactory.create(rsyncExecutable, sshExecutable, timeoutInMillis);
+ copier.check();
+ String rsyncModule = hostAwareFile.tryGetRsyncModule();
+ String rsyncPasswordFile = properties.getProperty(RSYNC_PASSWORD_FILE_KEY);
+ FileUtilities.checkPathCopier(copier, hostOrNull, null, rsyncModule, rsyncPasswordFile,
+ timeoutInMillis);
+ ISshCommandExecutor sshCommandExecutor = sshCommandExecutorFactory.create(sshExecutable, hostOrNull);
+ RemoteDataSetFileOperationsExecutor result = new RemoteDataSetFileOperationsExecutor(sshCommandExecutor, copier,
+ gfindExecutable, hostOrNull, rsyncModule, rsyncPasswordFile,
+ timeoutInMillis);
+ return result;
+ }
+
+ protected static LocalDataSetFileOperationsExcecutor createLocalDataSetFileOperationsExecutor(Properties properties,
+ IPathCopierFactory pathCopierFactory,
+ HostAwareFile hostAwareFile, long timeoutInMillis)
+ {
+ File sshExecutable = null; // don't use ssh locally
+ File rsyncExecutable = Copier.getExecutable(properties, RSYNC_EXEC);
+ IPathCopier copier =
+ pathCopierFactory.create(rsyncExecutable, sshExecutable, timeoutInMillis);
+ copier.check();
+ String rsyncModule = hostAwareFile.tryGetRsyncModule();
+ String rsyncPasswordFile = properties.getProperty(RSYNC_PASSWORD_FILE_KEY);
+ LocalDataSetFileOperationsExcecutor result = new LocalDataSetFileOperationsExcecutor(
+ FileOperations.getMonitoredInstanceForCurrentThread(), copier,
+ rsyncModule, rsyncPasswordFile);
+ return result;
+ }
+
+}
--
GitLab