From ef2afb12112bcd2c376d7c068c7992d78d8cca7e Mon Sep 17 00:00:00 2001
From: buczekp <buczekp>
Date: Wed, 23 Mar 2011 14:23:13 +0000
Subject: [PATCH] [LMS-2142] remote implementation of folder comparison
SVN: 20472
---
.../RemoteDataSetFileOperationsExecutor.java | 69 +++++++++++++++++--
.../plugins/standard/DataSetCopier.java | 2 +
.../DataSetFileOperationsManager.java | 35 ++++++----
.../DataSetFileOperationsManagerTest.java | 51 ++++++++++----
4 files changed, 125 insertions(+), 32 deletions(-)
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/RemoteDataSetFileOperationsExecutor.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/RemoteDataSetFileOperationsExecutor.java
index 5c2618cb352..28dd5d211e0 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/RemoteDataSetFileOperationsExecutor.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/RemoteDataSetFileOperationsExecutor.java
@@ -17,19 +17,25 @@
package ch.systemsx.cisd.openbis.dss.generic.server;
import java.io.File;
+import java.io.FileFilter;
+import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
import org.apache.log4j.Logger;
+import ch.systemsx.cisd.bds.StringUtils;
import ch.systemsx.cisd.common.exceptions.ExceptionWithStatus;
import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.filesystem.BooleanStatus;
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.common.filesystem.IPathCopier;
import ch.systemsx.cisd.common.filesystem.ssh.ISshCommandExecutor;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.process.ProcessResult;
import ch.systemsx.cisd.common.utilities.StringUtilities;
+import ch.systemsx.cisd.openbis.dss.generic.server.LocalDataSetFileOperationsExcecutor.FolderFileSizesReportGenerator;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.DataSetCopier;
public final class RemoteDataSetFileOperationsExecutor implements IDataSetFileOperationsExecutor
@@ -48,14 +54,18 @@ public final class RemoteDataSetFileOperationsExecutor implements IDataSetFileOp
private final String rsyncPasswordFileOrNull;
+ private final File gfindExecutable;
+
public RemoteDataSetFileOperationsExecutor(ISshCommandExecutor executor, IPathCopier copier,
- String host, String rsyncModuleNameOrNull, String rsyncPasswordFileOrNull)
+ File gfindExecutable, String host, String rsyncModuleNameOrNull,
+ String rsyncPasswordFileOrNull)
{
this.executor = executor;
this.copier = copier;
this.host = host;
this.rsyncModuleNameOrNull = rsyncModuleNameOrNull;
this.rsyncPasswordFileOrNull = rsyncPasswordFileOrNull;
+ this.gfindExecutable = gfindExecutable;
}
public BooleanStatus exists(File file)
@@ -179,12 +189,57 @@ public final class RemoteDataSetFileOperationsExecutor implements IDataSetFileOp
{
return BooleanStatus.createFalse("Data set location '" + dataSet + "' doesn't exist");
}
- // else if (false == fileOperations.exists(destination))
- // {
- // return BooleanStatus.createFalse("Destination location '" + destination
- // + "' doesn't exist");
- // }
- return executor.exists(destination.getPath(), DataSetCopier.SSH_TIMEOUT_MILLIS);
+ BooleanStatus existsStatus =
+ executor.exists(destination.getPath(), DataSetCopier.SSH_TIMEOUT_MILLIS);
+ if (false == existsStatus.isSuccess())
+ {
+ return existsStatus;
+ }
+ FileFilter nullFilter = null;
+ List<File> storeFiles = FileUtilities.listFiles(dataSet, nullFilter, true);
+ Map<String, Long> dataSetFileSizesByPaths =
+ FolderFileSizesReportGenerator.extractSizesByPaths(storeFiles, dataSet);
+ String cmd = createListFilesWithFileSizeCmd(destination.getPath(), gfindExecutable);
+ ProcessResult result =
+ executor.executeCommandRemotely(cmd, DataSetCopier.SSH_TIMEOUT_MILLIS);
+
+ Map<String, Long> destinationFileSizesByPaths = new LinkedHashMap<String, Long>();
+ if (result.isOK() && result.getOutput() != null)
+ {
+ List<String> output = result.getOutput();
+ for (String line : output)
+ {
+ String split[] = line.split("\t");
+ assert split.length == 2;
+ destinationFileSizesByPaths.put(split[0], Long.parseLong(split[1]));
+ }
+ } else
+ {
+ String errorOutput = StringUtilities.concatenateWithNewLine(result.getErrorOutput());
+ operationLog.error("Listing files in '" + destination + "' failed with exit value: "
+ + result.getExitValue() + "; error output: " + errorOutput);
+ return BooleanStatus.createError("listing files failed");
+ }
+
+ String inconsistenciesReport =
+ FolderFileSizesReportGenerator.findInconsistencies(dataSetFileSizesByPaths,
+ destinationFileSizesByPaths);
+ if (StringUtils.isBlank(inconsistenciesReport))
+ {
+ return BooleanStatus.createTrue();
+ } else
+ {
+ return BooleanStatus.createFalse("Inconsistencies:\n" + inconsistenciesReport);
+ }
+ }
+
+ /**
+ * Returns a bash command listing relative file paths of regular files with their sizes in
+ * bytes.
+ */
+ private static String createListFilesWithFileSizeCmd(final String path, final File findExec)
+ {
+ return findExec + " " + path + " -type f -printf \"%p\\t%s\\n\"";
}
}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetCopier.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetCopier.java
index 10e875b4587..5eeb862016a 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetCopier.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetCopier.java
@@ -50,6 +50,8 @@ public class DataSetCopier extends AbstractDropboxProcessingPlugin
@Private
static final String COPYING_FAILED_MSG = "copying failed";
+ public static final String GFIND_EXEC = "find";
+
public static final String RSYNC_EXEC = "rsync";
public static final String SSH_EXEC = "ssh";
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManager.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManager.java
index f9f92886b68..95c27396ae3 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManager.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManager.java
@@ -70,6 +70,9 @@ public class DataSetFileOperationsManager
@Private
static final String SSH_EXEC = "ssh";
+ @Private
+ static final String GFIND_EXEC = "find";
+
@Private
static final long SSH_TIMEOUT_MILLIS = 15 * 1000; // 15s
@@ -96,6 +99,7 @@ public class DataSetFileOperationsManager
{
File sshExecutable = Copier.getExecutable(properties, SSH_EXEC);
File rsyncExecutable = Copier.getExecutable(properties, RSYNC_EXEC);
+ File gfindExecutable = Copier.getExecutable(properties, GFIND_EXEC);
IPathCopier copier = pathCopierFactory.create(rsyncExecutable, sshExecutable);
copier.check();
String rsyncModule = hostAwareFile.tryGetRsyncModule();
@@ -105,8 +109,9 @@ public class DataSetFileOperationsManager
ISshCommandExecutor sshCommandExecutor =
sshCommandExecutorFactory.create(sshExecutable, hostOrNull);
this.executor =
- new RemoteDataSetFileOperationsExecutor(sshCommandExecutor, copier, hostOrNull,
- rsyncModule, rsyncPasswordFile);
+ new RemoteDataSetFileOperationsExecutor(sshCommandExecutor, copier,
+ gfindExecutable, hostOrNull, rsyncModule, rsyncPasswordFile);
+
}
}
@@ -119,8 +124,10 @@ public class DataSetFileOperationsManager
try
{
File destinationFolder = new File(destination, dataset.getDataSetLocation());
- createFolderIfNotExists(destinationFolder.getParentFile());
- // deleteFolderIfExists(destinationFolder.getParentFile());
+ if (false == createFolderIfNotExists(destinationFolder.getParentFile()))
+ {
+ deleteFolderIfExists(destinationFolder);
+ }
operationLog.info("Copy dataset '" + dataset.getDatasetCode() + "' from '"
+ originalData.getPath() + "' to '" + destinationFolder.getParentFile());
executor.copyDataSetToDestination(originalData, destinationFolder.getParentFile());
@@ -203,22 +210,24 @@ public class DataSetFileOperationsManager
}
}
- // private void deleteFolderIfExists(File destinationFolder)
- // {
- // BooleanStatus destinationExists = destinationExists(destinationFolder);
- // if (destinationExists.isSuccess())
- // {
- // executor.deleteFolder(destinationFolder);
- // }
- // }
+ private void deleteFolderIfExists(File destinationFolder)
+ {
+ BooleanStatus destinationExists = destinationExists(destinationFolder);
+ if (destinationExists.isSuccess())
+ {
+ executor.deleteFolder(destinationFolder);
+ }
+ }
- private void createFolderIfNotExists(File destinationFolder)
+ private boolean createFolderIfNotExists(File destinationFolder)
{
BooleanStatus destinationExists = destinationExists(destinationFolder);
if (destinationExists.isSuccess() == false)
{
executor.createFolder(destinationFolder);
+ return true;
}
+ return false;
}
private BooleanStatus destinationExists(File destinationFolder)
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManagerTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManagerTest.java
index 7fc714ffa71..0f101b7b7e0 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManagerTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManagerTest.java
@@ -128,6 +128,8 @@ public class DataSetFileOperationsManagerTest extends AbstractFileSystemTestCase
private File sshExec;
+ private File gfindExec;
+
@BeforeClass
public void init()
{
@@ -179,6 +181,8 @@ public class DataSetFileOperationsManagerTest extends AbstractFileSystemTestCase
rsyncExec.createNewFile();
sshExec = new File(workingDirectory, "my-rssh");
sshExec.createNewFile();
+ gfindExec = new File(workingDirectory, "my-gfind");
+ gfindExec.createNewFile();
}
private DatasetDescription createDataSetDescription(String dataSetCode, String location,
@@ -506,12 +510,19 @@ public class DataSetFileOperationsManagerTest extends AbstractFileSystemTestCase
will(returnValue(Status.OK));
/*
- * ds2: directory exists in archive -> only copy
+ * ds2: directory exists in archive -> delete and copy
*/
- one(sshExecutor).exists(ds1ArchivedLocationFile.getParentFile().getPath(),
+ one(sshExecutor).exists(ds2ArchivedLocationFile.getParentFile().getPath(),
SSH_TIMEOUT_MILLIS);
will(returnValue(BooleanStatus.createTrue()));
+ one(sshExecutor).exists(ds2ArchivedLocationFile.getPath(), SSH_TIMEOUT_MILLIS);
+ will(returnValue(BooleanStatus.createTrue()));
+
+ one(sshExecutor).executeCommandRemotely(
+ "rm -rf " + ds2ArchivedLocationFile.getPath(), SSH_TIMEOUT_MILLIS);
+ will(returnValue(OK_RESULT));
+
one(copier).copyToRemote(ds2Location, ds2ArchivedLocationFile.getParentFile(),
HOST, null, null);
will(returnValue(Status.OK));
@@ -537,7 +548,13 @@ public class DataSetFileOperationsManagerTest extends AbstractFileSystemTestCase
{
one(sshExecutor).exists(ds1ArchivedLocationFile.getParentFile().getPath(),
SSH_TIMEOUT_MILLIS);
- will(returnValue(BooleanStatus.createTrue()));
+ will(returnValue(BooleanStatus.createFalse()));
+
+ one(sshExecutor).executeCommandRemotely(
+ "mkdir -p " + ds1ArchivedLocationFile.getParentFile().getPath(),
+ SSH_TIMEOUT_MILLIS);
+ will(returnValue(OK_RESULT));
+
one(copier).copyToRemote(ds1Location, ds1ArchivedLocationFile.getParentFile(),
HOST, null, null);
will(returnValue(Status.createError(DUMMY_ERROR_MESSAGE)));
@@ -649,22 +666,28 @@ public class DataSetFileOperationsManagerTest extends AbstractFileSystemTestCase
{
{
/*
- * ds1: TODO directory is present but content is different
+ * ds1: directory is present and content is OK
*/
one(sshExecutor).exists(ds1ArchivedLocationFile.getPath(), SSH_TIMEOUT_MILLIS);
will(returnValue(BooleanStatus.createTrue()));
- /*
- * ds2: directory is present and content is OK
- */
- one(sshExecutor).exists(ds2ArchivedLocationFile.getPath(), SSH_TIMEOUT_MILLIS);
- will(returnValue(BooleanStatus.createTrue()));
+ one(sshExecutor).executeCommandRemotely(
+ gfindExec.getPath() + " " + ds1ArchivedLocationFile.getPath()
+ + " -type f -printf \"%p\\t%s\\n\"", SSH_TIMEOUT_MILLIS);
+ will(returnValue(ERROR_RESULT));
+
+ // /*
+ // * ds2: directory is present and content is OK
+ // */
+ // one(sshExecutor).exists(ds2ArchivedLocationFile.getPath(),
+ // SSH_TIMEOUT_MILLIS);
+ // will(returnValue(BooleanStatus.createTrue()));
}
});
BooleanStatus status1 = dataSetCopier.isPresentInDestination(ds1Location, ds1);
- BooleanStatus status2 = dataSetCopier.isPresentInDestination(ds2Location, ds2);
- assertTrue(status1);
- assertTrue(status2);
+ // BooleanStatus status2 = dataSetCopier.isPresentInDestination(ds2Location, ds2);
+ assertError(status1, "listing files failed");
+ // assertTrue(status2);
context.assertIsSatisfied();
}
@@ -840,6 +863,8 @@ public class DataSetFileOperationsManagerTest extends AbstractFileSystemTestCase
rsyncExec.getPath());
properties.setProperty(DataSetFileOperationsManager.SSH_EXEC + "-executable",
sshExec.getPath());
+ properties.setProperty(DataSetFileOperationsManager.GFIND_EXEC + "-executable",
+ gfindExec.getPath());
return properties;
}
@@ -855,6 +880,8 @@ public class DataSetFileOperationsManagerTest extends AbstractFileSystemTestCase
rsyncExec.getPath());
properties.setProperty(DataSetFileOperationsManager.SSH_EXEC + "-executable",
sshExec.getPath());
+ properties.setProperty(DataSetFileOperationsManager.GFIND_EXEC + "-executable",
+ gfindExec.getPath());
return properties;
}
--
GitLab