From a98f91f51df7a527063ca3472b191c363dba5167 Mon Sep 17 00:00:00 2001
From: jakubs <jakubs>
Date: Mon, 30 Jul 2012 08:34:09 +0000
Subject: [PATCH] SP-155 BIS-38 make checksum verification optional in rsync
archiver
SVN: 26273
---
.../plugins/standard/RsyncArchiver.java | 30 +++++---
.../plugins/standard/RsyncArchiverTest.java | 70 ++++++++++++-------
2 files changed, 64 insertions(+), 36 deletions(-)
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java
index a6fd7b71349..696665022dd 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java
@@ -56,6 +56,9 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
@Private
static final String STAGING_FOLDER = "archive-staging";
+ @Private
+ static final String VERIFY_CHECKSUMS_KEY = "verify-checksums";
+
private static final long serialVersionUID = 1L;
private static final Comparator<IHierarchicalContentNode> NODE_COMPARATOR =
@@ -106,6 +109,8 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
private final DeleteAction deleteAction;
+ private final boolean verifyChecksums;
+
public RsyncArchiver(Properties properties, File storeRoot)
{
this(properties, storeRoot, new DataSetFileOperationsManager(properties,
@@ -125,6 +130,9 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
{
deleteAction = DeleteAction.DELETE;
}
+
+ verifyChecksums = PropertyUtils.getBoolean(properties, VERIFY_CHECKSUMS_KEY, true);
+
}
@Override
@@ -170,7 +178,7 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
IHierarchicalContentNode root = content.getRootNode();
IHierarchicalContentNode archivedRoot = archivedContent.getRootNode();
- status = checkHierarchySizeAndChecksums(root, archivedRoot);
+ status = checkHierarchySizeAndChecksums(root, archivedRoot, verifyChecksums);
} finally
{
FileUtils.deleteQuietly(temp);
@@ -184,7 +192,7 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
@Private
static Status checkHierarchySizeAndChecksums(IHierarchicalContentNode node,
- IHierarchicalContentNode retrievedNode)
+ IHierarchicalContentNode retrievedNode, boolean verifyChecksums)
{
String relativePath = node.getRelativePath();
String relativePathOfRetrieved = retrievedNode.getRelativePath();
@@ -218,7 +226,7 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
{
Status status =
checkHierarchySizeAndChecksums(childNodes.get(i),
- childNodesOfRetrieved.get(i));
+ childNodesOfRetrieved.get(i), verifyChecksums);
if (status.isError())
{
return status;
@@ -233,14 +241,16 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
return Status.createError("The file '" + relativePath + "' has in the store "
+ fileLength + " bytes but " + fileLengthOfRetrieved + " in the archive.");
}
- // TODO: check the configuration if should allow checksum checking for archiver
- long checksum = node.getChecksumCRC32();
- long checksumOfRetrieved = retrievedNode.getChecksumCRC32();
- if (checksum != checksumOfRetrieved)
+ if (verifyChecksums)
{
- return Status.createError("The file '" + relativePath
- + "' has in the store the checksum " + renderChecksum(checksum) + " but "
- + renderChecksum(checksumOfRetrieved) + " in the archive.");
+ long checksum = node.getChecksumCRC32();
+ long checksumOfRetrieved = retrievedNode.getChecksumCRC32();
+ if (checksum != checksumOfRetrieved)
+ {
+ return Status.createError("The file '" + relativePath
+ + "' has in the store the checksum " + renderChecksum(checksum)
+ + " but " + renderChecksum(checksumOfRetrieved) + " in the archive.");
+ }
}
}
return Status.OK;
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiverTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiverTest.java
index 7a49b8eaf9c..ffa09311d87 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiverTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiverTest.java
@@ -199,7 +199,7 @@ public class RsyncArchiverTest extends AbstractFileSystemTestCase
{
// ignored
}
-
+
allowing(dataSetDirectoryProvider).getStoreRoot();
will(returnValue(store));
}
@@ -299,15 +299,17 @@ public class RsyncArchiverTest extends AbstractFileSystemTestCase
one(fileOperationsManager).copyToDestination(file, ds1);
will(returnValue(Status.OK));
+ one(fileOperationsManager).isHosted();
+ will(returnValue(true));
+
one(statusUpdater).update(Arrays.asList("ds1"),
DataSetArchivingStatus.AVAILABLE, true);
-
+
one(contentProvider).asContent("ds1");
will(returnValue(new MockContent(":0:0", "f.txt:9:8DA988AF")));
-
+
FileUtilities.writeToFile(new File(retrievedDataSet, "f.txt"), "abcdefghi");
- one(fileOperationsManager).retrieveFromDestination(
- retrievedDataSet, ds1);
+ one(fileOperationsManager).retrieveFromDestination(retrievedDataSet, ds1);
will(returnValue(Status.OK));
}
});
@@ -504,14 +506,15 @@ public class RsyncArchiverTest extends AbstractFileSystemTestCase
public void testCheckHierarchySizeAndChecksumsHappyCase()
{
IHierarchicalContentNode root1 =
- new MockContent(":0:0", "a/:0:0", "a/f1.txt:5:-3", "a/f2.txt:15:13",
- "r.txt:7:17").getRootNode();
+ new MockContent(":0:0", "a/:0:0", "a/f1.txt:5:-3", "a/f2.txt:15:13", "r.txt:7:17")
+ .getRootNode();
IHierarchicalContentNode root2 =
- new MockContent(":0:0", "a/:0:0", "a/f2.txt:15:13", "a/f1.txt:5:-3",
- "r.txt:7:17").getRootNode();
- assertEquals("OK", RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2).toString());
+ new MockContent(":0:0", "a/:0:0", "a/f2.txt:15:13", "a/f1.txt:5:-3", "r.txt:7:17")
+ .getRootNode();
+ assertEquals("OK", RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2, true)
+ .toString());
}
-
+
@Test
public void testCheckHierarchySizeAndChecksumsWrongPaths()
{
@@ -521,9 +524,9 @@ public class RsyncArchiverTest extends AbstractFileSystemTestCase
new MockContent(":0:0", "a/:0:0", "a/f3.txt:15:13").getRootNode();
assertEquals("ERROR: \"Different paths: Path in the store is 'a/f1.txt' "
+ "and in the archive 'a/f3.txt'.\"",
- RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2).toString());
+ RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2, true).toString());
}
-
+
@Test
public void testCheckHierarchySizeAndChecksumsFileInsteadOfDirectory()
{
@@ -532,7 +535,7 @@ public class RsyncArchiverTest extends AbstractFileSystemTestCase
assertEquals("ERROR: \"The path 'a' should be in store and archive either "
+ "both directories or files but not mixed: In the store it is a directory "
+ "but in the archive it is a file.\"", RsyncArchiver
- .checkHierarchySizeAndChecksums(root1, root2).toString());
+ .checkHierarchySizeAndChecksums(root1, root2, true).toString());
}
@Test
@@ -543,16 +546,16 @@ public class RsyncArchiverTest extends AbstractFileSystemTestCase
IHierarchicalContentNode root2 =
new MockContent(":0:0", "a/:0:0", "a/f2.txt:15:13").getRootNode();
assertEquals("ERROR: \"The directory 'a' has in the store 2 files but 1 in the archive.\"",
- RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2).toString());
+ RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2, true).toString());
}
-
+
@Test
public void testCheckHierarchySizeAndChecksumsWrongSize()
{
IHierarchicalContentNode root1 = new MockContent(":0:0", "r.txt:7:17").getRootNode();
IHierarchicalContentNode root2 = new MockContent(":0:0", "r.txt:9:17").getRootNode();
assertEquals("ERROR: \"The file 'r.txt' has in the store 7 bytes but 9 in the archive.\"",
- RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2).toString());
+ RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2, true).toString());
}
@Test
@@ -562,20 +565,34 @@ public class RsyncArchiverTest extends AbstractFileSystemTestCase
IHierarchicalContentNode root2 = new MockContent(":0:0", "r.txt:7:18").getRootNode();
assertEquals("ERROR: \"The file 'r.txt' has in the store the checksum 00000017 "
+ "but 00000018 in the archive.\"",
- RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2).toString());
+ RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2, true).toString());
}
-
+
+ public void testCheckHierarchySizeAndChecksumsWrongChecksumAreNotChecked()
+ {
+ IHierarchicalContentNode root1 = new MockContent(":0:0", "r.txt:7:17").getRootNode();
+ IHierarchicalContentNode root2 = new MockContent(":0:0", "r.txt:7:18").getRootNode();
+ assertEquals("OK", RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2, false)
+ .toString());
+ }
+
private static final class MockNode implements IHierarchicalContentNode
{
- private final List<IHierarchicalContentNode> children = new ArrayList<IHierarchicalContentNode>();
-
+ private final List<IHierarchicalContentNode> children =
+ new ArrayList<IHierarchicalContentNode>();
+
private String name;
+
private String relativePath;
+
private IHierarchicalContentNode parent;
+
private boolean directory;
+
private long size;
+
private int checksum;
-
+
void addNode(MockNode node)
{
node.parent = this;
@@ -653,7 +670,7 @@ public class RsyncArchiverTest extends AbstractFileSystemTestCase
{
return true;
}
-
+
@Override
public IRandomAccessFile getFileContent() throws UnsupportedOperationException,
IOExceptionUnchecked
@@ -669,12 +686,13 @@ public class RsyncArchiverTest extends AbstractFileSystemTestCase
}
}
-
+
private static final class MockContent implements IHierarchicalContent
{
private MockNode root;
+
private final Map<String, MockNode> nodes = new HashMap<String, MockNode>();
-
+
MockContent(String... contentDescriptions)
{
for (String contentDescription : contentDescriptions)
@@ -745,6 +763,6 @@ public class RsyncArchiverTest extends AbstractFileSystemTestCase
public void close()
{
}
-
+
}
}
--
GitLab