SP-155 BIS-38 make checksum verification optional in rsync archiver

SVN: 26273

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java

@@ -56,6 +56,9 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
     @Private
     static final String STAGING_FOLDER = "archive-staging";
 
+    @Private
+    static final String VERIFY_CHECKSUMS_KEY = "verify-checksums";
+
     private static final long serialVersionUID = 1L;
 
     private static final Comparator<IHierarchicalContentNode> NODE_COMPARATOR =
@@ -106,6 +109,8 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
 
     private final DeleteAction deleteAction;
 
+    private final boolean verifyChecksums;
+
     public RsyncArchiver(Properties properties, File storeRoot)
     {
         this(properties, storeRoot, new DataSetFileOperationsManager(properties,
@@ -125,6 +130,9 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
         {
             deleteAction = DeleteAction.DELETE;
         }
+
+        verifyChecksums = PropertyUtils.getBoolean(properties, VERIFY_CHECKSUMS_KEY, true);
+
     }
 
     @Override
@@ -170,7 +178,7 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
                     IHierarchicalContentNode root = content.getRootNode();
                     IHierarchicalContentNode archivedRoot = archivedContent.getRootNode();
 
-                    status = checkHierarchySizeAndChecksums(root, archivedRoot);
+                    status = checkHierarchySizeAndChecksums(root, archivedRoot, verifyChecksums);
                 } finally
                 {
                     FileUtils.deleteQuietly(temp);
@@ -184,7 +192,7 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
 
     @Private
     static Status checkHierarchySizeAndChecksums(IHierarchicalContentNode node,
-            IHierarchicalContentNode retrievedNode)
+            IHierarchicalContentNode retrievedNode, boolean verifyChecksums)
     {
         String relativePath = node.getRelativePath();
         String relativePathOfRetrieved = retrievedNode.getRelativePath();
@@ -218,7 +226,7 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
             {
                 Status status =
                         checkHierarchySizeAndChecksums(childNodes.get(i),
-                                childNodesOfRetrieved.get(i));
+                                childNodesOfRetrieved.get(i), verifyChecksums);
                 if (status.isError())
                 {
                     return status;
@@ -233,14 +241,16 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
                 return Status.createError("The file '" + relativePath + "' has in the store "
                         + fileLength + " bytes but " + fileLengthOfRetrieved + " in the archive.");
             }
-            // TODO: check the configuration if should allow checksum checking for archiver
-            long checksum = node.getChecksumCRC32();
-            long checksumOfRetrieved = retrievedNode.getChecksumCRC32();
-            if (checksum != checksumOfRetrieved)
+            if (verifyChecksums)
             {
-                return Status.createError("The file '" + relativePath
-                        + "' has in the store the checksum " + renderChecksum(checksum) + " but "
-                        + renderChecksum(checksumOfRetrieved) + " in the archive.");
+                long checksum = node.getChecksumCRC32();
+                long checksumOfRetrieved = retrievedNode.getChecksumCRC32();
+                if (checksum != checksumOfRetrieved)
+                {
+                    return Status.createError("The file '" + relativePath
+                            + "' has in the store the checksum " + renderChecksum(checksum)
+                            + " but " + renderChecksum(checksumOfRetrieved) + " in the archive.");
+                }
             }
         }
         return Status.OK;

datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiverTest.java

@@ -199,7 +199,7 @@ public class RsyncArchiverTest extends AbstractFileSystemTestCase
                     {
                         // ignored
                     }
-                    
+
                     allowing(dataSetDirectoryProvider).getStoreRoot();
                     will(returnValue(store));
                 }
@@ -299,15 +299,17 @@ public class RsyncArchiverTest extends AbstractFileSystemTestCase
                     one(fileOperationsManager).copyToDestination(file, ds1);
                     will(returnValue(Status.OK));
 
+                    one(fileOperationsManager).isHosted();
+                    will(returnValue(true));
+
                     one(statusUpdater).update(Arrays.asList("ds1"),
                             DataSetArchivingStatus.AVAILABLE, true);
-                    
+
                     one(contentProvider).asContent("ds1");
                     will(returnValue(new MockContent(":0:0", "f.txt:9:8DA988AF")));
-                    
+
                     FileUtilities.writeToFile(new File(retrievedDataSet, "f.txt"), "abcdefghi");
-                    one(fileOperationsManager).retrieveFromDestination(
-                            retrievedDataSet, ds1);
+                    one(fileOperationsManager).retrieveFromDestination(retrievedDataSet, ds1);
                     will(returnValue(Status.OK));
                 }
             });
@@ -504,14 +506,15 @@ public class RsyncArchiverTest extends AbstractFileSystemTestCase
     public void testCheckHierarchySizeAndChecksumsHappyCase()
     {
         IHierarchicalContentNode root1 =
-                new MockContent(":0:0", "a/:0:0", "a/f1.txt:5:-3", "a/f2.txt:15:13",
-                        "r.txt:7:17").getRootNode();
+                new MockContent(":0:0", "a/:0:0", "a/f1.txt:5:-3", "a/f2.txt:15:13", "r.txt:7:17")
+                        .getRootNode();
         IHierarchicalContentNode root2 =
-                new MockContent(":0:0", "a/:0:0", "a/f2.txt:15:13", "a/f1.txt:5:-3",
-                        "r.txt:7:17").getRootNode();
-        assertEquals("OK", RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2).toString());
+                new MockContent(":0:0", "a/:0:0", "a/f2.txt:15:13", "a/f1.txt:5:-3", "r.txt:7:17")
+                        .getRootNode();
+        assertEquals("OK", RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2, true)
+                .toString());
     }
-    
+
     @Test
     public void testCheckHierarchySizeAndChecksumsWrongPaths()
     {
@@ -521,9 +524,9 @@ public class RsyncArchiverTest extends AbstractFileSystemTestCase
                 new MockContent(":0:0", "a/:0:0", "a/f3.txt:15:13").getRootNode();
         assertEquals("ERROR: \"Different paths: Path in the store is 'a/f1.txt' "
                 + "and in the archive 'a/f3.txt'.\"",
-                RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2).toString());
+                RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2, true).toString());
     }
-    
+
     @Test
     public void testCheckHierarchySizeAndChecksumsFileInsteadOfDirectory()
     {
@@ -532,7 +535,7 @@ public class RsyncArchiverTest extends AbstractFileSystemTestCase
         assertEquals("ERROR: \"The path 'a' should be in store and archive either "
                 + "both directories or files but not mixed: In the store it is a directory "
                 + "but in the archive it is a file.\"", RsyncArchiver
-                .checkHierarchySizeAndChecksums(root1, root2).toString());
+                .checkHierarchySizeAndChecksums(root1, root2, true).toString());
     }
 
     @Test
@@ -543,16 +546,16 @@ public class RsyncArchiverTest extends AbstractFileSystemTestCase
         IHierarchicalContentNode root2 =
                 new MockContent(":0:0", "a/:0:0", "a/f2.txt:15:13").getRootNode();
         assertEquals("ERROR: \"The directory 'a' has in the store 2 files but 1 in the archive.\"",
-                RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2).toString());
+                RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2, true).toString());
     }
-    
+
     @Test
     public void testCheckHierarchySizeAndChecksumsWrongSize()
     {
         IHierarchicalContentNode root1 = new MockContent(":0:0", "r.txt:7:17").getRootNode();
         IHierarchicalContentNode root2 = new MockContent(":0:0", "r.txt:9:17").getRootNode();
         assertEquals("ERROR: \"The file 'r.txt' has in the store 7 bytes but 9 in the archive.\"",
-                RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2).toString());
+                RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2, true).toString());
     }
 
     @Test
@@ -562,20 +565,34 @@ public class RsyncArchiverTest extends AbstractFileSystemTestCase
         IHierarchicalContentNode root2 = new MockContent(":0:0", "r.txt:7:18").getRootNode();
         assertEquals("ERROR: \"The file 'r.txt' has in the store the checksum 00000017 "
                 + "but 00000018 in the archive.\"",
-                RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2).toString());
+                RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2, true).toString());
     }
-    
+
+    public void testCheckHierarchySizeAndChecksumsWrongChecksumAreNotChecked()
+    {
+        IHierarchicalContentNode root1 = new MockContent(":0:0", "r.txt:7:17").getRootNode();
+        IHierarchicalContentNode root2 = new MockContent(":0:0", "r.txt:7:18").getRootNode();
+        assertEquals("OK", RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2, false)
+                .toString());
+    }
+
     private static final class MockNode implements IHierarchicalContentNode
     {
-        private final List<IHierarchicalContentNode> children = new ArrayList<IHierarchicalContentNode>();
-        
+        private final List<IHierarchicalContentNode> children =
+                new ArrayList<IHierarchicalContentNode>();
+
         private String name;
+
         private String relativePath;
+
         private IHierarchicalContentNode parent;
+
         private boolean directory;
+
         private long size;
+
         private int checksum;
-        
+
         void addNode(MockNode node)
         {
             node.parent = this;
@@ -653,7 +670,7 @@ public class RsyncArchiverTest extends AbstractFileSystemTestCase
         {
             return true;
         }
-        
+
         @Override
         public IRandomAccessFile getFileContent() throws UnsupportedOperationException,
                 IOExceptionUnchecked
@@ -669,12 +686,13 @@ public class RsyncArchiverTest extends AbstractFileSystemTestCase
         }
 
     }
-    
+
     private static final class MockContent implements IHierarchicalContent
     {
         private MockNode root;
+
         private final Map<String, MockNode> nodes = new HashMap<String, MockNode>();
-        
+
         MockContent(String... contentDescriptions)
         {
             for (String contentDescription : contentDescriptions)
@@ -745,6 +763,6 @@ public class RsyncArchiverTest extends AbstractFileSystemTestCase
         public void close()
         {
         }
-        
+
     }
 }