BIS-506 / SP-801: Zip file based archiver consistency checks

SVN: 29642

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/DatabaseBasedDataSetPathInfoProvider.java

@@ -52,7 +52,7 @@ public class DatabaseBasedDataSetPathInfoProvider implements IDataSetPathInfoPro
         public String file_name;
 
         public long size_in_bytes;
-        
+
         public Integer checksum_crc32;
 
         public boolean is_directory;
@@ -156,6 +156,23 @@ public class DatabaseBasedDataSetPathInfoProvider implements IDataSetPathInfoPro
             }).getRoot();
     }
 
+    @Override
+    public Map<String, Integer> getDataSetChecksums(String dataSetCode)
+    {
+        Map<String, Integer> files = new HashMap<String, Integer>();
+        IPathInfoDAO queries = getDao();
+        Long id = queries.tryToGetDataSetId(dataSetCode);
+        if (id == null) {
+            return files;
+        }
+        List<DataSetFileRecord> records = getDao().listDataSetFiles(id);
+        for (DataSetFileRecord record : records)
+        {
+            files.put(record.relative_path, record.checksum_crc32);
+        }
+        return files;
+    }
+
     @Override
     public ISingleDataSetPathInfoProvider tryGetSingleDataSetPathInfoProvider(String dataSetCode)
     {

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DistributedPackagingDataSetFileOperationsManager.java

@@ -22,8 +22,10 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Enumeration;
+import java.util.List;
 import java.util.Properties;
 
 import org.apache.commons.io.IOUtils;
@@ -153,7 +155,6 @@ public class DistributedPackagingDataSetFileOperationsManager implements IDataSe
         {
             dataSetPackager = createPackager(file, dataSetExistenceChecker);
             dataSetPackager.addDataSetTo("", dataSet);
-            operationLog.info("Data set '" + dataSetCode + "' archived: " + file);
         } catch (Exception ex)
         {
             status = Status.createError(ex.toString());
@@ -165,9 +166,19 @@ public class DistributedPackagingDataSetFileOperationsManager implements IDataSe
                 try
                 {
                     dataSetPackager.close();
+
+                    List<String> errors =
+                            verify(file, new ZipFileIntegrityVerifier(),
+                                    new PathInfoCrcVerifier(new CrcProvider(datasetDescription.getDataSetCode())));
+
+                    if (errors.size() > 0)
+                    {
+                        throw new RuntimeException(errors.toString());
+                    }
+                    operationLog.info("Data set '" + dataSetCode + "' archived: " + file);
                 } catch (Exception ex)
                 {
-                    status = Status.createError("Couldn't close package file: " + file + ": " + ex);
+                    operationLog.error("Couldn't create package file: " + file, ex);
                 }
             }
             shareIdManager.releaseLock(dataSetCode);
@@ -175,6 +186,16 @@ public class DistributedPackagingDataSetFileOperationsManager implements IDataSe
         return status;
     }
 
+    private List<String> verify(File file, IArchiveFileVerifier... verifiers)
+    {
+        List<String> errors = new ArrayList<String>();
+        for (IArchiveFileVerifier verifier : verifiers)
+        {
+            errors.addAll(verifier.verify(file));
+        }
+        return errors;
+    }
+
     private AbstractDataSetPackager createPackager(File file, DataSetExistenceChecker dataSetExistenceChecker)
     {
         return new ZipDataSetPackager(file, compress, getContentProvider(), dataSetExistenceChecker);

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/Verifier.java → datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/IArchiveFileVerifier.java

@@ -16,12 +16,13 @@
 
 package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard;
 
+import java.io.File;
 import java.util.List;
 
 /**
  * @author anttil
  */
-public interface Verifier
+public interface IArchiveFileVerifier
 {
-    public List<String> verify();
+    public List<String> verify(File file);
 }

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/ZipFilePathInfoVerifier.java → datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/PathInfoCrcVerifier.java

@@ -29,21 +29,18 @@ import de.schlichtherle.util.zip.ZipFile;
 /**
  * @author anttil
  */
-public class ZipFilePathInfoVerifier implements Verifier
+public class PathInfoCrcVerifier implements IArchiveFileVerifier
 {
 
-    private final File file;
-
     private final ICrcProvider crcProvider;
 
-    public ZipFilePathInfoVerifier(File file, ICrcProvider crcProvider)
+    public PathInfoCrcVerifier(ICrcProvider crcProvider)
     {
-        this.file = file;
         this.crcProvider = crcProvider;
     }
 
     @Override
-    public List<String> verify()
+    public List<String> verify(File file)
     {
         List<String> errors = new ArrayList<String>();
         ZipFile zip;

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/ZipFileCrcVerifier.java → datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/ZipFileIntegrityVerifier.java

@@ -34,18 +34,11 @@ import de.schlichtherle.util.zip.ZipFile;
 /**
  * @author anttil
  */
-public class ZipFileCrcVerifier implements Verifier
+public class ZipFileIntegrityVerifier implements IArchiveFileVerifier
 {
 
-    private final File file;
-
-    public ZipFileCrcVerifier(File file)
-    {
-        this.file = file;
-    }
-
     @Override
-    public List<String> verify()
+    public List<String> verify(File file)
     {
         List<String> errors = new ArrayList<String>();
         ZipFile zip;

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IDataSetPathInfoProvider.java

@@ -17,6 +17,7 @@
 package ch.systemsx.cisd.openbis.dss.generic.shared;
 
 import java.util.List;
+import java.util.Map;
 
 import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetPathInfo;
 
@@ -34,4 +35,5 @@ public interface IDataSetPathInfoProvider
 
     public ISingleDataSetPathInfoProvider tryGetSingleDataSetPathInfoProvider(String dataSetCode);
 
+    public Map<String, Integer> getDataSetChecksums(String dataSetCode);
 }