From a990ffc5c7f76469c657cb6e244accae8acaf73d Mon Sep 17 00:00:00 2001
From: anttil <anttil>
Date: Tue, 13 Aug 2013 10:04:23 +0000
Subject: [PATCH] BIS-506 / SP-801: Zip file based archiver consistency checks
SVN: 29642
---
.../DatabaseBasedDataSetPathInfoProvider.java | 19 +++++++++++++-
...PackagingDataSetFileOperationsManager.java | 25 +++++++++++++++++--
...erifier.java => IArchiveFileVerifier.java} | 5 ++--
...Verifier.java => PathInfoCrcVerifier.java} | 9 +++----
...ier.java => ZipFileIntegrityVerifier.java} | 11 ++------
.../shared/IDataSetPathInfoProvider.java | 2 ++
6 files changed, 51 insertions(+), 20 deletions(-)
rename datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/{Verifier.java => IArchiveFileVerifier.java} (87%)
rename datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/{ZipFilePathInfoVerifier.java => PathInfoCrcVerifier.java} (90%)
rename datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/{ZipFileCrcVerifier.java => ZipFileIntegrityVerifier.java} (94%)
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/DatabaseBasedDataSetPathInfoProvider.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/DatabaseBasedDataSetPathInfoProvider.java
index af4dbf9381c..31a549ef986 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/DatabaseBasedDataSetPathInfoProvider.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/DatabaseBasedDataSetPathInfoProvider.java
@@ -52,7 +52,7 @@ public class DatabaseBasedDataSetPathInfoProvider implements IDataSetPathInfoPro
public String file_name;
public long size_in_bytes;
-
+
public Integer checksum_crc32;
public boolean is_directory;
@@ -156,6 +156,23 @@ public class DatabaseBasedDataSetPathInfoProvider implements IDataSetPathInfoPro
}).getRoot();
}
+ @Override
+ public Map<String, Integer> getDataSetChecksums(String dataSetCode)
+ {
+ Map<String, Integer> files = new HashMap<String, Integer>();
+ IPathInfoDAO queries = getDao();
+ Long id = queries.tryToGetDataSetId(dataSetCode);
+ if (id == null) {
+ return files;
+ }
+ List<DataSetFileRecord> records = getDao().listDataSetFiles(id);
+ for (DataSetFileRecord record : records)
+ {
+ files.put(record.relative_path, record.checksum_crc32);
+ }
+ return files;
+ }
+
@Override
public ISingleDataSetPathInfoProvider tryGetSingleDataSetPathInfoProvider(String dataSetCode)
{
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DistributedPackagingDataSetFileOperationsManager.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DistributedPackagingDataSetFileOperationsManager.java
index 1e4ab237ebb..8c43ab84992 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DistributedPackagingDataSetFileOperationsManager.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DistributedPackagingDataSetFileOperationsManager.java
@@ -22,8 +22,10 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Enumeration;
+import java.util.List;
import java.util.Properties;
import org.apache.commons.io.IOUtils;
@@ -153,7 +155,6 @@ public class DistributedPackagingDataSetFileOperationsManager implements IDataSe
{
dataSetPackager = createPackager(file, dataSetExistenceChecker);
dataSetPackager.addDataSetTo("", dataSet);
- operationLog.info("Data set '" + dataSetCode + "' archived: " + file);
} catch (Exception ex)
{
status = Status.createError(ex.toString());
@@ -165,9 +166,19 @@ public class DistributedPackagingDataSetFileOperationsManager implements IDataSe
try
{
dataSetPackager.close();
+
+ List<String> errors =
+ verify(file, new ZipFileIntegrityVerifier(),
+ new PathInfoCrcVerifier(new CrcProvider(datasetDescription.getDataSetCode())));
+
+ if (errors.size() > 0)
+ {
+ throw new RuntimeException(errors.toString());
+ }
+ operationLog.info("Data set '" + dataSetCode + "' archived: " + file);
} catch (Exception ex)
{
- status = Status.createError("Couldn't close package file: " + file + ": " + ex);
+ operationLog.error("Couldn't create package file: " + file, ex);
}
}
shareIdManager.releaseLock(dataSetCode);
@@ -175,6 +186,16 @@ public class DistributedPackagingDataSetFileOperationsManager implements IDataSe
return status;
}
+ private List<String> verify(File file, IArchiveFileVerifier... verifiers)
+ {
+ List<String> errors = new ArrayList<String>();
+ for (IArchiveFileVerifier verifier : verifiers)
+ {
+ errors.addAll(verifier.verify(file));
+ }
+ return errors;
+ }
+
private AbstractDataSetPackager createPackager(File file, DataSetExistenceChecker dataSetExistenceChecker)
{
return new ZipDataSetPackager(file, compress, getContentProvider(), dataSetExistenceChecker);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/Verifier.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/IArchiveFileVerifier.java
similarity index 87%
rename from datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/Verifier.java
rename to datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/IArchiveFileVerifier.java
index ce61d68dbec..b4ed208e1b5 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/Verifier.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/IArchiveFileVerifier.java
@@ -16,12 +16,13 @@
package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard;
+import java.io.File;
import java.util.List;
/**
* @author anttil
*/
-public interface Verifier
+public interface IArchiveFileVerifier
{
- public List<String> verify();
+ public List<String> verify(File file);
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/ZipFilePathInfoVerifier.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/PathInfoCrcVerifier.java
similarity index 90%
rename from datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/ZipFilePathInfoVerifier.java
rename to datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/PathInfoCrcVerifier.java
index 61b570bb7af..25d6f8ed730 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/ZipFilePathInfoVerifier.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/PathInfoCrcVerifier.java
@@ -29,21 +29,18 @@ import de.schlichtherle.util.zip.ZipFile;
/**
* @author anttil
*/
-public class ZipFilePathInfoVerifier implements Verifier
+public class PathInfoCrcVerifier implements IArchiveFileVerifier
{
- private final File file;
-
private final ICrcProvider crcProvider;
- public ZipFilePathInfoVerifier(File file, ICrcProvider crcProvider)
+ public PathInfoCrcVerifier(ICrcProvider crcProvider)
{
- this.file = file;
this.crcProvider = crcProvider;
}
@Override
- public List<String> verify()
+ public List<String> verify(File file)
{
List<String> errors = new ArrayList<String>();
ZipFile zip;
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/ZipFileCrcVerifier.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/ZipFileIntegrityVerifier.java
similarity index 94%
rename from datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/ZipFileCrcVerifier.java
rename to datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/ZipFileIntegrityVerifier.java
index 0eeb48854dc..b3055dcd8c4 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/ZipFileCrcVerifier.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/ZipFileIntegrityVerifier.java
@@ -34,18 +34,11 @@ import de.schlichtherle.util.zip.ZipFile;
/**
* @author anttil
*/
-public class ZipFileCrcVerifier implements Verifier
+public class ZipFileIntegrityVerifier implements IArchiveFileVerifier
{
- private final File file;
-
- public ZipFileCrcVerifier(File file)
- {
- this.file = file;
- }
-
@Override
- public List<String> verify()
+ public List<String> verify(File file)
{
List<String> errors = new ArrayList<String>();
ZipFile zip;
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IDataSetPathInfoProvider.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IDataSetPathInfoProvider.java
index b132a332f9f..14338a0f59f 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IDataSetPathInfoProvider.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IDataSetPathInfoProvider.java
@@ -17,6 +17,7 @@
package ch.systemsx.cisd.openbis.dss.generic.shared;
import java.util.List;
+import java.util.Map;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetPathInfo;
@@ -34,4 +35,5 @@ public interface IDataSetPathInfoProvider
public ISingleDataSetPathInfoProvider tryGetSingleDataSetPathInfoProvider(String dataSetCode);
+ public Map<String, Integer> getDataSetChecksums(String dataSetCode);
}
--
GitLab