diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/path/DataSetAndPathInfoDBConsistencyCheckTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/path/DataSetAndPathInfoDBConsistencyCheckTask.java
new file mode 100644
index 0000000000000000000000000000000000000000..aaa1b29d64513b1647115b10b14dacd035382498
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/path/DataSetAndPathInfoDBConsistencyCheckTask.java
@@ -0,0 +1,114 @@
+/*
+ * Copyright 2013 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.path;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.commons.lang.time.DateUtils;
+import org.apache.log4j.Logger;
+
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.maintenance.IMaintenanceTask;
+import ch.systemsx.cisd.common.time.DateTimeUtils;
+import ch.systemsx.cisd.common.utilities.ITimeProvider;
+import ch.systemsx.cisd.common.utilities.SystemTimeProvider;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.DataSetAndPathInfoDBConsistencyChecker;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
+import ch.systemsx.cisd.openbis.generic.shared.basic.BasicConstant;
+import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
+
+/**
+ * @author Franz-Josef Elmer
+ */
+public class DataSetAndPathInfoDBConsistencyCheckTask implements IMaintenanceTask
+{
+ static final String CHECKING_TIME_INTERVAL_KEY = "checking-time-interval";
+
+ private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat(
+ BasicConstant.DATE_WITHOUT_TIMEZONE_PATTERN);
+
+ private static final Logger notificationLog = LogFactory.getLogger(LogCategory.NOTIFY,
+ DataSetAndPathInfoDBConsistencyCheckTask.class);
+
+ private final IEncapsulatedOpenBISService service;
+
+ private IHierarchicalContentProvider fileProvider;
+
+ private IHierarchicalContentProvider pathInfoProvider;
+
+ private long timeInterval;
+
+ private ITimeProvider timeProvider;
+
+ public DataSetAndPathInfoDBConsistencyCheckTask(Properties properties,
+ IEncapsulatedOpenBISService service)
+ {
+ this.service = service;
+ }
+
+ DataSetAndPathInfoDBConsistencyCheckTask(IHierarchicalContentProvider fileProvider,
+ IHierarchicalContentProvider pathInfoProvider, IEncapsulatedOpenBISService service,
+ ITimeProvider timeProvider)
+ {
+ this.fileProvider = fileProvider;
+ this.pathInfoProvider = pathInfoProvider;
+ this.service = service;
+ this.timeProvider = timeProvider;
+ }
+
+ @Override
+ public void setUp(String pluginName, Properties properties)
+ {
+ timeInterval =
+ DateTimeUtils.getDurationInMillis(properties, CHECKING_TIME_INTERVAL_KEY,
+ DateUtils.MILLIS_PER_DAY);
+ }
+
+ @Override
+ public void execute()
+ {
+ Date youngerThanDate = new Date(getTimeProvider().getTimeInMilliseconds() - timeInterval);
+ List<SimpleDataSetInformationDTO> dataSets =
+ service.listOldestPhysicalDataSets(youngerThanDate, Integer.MAX_VALUE);
+ DataSetAndPathInfoDBConsistencyChecker checker =
+ new DataSetAndPathInfoDBConsistencyChecker(fileProvider, pathInfoProvider);
+ checker.check(dataSets);
+ if (checker.noErrorAndInconsitencyFound() == false)
+ {
+ StringBuilder builder = new StringBuilder();
+ builder.append("File system and path info DB consistency check report for all data sets since ");
+ builder.append(DATE_FORMAT.format(youngerThanDate)).append("\n\n");
+ builder.append(checker.createReport());
+ notificationLog.error(builder.toString());
+ }
+ }
+
+ private ITimeProvider getTimeProvider()
+ {
+ if (timeProvider == null)
+ {
+ timeProvider = SystemTimeProvider.SYSTEM_TIME_PROVIDER;
+ }
+ return timeProvider;
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetAndPathInfoDBConsistencyCheckProcessingPlugin.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetAndPathInfoDBConsistencyCheckProcessingPlugin.java
index f27722bc00f8678e37bf07b4170e65d9fa1766e2..dc762a3d568394531a784efa15864d55210d8104 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetAndPathInfoDBConsistencyCheckProcessingPlugin.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetAndPathInfoDBConsistencyCheckProcessingPlugin.java
@@ -17,71 +17,42 @@
package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard;
import java.io.File;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
import java.util.List;
-import java.util.Map;
import java.util.Properties;
-import java.util.Set;
-import java.util.TreeMap;
-import org.apache.log4j.Logger;
-
-import ch.systemsx.cisd.common.exceptions.Status;
-import ch.systemsx.cisd.common.io.IOUtilities;
-import ch.systemsx.cisd.common.logging.LogCategory;
-import ch.systemsx.cisd.common.logging.LogFactory;
-import ch.systemsx.cisd.openbis.common.io.hierarchical_content.DefaultFileBasedHierarchicalContentFactory;
-import ch.systemsx.cisd.openbis.common.io.hierarchical_content.IHierarchicalContentFactory;
-import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContent;
-import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContentNode;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.jython.MailService;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.jython.api.IEmailSender;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.IProcessingPluginTask;
import ch.systemsx.cisd.openbis.dss.generic.shared.DataSetProcessingContext;
-import ch.systemsx.cisd.openbis.dss.generic.shared.HierarchicalContentProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.ProcessingStatus;
-import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
-import ch.systemsx.cisd.openbis.dss.generic.shared.content.DssServiceRpcGenericFactory;
-import ch.systemsx.cisd.openbis.dss.generic.shared.content.IDssServiceRpcGenericFactory;
-import ch.systemsx.cisd.openbis.dss.generic.shared.content.PathInfoDBOnlyHierarchicalContentFactory;
import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
/**
+ * Processing plugin which cheks consitency between data set files in the store and the information
+ * stored in pathinfo database.
+ *
* @author pkupczyk
*/
public class DataSetAndPathInfoDBConsistencyCheckProcessingPlugin implements IProcessingPluginTask
{
-
- private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
- DataSetAndPathInfoDBConsistencyCheckProcessingPlugin.class);
+ private static final long serialVersionUID = 1L;
private transient IHierarchicalContentProvider fileProvider;
private transient IHierarchicalContentProvider pathInfoProvider;
- private transient IDssServiceRpcGenericFactory serviceFactory;
-
- private static final long serialVersionUID = 1L;
-
public DataSetAndPathInfoDBConsistencyCheckProcessingPlugin(Properties properties,
File storeRoot)
{
}
/**
- * A package visible constructor for testing purposes.
*
- * @param properties The configuration properties.
- * @param storeRoot The root of the dss store.
* @param fileProvider The hierarchical content provider that references the file system.
* @param pathInfoProvider The hierarchical content provider that references the path-info db.
*/
- DataSetAndPathInfoDBConsistencyCheckProcessingPlugin(Properties properties, File storeRoot,
+ public DataSetAndPathInfoDBConsistencyCheckProcessingPlugin(
IHierarchicalContentProvider fileProvider, IHierarchicalContentProvider pathInfoProvider)
{
this.fileProvider = fileProvider;
@@ -92,557 +63,17 @@ public class DataSetAndPathInfoDBConsistencyCheckProcessingPlugin implements IPr
public ProcessingStatus process(List<DatasetDescription> datasets,
DataSetProcessingContext context)
{
- Map<DatasetDescription, List<Difference>> differences =
- new HashMap<DatasetDescription, List<Difference>>();
- ProcessingStatus status = new ProcessingStatus();
-
- for (DatasetDescription dataset : datasets)
- {
- IHierarchicalContent fileContent = null;
- IHierarchicalContent pathInfoContent = null;
-
- try
- {
- fileContent = tryGetContent(getFileProvider(), dataset.getDataSetCode());
- pathInfoContent = tryGetContent(getPathInfoProvider(), dataset.getDataSetCode());
-
- List<Difference> datasetDifferences = new ArrayList<Difference>();
-
- compare(fileContent, pathInfoContent, datasetDifferences);
-
- if (datasetDifferences.isEmpty() == false)
- {
- differences.put(dataset, datasetDifferences);
- }
- status.addDatasetStatus(dataset, Status.OK);
-
- } catch (Exception e)
- {
- operationLog.error(
- "Couldn't check consistency of the file system and the path info database for a data set: "
- + dataset.getDataSetCode(), e);
- status.addDatasetStatus(
- dataset,
- Status.createError("Couldn't check consistency of the file system and the path info database for a data set: "
- + dataset.getDataSetCode()
- + " because of the following exception: " + e.getMessage()));
- } finally
- {
- if (null != fileContent)
- {
- fileContent.close();
- }
- if (null != pathInfoContent)
- {
- pathInfoContent.close();
- }
- }
- }
-
- if (status.getErrorStatuses().isEmpty())
- {
- sendEmail(datasets, context, differences);
- } else
- {
- sendErrorEmail(context, status);
- }
-
- return status;
- }
-
- private void compare(IHierarchicalContent fileContent, IHierarchicalContent pathInfoContent,
- List<Difference> differences)
- {
- IHierarchicalContentNode fileRoot = tryGetRoot(fileContent);
- IHierarchicalContentNode pathInfoRoot = tryGetRoot(pathInfoContent);
-
- if (fileRoot != null && pathInfoRoot != null)
- {
- compare(fileRoot, pathInfoRoot, differences);
- } else if (fileRoot == null ^ pathInfoRoot == null)
- {
- differences.add(new RootExistenceDifference(fileRoot != null));
- } else
- {
- throw new IllegalArgumentException(
- "Data set does not exist on the file system nor in the path info database");
- }
- }
-
- @SuppressWarnings("null")
- private void compare(IHierarchicalContentNode fileNode, IHierarchicalContentNode pathInfoNode,
- List<Difference> differences)
- {
- boolean fileNodeExists = fileNode != null && fileNode.exists();
- boolean pathInfoNodeExists = pathInfoNode != null && pathInfoNode.exists();
-
- // report a difference if one node is null and the other not
- if (fileNodeExists == false || pathInfoNodeExists == false)
- {
- if (fileNodeExists && pathInfoNodeExists == false)
- {
- differences.add(new NodeExistenceDifference(fileNode.getRelativePath(), true));
- }
- if (pathInfoNodeExists && fileNodeExists == false)
- {
- differences.add(new NodeExistenceDifference(pathInfoNode.getRelativePath(), false));
- }
- return;
- }
-
- // report a difference if node paths do not match
- if (fileNode.getRelativePath().equals(pathInfoNode.getRelativePath()) == false)
- {
- differences.add(new NodeChildrenDifference(fileNode.getRelativePath(), true));
- differences.add(new NodeChildrenDifference(pathInfoNode.getRelativePath(), false));
- }
-
- if (fileNode.isDirectory() && pathInfoNode.isDirectory())
- {
- Children children = new Children(fileNode, pathInfoNode);
-
- // compare children nodes that exist both in the file system and the path info database
- for (String commonPath : children.getCommonPaths())
- {
- compare(children.getFileNode(commonPath), children.getPathInfoNode(commonPath),
- differences);
- }
-
- // report differences for nodes that exist only in one place
- for (IHierarchicalContentNode uncommonNode : children.getFileUncommonNodes())
- {
- differences.add(new NodeChildrenDifference(uncommonNode.getRelativePath(), true));
- }
- for (IHierarchicalContentNode uncommonNode : children.getPathInfoUncommonNodes())
- {
- differences.add(new NodeChildrenDifference(uncommonNode.getRelativePath(), false));
- }
-
- } else if (fileNode.isDirectory() == false && pathInfoNode.isDirectory() == false)
- {
- // check file lengths
- if (fileNode.getFileLength() != pathInfoNode.getFileLength())
- {
- differences.add(new SizeDifference(fileNode.getRelativePath(), fileNode
- .getFileLength(), pathInfoNode.getFileLength()));
- }
- // check checksums if stored in path Info db
- if (pathInfoNode.isChecksumCRC32Precalculated()
- && (fileNode.getChecksumCRC32() != pathInfoNode.getChecksumCRC32()))
- {
- differences.add(new ChecksumDifference(fileNode.getRelativePath(), fileNode
- .getChecksumCRC32(), pathInfoNode.getChecksumCRC32()));
- }
- } else
- {
- // report a difference if one node is a directory and the other is a file
- differences.add(new DirectoryDifference(fileNode.getRelativePath(), fileNode
- .isDirectory()));
- }
- }
-
- private void sendEmail(List<DatasetDescription> datasets, DataSetProcessingContext context,
- Map<DatasetDescription, List<Difference>> differences)
- {
+ DataSetAndPathInfoDBConsistencyChecker checker = new DataSetAndPathInfoDBConsistencyChecker(fileProvider, pathInfoProvider);
+ checker.check(datasets);
+ String report = checker.createReport();
+
IEmailSender mailSender =
new MailService(context.getMailClient(), context.getUserEmailOrNull())
.createEmailSender();
-
mailSender.withSubject("File system and path info DB consistency check report");
-
- StringBuilder body = new StringBuilder();
- body.append("Data sets checked:\n\n");
-
- Iterator<DatasetDescription> datasetsIterator = datasets.iterator();
- while (datasetsIterator.hasNext())
- {
- body.append(datasetsIterator.next().getDataSetCode());
- if (datasetsIterator.hasNext())
- {
- body.append(", ");
- }
- }
-
- body.append("\n\n");
- body.append("Differences found:\n\n");
-
- if (differences.isEmpty())
- {
- body.append("None");
- } else
- {
- for (Map.Entry<DatasetDescription, List<Difference>> differencesEntry : differences
- .entrySet())
- {
- DatasetDescription dataset = differencesEntry.getKey();
- List<Difference> datasetDifferences = differencesEntry.getValue();
-
- Collections.sort(datasetDifferences);
-
- body.append("Data set " + dataset.getDataSetCode() + ":\n");
- for (Difference datasetDifference : datasetDifferences)
- {
- body.append("- " + datasetDifference.getDescription() + "\n");
- }
- body.append("\n");
- }
- }
-
- mailSender.withBody(body.toString());
+ mailSender.withBody(report);
mailSender.send();
- }
-
- private void sendErrorEmail(DataSetProcessingContext context, ProcessingStatus status)
- {
- IEmailSender mailSender =
- new MailService(context.getMailClient(), context.getUserEmailOrNull())
- .createEmailSender();
-
- mailSender.withSubject("File system and path info DB consistency check report");
-
- final StringBuilder body = new StringBuilder();
- body.append("Error when checking datasets:\n\n");
-
- for (Status s : status.getErrorStatuses())
- {
- body.append(s.toString());
- body.append("\n");
- }
-
- mailSender.withBody(body.toString());
- mailSender.send();
- }
-
- private abstract class Difference implements Comparable<Difference>
- {
-
- private String path;
-
- public Difference(String path)
- {
- this.path = path;
- }
-
- public String getPath()
- {
- return path;
- }
-
- public abstract String getDescription();
-
- @Override
- public int compareTo(Difference o)
- {
- return getPath().compareTo(o.getPath());
- }
-
- }
-
- private class DirectoryDifference extends Difference
- {
-
- private boolean dirInFS;
-
- public DirectoryDifference(String path, boolean dirInFS)
- {
- super(path);
- this.dirInFS = dirInFS;
- }
-
- @Override
- public String getDescription()
- {
- if (dirInFS)
- {
- return "'"
- + getPath()
- + "' is a directory in the file system but a file in the path info database";
- } else
- {
- return "'"
- + getPath()
- + "' is a directory in the path info database but a file in the file system";
- }
- }
-
- }
- private class RootExistenceDifference extends Difference
- {
-
- private boolean existsInFS;
-
- public RootExistenceDifference(boolean existsInFS)
- {
- super(null);
- this.existsInFS = existsInFS;
- }
-
- @Override
- public String getDescription()
- {
- if (existsInFS)
- {
- return "exists in the file system but does not exist in the path info database";
- } else
- {
- return "exists in the path info database but does not exist in the file system";
- }
- }
-
- }
-
- private class NodeExistenceDifference extends Difference
- {
-
- private boolean existsInFS;
-
- public NodeExistenceDifference(String path, boolean existsInFS)
- {
- super(path);
- this.existsInFS = existsInFS;
- }
-
- @Override
- public String getDescription()
- {
- if (existsInFS)
- {
- return "'"
- + getPath()
- + "' exists on the file system but does not exist in the path info database";
- } else
- {
- return "'"
- + getPath()
- + "' exists in the path info database but does not exist on the file system";
- }
- }
-
- }
-
- private class NodeChildrenDifference extends Difference
- {
-
- private boolean existsInFS;
-
- public NodeChildrenDifference(String path, boolean existsInFS)
- {
- super(path);
- this.existsInFS = existsInFS;
- }
-
- @Override
- public String getDescription()
- {
- if (existsInFS)
- {
- return "'" + getPath()
- + "' is on the file system but is not referenced in the path info database";
- } else
- {
- return "'"
- + getPath()
- + "' is referenced in the path info database but does not exist on the file system";
- }
- }
-
- }
-
- private class SizeDifference extends Difference
- {
-
- private long sizeInFS;
-
- private long sizeInDB;
-
- public SizeDifference(String path, long sizeInFS, long sizeInDB)
- {
- super(path);
- this.sizeInFS = sizeInFS;
- this.sizeInDB = sizeInDB;
- }
-
- @Override
- public String getDescription()
- {
- return "'" + getPath() + "' size in the file system = " + sizeInFS
- + " bytes but in the path info database = " + sizeInDB + " bytes.";
- }
-
- }
-
- private class ChecksumDifference extends Difference
- {
-
- private int checksumInFS;
-
- private int checksumInDB;
-
- public ChecksumDifference(String path, int checksumInFS, int checksumInDB)
- {
- super(path);
- this.checksumInFS = checksumInFS;
- this.checksumInDB = checksumInDB;
- }
-
- @Override
- public String getDescription()
- {
- return "'" + getPath() + "' CRC32 checksum in the file system = "
- + IOUtilities.crc32ToString(checksumInFS) + " but in the path info database = "
- + IOUtilities.crc32ToString(checksumInDB);
- }
-
- }
-
- private class Children
- {
-
- private Map<String, IHierarchicalContentNode> fileChildrenMap;
-
- private Map<String, IHierarchicalContentNode> pathInfoChildrenMap;
-
- public Children(IHierarchicalContentNode fileNode, IHierarchicalContentNode pathInfoNode)
- {
- this.fileChildrenMap = getMap(fileNode);
- this.pathInfoChildrenMap = getMap(pathInfoNode);
- }
-
- public IHierarchicalContentNode getFileNode(String path)
- {
- return fileChildrenMap.get(path);
- }
-
- public IHierarchicalContentNode getPathInfoNode(String path)
- {
- return pathInfoChildrenMap.get(path);
- }
-
- public Set<IHierarchicalContentNode> getFileUncommonNodes()
- {
- return getUncommonNodes(fileChildrenMap);
- }
-
- public Set<IHierarchicalContentNode> getPathInfoUncommonNodes()
- {
- return getUncommonNodes(pathInfoChildrenMap);
- }
-
- public Set<String> getCommonPaths()
- {
- Set<String> commonPaths = new HashSet<String>();
- for (String path : fileChildrenMap.keySet())
- {
- if (pathInfoChildrenMap.containsKey(path))
- {
- commonPaths.add(path);
- }
- }
- return commonPaths;
- }
-
- private Map<String, IHierarchicalContentNode> getMap(IHierarchicalContentNode node)
- {
- Map<String, IHierarchicalContentNode> map =
- new TreeMap<String, IHierarchicalContentNode>();
- for (IHierarchicalContentNode child : node.getChildNodes())
- {
- map.put(child.getRelativePath(), child);
- }
- return map;
- }
-
- public Set<IHierarchicalContentNode> getUncommonNodes(
- Map<String, IHierarchicalContentNode> childrenMap)
- {
- Set<String> commonNames = getCommonPaths();
- Set<IHierarchicalContentNode> uncommonNodes = new HashSet<IHierarchicalContentNode>();
-
- for (Map.Entry<String, IHierarchicalContentNode> child : childrenMap.entrySet())
- {
- if (commonNames.contains(child.getValue().getRelativePath()) == false)
- {
- uncommonNodes.add(child.getValue());
- }
- }
- return uncommonNodes;
- }
-
- }
-
- private IHierarchicalContent tryGetContent(IHierarchicalContentProvider contentProvider,
- String datasetCode)
- {
- try
- {
- return contentProvider.asContent(datasetCode);
- } catch (IllegalArgumentException e)
- {
- return null;
- }
- }
-
- private IHierarchicalContentNode tryGetRoot(IHierarchicalContent content)
- {
- try
- {
- if (content == null)
- {
- return null;
- } else
- {
- return content.getRootNode();
- }
- } catch (IllegalArgumentException e)
- {
- return null;
- }
- }
-
- private IHierarchicalContentProvider getFileProvider()
- {
- if (fileProvider == null)
- {
- fileProvider =
- new HierarchicalContentProvider(ServiceProvider.getOpenBISService(),
- ServiceProvider.getShareIdManager(),
- ServiceProvider.getConfigProvider(), ServiceProvider.getContentCache(),
- new DefaultFileBasedHierarchicalContentFactory(), getServiceFactory(),
- null, null);
- }
- return fileProvider;
- }
-
- private IHierarchicalContentProvider getPathInfoProvider()
- {
- if (pathInfoProvider == null)
- {
- IHierarchicalContentFactory pathInfoDBFactory =
- PathInfoDBOnlyHierarchicalContentFactory.create();
-
- if (pathInfoDBFactory == null)
- {
- throw new IllegalArgumentException("Path info database is not configured.");
- } else
- {
- pathInfoProvider =
- new HierarchicalContentProvider(ServiceProvider.getOpenBISService(),
- ServiceProvider.getShareIdManager(),
- ServiceProvider.getConfigProvider(),
- ServiceProvider.getContentCache(), pathInfoDBFactory,
- getServiceFactory(), null, null);
- }
- }
- return pathInfoProvider;
- }
-
- private IDssServiceRpcGenericFactory getServiceFactory()
- {
- if (serviceFactory == null)
- {
- serviceFactory = new DssServiceRpcGenericFactory();
- }
- return serviceFactory;
+ return checker.getStatus();
}
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetAndPathInfoDBConsistencyChecker.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetAndPathInfoDBConsistencyChecker.java
new file mode 100644
index 0000000000000000000000000000000000000000..893e25e60d2e91fca4caa6a91c5d5703e4a6f907
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetAndPathInfoDBConsistencyChecker.java
@@ -0,0 +1,641 @@
+/*
+ * Copyright 2013 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.log4j.Logger;
+
+import ch.systemsx.cisd.common.exceptions.Status;
+import ch.systemsx.cisd.common.io.IOUtilities;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.DefaultFileBasedHierarchicalContentFactory;
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.IHierarchicalContentFactory;
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContent;
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContentNode;
+import ch.systemsx.cisd.openbis.dss.generic.shared.HierarchicalContentProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ProcessingStatus;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.content.DssServiceRpcGenericFactory;
+import ch.systemsx.cisd.openbis.dss.generic.shared.content.IDssServiceRpcGenericFactory;
+import ch.systemsx.cisd.openbis.dss.generic.shared.content.PathInfoDBOnlyHierarchicalContentFactory;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IDatasetLocation;
+
+/**
+ *
+ *
+ * @author Franz-Josef Elmer
+ */
+public class DataSetAndPathInfoDBConsistencyChecker
+{
+ private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
+ DataSetAndPathInfoDBConsistencyChecker.class);
+
+ private static final Comparator<Map.Entry<IDatasetLocation, List<Difference>>> DATA_SET_COMPARATOR =
+ new Comparator<Map.Entry<IDatasetLocation, List<Difference>>>()
+ {
+ @Override
+ public int compare(Entry<IDatasetLocation, List<Difference>> e1,
+ Entry<IDatasetLocation, List<Difference>> e2)
+ {
+ return e1.getKey().getDataSetCode().compareTo(e2.getKey().getDataSetCode());
+ }
+ };
+
+ private IHierarchicalContentProvider fileProvider;
+ private IHierarchicalContentProvider pathInfoProvider;
+ private IDssServiceRpcGenericFactory serviceFactory;
+
+ private Map<IDatasetLocation, List<Difference>> differences;
+
+ private ProcessingStatus status;
+
+ private List<? extends IDatasetLocation> dataSets;
+
+ public DataSetAndPathInfoDBConsistencyChecker(
+ IHierarchicalContentProvider fileProvider, IHierarchicalContentProvider pathInfoProvider)
+ {
+ this.fileProvider = fileProvider;
+ this.pathInfoProvider = pathInfoProvider;
+ }
+
+ public void check(List<? extends IDatasetLocation> datasets)
+ {
+ dataSets = datasets;
+ differences = new HashMap<IDatasetLocation, List<Difference>>();
+ status = new ProcessingStatus();
+
+ for (IDatasetLocation dataset : datasets)
+ {
+ IHierarchicalContent fileContent = null;
+ IHierarchicalContent pathInfoContent = null;
+
+ try
+ {
+ fileContent = tryGetContent(getFileProvider(), dataset.getDataSetCode());
+ pathInfoContent = tryGetContent(getPathInfoProvider(), dataset.getDataSetCode());
+
+ List<Difference> datasetDifferences = new ArrayList<Difference>();
+
+ compare(fileContent, pathInfoContent, datasetDifferences);
+
+ if (datasetDifferences.isEmpty() == false)
+ {
+ differences.put(dataset, datasetDifferences);
+ }
+ status.addDatasetStatus(dataset.getDataSetCode(), Status.OK);
+
+ } catch (Exception e)
+ {
+ operationLog.error(
+ "Couldn't check consistency of the file system and the path info database for a data set: "
+ + dataset.getDataSetCode(), e);
+ status.addDatasetStatus(
+ dataset.getDataSetCode(),
+ Status.createError("Couldn't check consistency of the file system and the path info database for a data set: "
+ + dataset.getDataSetCode()
+ + " because of the following exception: " + e.getMessage()));
+ } finally
+ {
+ if (null != fileContent)
+ {
+ fileContent.close();
+ }
+ if (null != pathInfoContent)
+ {
+ pathInfoContent.close();
+ }
+ }
+ }
+ }
+
+ public ProcessingStatus getStatus()
+ {
+ if (status == null)
+ {
+ throw new IllegalStateException("Undefined status before check() has been executed.");
+ }
+ return status;
+ }
+
+ public boolean noErrorAndInconsitencyFound()
+ {
+ return status.getErrorStatuses().isEmpty() && differences.isEmpty();
+ }
+
+ public String createReport()
+ {
+ return status.getErrorStatuses().isEmpty() ? createNormalReport() : createErrorReport();
+ }
+
+ private String createNormalReport()
+ {
+ StringBuilder builder = new StringBuilder();
+ builder.append("Data sets checked:\n\n");
+
+ Iterator<? extends IDatasetLocation> datasetsIterator = dataSets.iterator();
+ while (datasetsIterator.hasNext())
+ {
+ builder.append(datasetsIterator.next().getDataSetCode());
+ if (datasetsIterator.hasNext())
+ {
+ builder.append(", ");
+ }
+ }
+
+ builder.append("\n\n");
+ builder.append("Differences found:\n\n");
+
+ if (differences.isEmpty())
+ {
+ builder.append("None");
+ } else
+ {
+ List<Map.Entry<IDatasetLocation, List<Difference>>> entries =
+ new ArrayList<Map.Entry<IDatasetLocation, List<Difference>>>(
+ differences.entrySet());
+ Collections.sort(entries, DATA_SET_COMPARATOR);
+ for (Map.Entry<IDatasetLocation, List<Difference>> differencesEntry : entries)
+ {
+ IDatasetLocation dataset = differencesEntry.getKey();
+ List<Difference> datasetDifferences = differencesEntry.getValue();
+
+ Collections.sort(datasetDifferences);
+
+ builder.append("Data set " + dataset.getDataSetCode() + ":\n");
+ for (Difference datasetDifference : datasetDifferences)
+ {
+ builder.append("- " + datasetDifference.getDescription() + "\n");
+ }
+ builder.append("\n");
+ }
+ }
+
+ return builder.toString();
+ }
+
+ private String createErrorReport()
+ {
+ final StringBuilder builder = new StringBuilder();
+ builder.append("Error when checking datasets:\n\n");
+
+ for (Status s : status.getErrorStatuses())
+ {
+ builder.append(s.toString());
+ builder.append("\n");
+ }
+
+ return builder.toString();
+
+ }
+
+ private void compare(IHierarchicalContent fileContent, IHierarchicalContent pathInfoContent,
+ List<Difference> diffs)
+ {
+ IHierarchicalContentNode fileRoot = tryGetRoot(fileContent);
+ IHierarchicalContentNode pathInfoRoot = tryGetRoot(pathInfoContent);
+
+ if (fileRoot != null && pathInfoRoot != null)
+ {
+ compare(fileRoot, pathInfoRoot, diffs);
+ } else if (fileRoot == null ^ pathInfoRoot == null)
+ {
+ diffs.add(new RootExistenceDifference(fileRoot != null));
+ } else
+ {
+ throw new IllegalArgumentException(
+ "Data set does not exist on the file system nor in the path info database");
+ }
+ }
+
+ @SuppressWarnings("null")
+ private void compare(IHierarchicalContentNode fileNode, IHierarchicalContentNode pathInfoNode,
+ List<Difference> diffs)
+ {
+ boolean fileNodeExists = fileNode != null && fileNode.exists();
+ boolean pathInfoNodeExists = pathInfoNode != null && pathInfoNode.exists();
+
+ // report a difference if one node is null and the other not
+ if (fileNodeExists == false || pathInfoNodeExists == false)
+ {
+ if (fileNodeExists && pathInfoNodeExists == false)
+ {
+ diffs.add(new NodeExistenceDifference(fileNode.getRelativePath(), true));
+ }
+ if (pathInfoNodeExists && fileNodeExists == false)
+ {
+ diffs.add(new NodeExistenceDifference(pathInfoNode.getRelativePath(), false));
+ }
+ return;
+ }
+
+ // report a difference if node paths do not match
+ if (fileNode.getRelativePath().equals(pathInfoNode.getRelativePath()) == false)
+ {
+ diffs.add(new NodeChildrenDifference(fileNode.getRelativePath(), true));
+ diffs.add(new NodeChildrenDifference(pathInfoNode.getRelativePath(), false));
+ }
+
+ if (fileNode.isDirectory() && pathInfoNode.isDirectory())
+ {
+ Children children = new Children(fileNode, pathInfoNode);
+
+ // compare children nodes that exist both in the file system and the path info database
+ for (String commonPath : children.getCommonPaths())
+ {
+ compare(children.getFileNode(commonPath), children.getPathInfoNode(commonPath),
+ diffs);
+ }
+
+ // report differences for nodes that exist only in one place
+ for (IHierarchicalContentNode uncommonNode : children.getFileUncommonNodes())
+ {
+ diffs.add(new NodeChildrenDifference(uncommonNode.getRelativePath(), true));
+ }
+ for (IHierarchicalContentNode uncommonNode : children.getPathInfoUncommonNodes())
+ {
+ diffs.add(new NodeChildrenDifference(uncommonNode.getRelativePath(), false));
+ }
+
+ } else if (fileNode.isDirectory() == false && pathInfoNode.isDirectory() == false)
+ {
+ // check file lengths
+ if (fileNode.getFileLength() != pathInfoNode.getFileLength())
+ {
+ diffs.add(new SizeDifference(fileNode.getRelativePath(), fileNode
+ .getFileLength(), pathInfoNode.getFileLength()));
+ }
+ // check checksums if stored in path Info db
+ if (pathInfoNode.isChecksumCRC32Precalculated()
+ && (fileNode.getChecksumCRC32() != pathInfoNode.getChecksumCRC32()))
+ {
+ diffs.add(new ChecksumDifference(fileNode.getRelativePath(), fileNode
+ .getChecksumCRC32(), pathInfoNode.getChecksumCRC32()));
+ }
+ } else
+ {
+ // report a difference if one node is a directory and the other is a file
+ diffs.add(new DirectoryDifference(fileNode.getRelativePath(), fileNode
+ .isDirectory()));
+ }
+ }
+
+ private abstract class Difference implements Comparable<Difference>
+ {
+
+ private String path;
+
+ public Difference(String path)
+ {
+ this.path = path;
+ }
+
+ public String getPath()
+ {
+ return path;
+ }
+
+ public abstract String getDescription();
+
+ @Override
+ public int compareTo(Difference o)
+ {
+ return getPath().compareTo(o.getPath());
+ }
+
+ }
+
+ private class DirectoryDifference extends Difference
+ {
+
+ private boolean dirInFS;
+
+ public DirectoryDifference(String path, boolean dirInFS)
+ {
+ super(path);
+ this.dirInFS = dirInFS;
+ }
+
+ @Override
+ public String getDescription()
+ {
+ if (dirInFS)
+ {
+ return "'"
+ + getPath()
+ + "' is a directory in the file system but a file in the path info database";
+ } else
+ {
+ return "'"
+ + getPath()
+ + "' is a directory in the path info database but a file in the file system";
+ }
+ }
+
+ }
+
+ private class RootExistenceDifference extends Difference
+ {
+
+ private boolean existsInFS;
+
+ public RootExistenceDifference(boolean existsInFS)
+ {
+ super(null);
+ this.existsInFS = existsInFS;
+ }
+
+ @Override
+ public String getDescription()
+ {
+ if (existsInFS)
+ {
+ return "exists in the file system but does not exist in the path info database";
+ } else
+ {
+ return "exists in the path info database but does not exist in the file system";
+ }
+ }
+
+ }
+
+ private class NodeExistenceDifference extends Difference
+ {
+
+ private boolean existsInFS;
+
+ public NodeExistenceDifference(String path, boolean existsInFS)
+ {
+ super(path);
+ this.existsInFS = existsInFS;
+ }
+
+ @Override
+ public String getDescription()
+ {
+ if (existsInFS)
+ {
+ return "'"
+ + getPath()
+ + "' exists on the file system but does not exist in the path info database";
+ } else
+ {
+ return "'"
+ + getPath()
+ + "' exists in the path info database but does not exist on the file system";
+ }
+ }
+
+ }
+
+ private class NodeChildrenDifference extends Difference
+ {
+
+ private boolean existsInFS;
+
+ public NodeChildrenDifference(String path, boolean existsInFS)
+ {
+ super(path);
+ this.existsInFS = existsInFS;
+ }
+
+ @Override
+ public String getDescription()
+ {
+ if (existsInFS)
+ {
+ return "'" + getPath()
+ + "' is on the file system but is not referenced in the path info database";
+ } else
+ {
+ return "'"
+ + getPath()
+ + "' is referenced in the path info database but does not exist on the file system";
+ }
+ }
+
+ }
+
+ private class SizeDifference extends Difference
+ {
+
+ private long sizeInFS;
+
+ private long sizeInDB;
+
+ public SizeDifference(String path, long sizeInFS, long sizeInDB)
+ {
+ super(path);
+ this.sizeInFS = sizeInFS;
+ this.sizeInDB = sizeInDB;
+ }
+
+ @Override
+ public String getDescription()
+ {
+ return "'" + getPath() + "' size in the file system = " + sizeInFS
+ + " bytes but in the path info database = " + sizeInDB + " bytes.";
+ }
+
+ }
+
+ private class ChecksumDifference extends Difference
+ {
+
+ private int checksumInFS;
+
+ private int checksumInDB;
+
+ public ChecksumDifference(String path, int checksumInFS, int checksumInDB)
+ {
+ super(path);
+ this.checksumInFS = checksumInFS;
+ this.checksumInDB = checksumInDB;
+ }
+
+ @Override
+ public String getDescription()
+ {
+ return "'" + getPath() + "' CRC32 checksum in the file system = "
+ + IOUtilities.crc32ToString(checksumInFS) + " but in the path info database = "
+ + IOUtilities.crc32ToString(checksumInDB);
+ }
+
+ }
+
+ private class Children
+ {
+
+ private Map<String, IHierarchicalContentNode> fileChildrenMap;
+
+ private Map<String, IHierarchicalContentNode> pathInfoChildrenMap;
+
+ public Children(IHierarchicalContentNode fileNode, IHierarchicalContentNode pathInfoNode)
+ {
+ this.fileChildrenMap = getMap(fileNode);
+ this.pathInfoChildrenMap = getMap(pathInfoNode);
+ }
+
+ public IHierarchicalContentNode getFileNode(String path)
+ {
+ return fileChildrenMap.get(path);
+ }
+
+ public IHierarchicalContentNode getPathInfoNode(String path)
+ {
+ return pathInfoChildrenMap.get(path);
+ }
+
+ public Set<IHierarchicalContentNode> getFileUncommonNodes()
+ {
+ return getUncommonNodes(fileChildrenMap);
+ }
+
+ public Set<IHierarchicalContentNode> getPathInfoUncommonNodes()
+ {
+ return getUncommonNodes(pathInfoChildrenMap);
+ }
+
+ public Set<String> getCommonPaths()
+ {
+ Set<String> commonPaths = new HashSet<String>();
+ for (String path : fileChildrenMap.keySet())
+ {
+ if (pathInfoChildrenMap.containsKey(path))
+ {
+ commonPaths.add(path);
+ }
+ }
+ return commonPaths;
+ }
+
+ private Map<String, IHierarchicalContentNode> getMap(IHierarchicalContentNode node)
+ {
+ Map<String, IHierarchicalContentNode> map =
+ new TreeMap<String, IHierarchicalContentNode>();
+ for (IHierarchicalContentNode child : node.getChildNodes())
+ {
+ map.put(child.getRelativePath(), child);
+ }
+ return map;
+ }
+
+ public Set<IHierarchicalContentNode> getUncommonNodes(
+ Map<String, IHierarchicalContentNode> childrenMap)
+ {
+ Set<String> commonNames = getCommonPaths();
+ Set<IHierarchicalContentNode> uncommonNodes = new HashSet<IHierarchicalContentNode>();
+
+ for (Map.Entry<String, IHierarchicalContentNode> child : childrenMap.entrySet())
+ {
+ if (commonNames.contains(child.getValue().getRelativePath()) == false)
+ {
+ uncommonNodes.add(child.getValue());
+ }
+ }
+ return uncommonNodes;
+ }
+
+ }
+
+ private IHierarchicalContent tryGetContent(IHierarchicalContentProvider contentProvider,
+ String datasetCode)
+ {
+ try
+ {
+ return contentProvider.asContent(datasetCode);
+ } catch (IllegalArgumentException e)
+ {
+ return null;
+ }
+ }
+
+ private IHierarchicalContentNode tryGetRoot(IHierarchicalContent content)
+ {
+ try
+ {
+ if (content == null)
+ {
+ return null;
+ } else
+ {
+ return content.getRootNode();
+ }
+ } catch (IllegalArgumentException e)
+ {
+ return null;
+ }
+ }
+
+ private IHierarchicalContentProvider getFileProvider()
+ {
+ if (fileProvider == null)
+ {
+ fileProvider =
+ new HierarchicalContentProvider(ServiceProvider.getOpenBISService(),
+ ServiceProvider.getShareIdManager(),
+ ServiceProvider.getConfigProvider(), ServiceProvider.getContentCache(),
+ new DefaultFileBasedHierarchicalContentFactory(), getServiceFactory(),
+ null, null);
+ }
+ return fileProvider;
+ }
+
+ private IHierarchicalContentProvider getPathInfoProvider()
+ {
+ if (pathInfoProvider == null)
+ {
+ IHierarchicalContentFactory pathInfoDBFactory =
+ PathInfoDBOnlyHierarchicalContentFactory.create();
+
+ if (pathInfoDBFactory == null)
+ {
+ throw new IllegalArgumentException("Path info database is not configured.");
+ } else
+ {
+ pathInfoProvider =
+ new HierarchicalContentProvider(ServiceProvider.getOpenBISService(),
+ ServiceProvider.getShareIdManager(),
+ ServiceProvider.getConfigProvider(),
+ ServiceProvider.getContentCache(), pathInfoDBFactory,
+ getServiceFactory(), null, null);
+ }
+ }
+ return pathInfoProvider;
+ }
+
+ private IDssServiceRpcGenericFactory getServiceFactory()
+ {
+ if (serviceFactory == null)
+ {
+ serviceFactory = new DssServiceRpcGenericFactory();
+ }
+ return serviceFactory;
+ }
+}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/path/DataSetAndPathInfoDBConsistencyCheckTaskTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/path/DataSetAndPathInfoDBConsistencyCheckTaskTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..435a4dfe9b2b910fd60a39e685ec180b50f82bcc
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/path/DataSetAndPathInfoDBConsistencyCheckTaskTest.java
@@ -0,0 +1,241 @@
+/*
+ * Copyright 2013 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.path;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.log4j.Level;
+import org.jmock.Expectations;
+import org.jmock.Mockery;
+import org.testng.AssertJUnit;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.common.logging.BufferedAppender;
+import ch.systemsx.cisd.common.logging.LogUtils;
+import ch.systemsx.cisd.common.utilities.MockTimeProvider;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.MockContent;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.PhysicalDataSet;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.builders.DataSetBuilder;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.builders.DataStoreBuilder;
+import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
+import ch.systemsx.cisd.openbis.generic.shared.translator.DataSetTranslator;
+import ch.systemsx.cisd.openbis.generic.shared.translator.SimpleDataSetHelper;
+
+/**
+ * @author Franz-Josef Elmer
+ */
+public class DataSetAndPathInfoDBConsistencyCheckTaskTest extends AssertJUnit
+{
+ private BufferedAppender logRecorder;
+
+ private Mockery context;
+
+ private IEncapsulatedOpenBISService service;
+
+ private IHierarchicalContentProvider fileProvider;
+
+ private IHierarchicalContentProvider pathInfoProvider;
+
+ private DataSetAndPathInfoDBConsistencyCheckTask task;
+
+ @BeforeMethod
+ public void setUpMocks()
+ {
+ logRecorder = new BufferedAppender("%-5p %c - %m%n", Level.INFO);
+ context = new Mockery();
+ service = context.mock(IEncapsulatedOpenBISService.class);
+ fileProvider = context.mock(IHierarchicalContentProvider.class, "fileProvider");
+ pathInfoProvider = context.mock(IHierarchicalContentProvider.class, "pathInfoProvider");
+ task =
+ new DataSetAndPathInfoDBConsistencyCheckTask(fileProvider, pathInfoProvider,
+ service, new MockTimeProvider(2010, 1000));
+ Properties properties = new Properties();
+ properties.setProperty(DataSetAndPathInfoDBConsistencyCheckTask.CHECKING_TIME_INTERVAL_KEY,
+ "1500 msec");
+ task.setUp("", properties);
+ }
+
+ @AfterMethod
+ public void checkContext()
+ {
+ logRecorder.reset();
+ // The following line of code should also be called at the end of each test method.
+ // Otherwise one do not known which test failed.
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testConsistentCase()
+ {
+ PhysicalDataSet ds1 = createDataSetBuilder().code("ds1").getDataSet();
+ prepareListDataSets(new Date(2010 - 1500), ds1);
+ MockContent fileContent =
+ prepareContentProvider(fileProvider, "ds1", ":0:0", "a/:0:0", "a/b:34:9");
+ MockContent pathInfoContent =
+ prepareContentProvider(pathInfoProvider, "ds1", ":0:0", "a/:0:0", "a/b:34:9");
+
+ task.execute();
+
+ assertEquals("", logRecorder.getLogContent());
+ assertEquals(true, fileContent.isClosed());
+ assertEquals(true, pathInfoContent.isClosed());
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testErrorCase()
+ {
+ PhysicalDataSet ds1 = createDataSetBuilder().code("ds1").getDataSet();
+ prepareListDataSets(new Date(2010 - 1500), ds1);
+ MockContent fileContent =
+ prepareContentProvider(fileProvider, "ds1", ":0:0", "a/:0:0", "a/b:34:9");
+ prepareContentProvider(pathInfoProvider, "ds1", new RuntimeException("Oohps!"));
+
+ task.execute();
+
+ assertEquals("ERROR OPERATION.DataSetAndPathInfoDBConsistencyChecker - "
+ + "Couldn't check consistency of the file system and "
+ + "the path info database for a data set: ds1\n"
+ + "java.lang.RuntimeException: Oohps!\n"
+ + "ERROR NOTIFY.DataSetAndPathInfoDBConsistencyCheckTask - "
+ + "File system and path info DB consistency check report "
+ + "for all data sets since 1970-01-01 01:00:00\n\n"
+ + "Error when checking datasets:\n\n"
+ + "ERROR: \"Couldn't check consistency of the file system and "
+ + "the path info database for a data set: ds1 "
+ + "because of the following exception: Oohps!\"",
+ LogUtils.removeEmbeddedStackTrace(logRecorder.getLogContent()));
+ assertEquals(true, fileContent.isClosed());
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testInconstitentCases()
+ {
+ PhysicalDataSet ds1 = createDataSetBuilder().code("ds1").getDataSet();
+ PhysicalDataSet ds2 = createDataSetBuilder().code("ds2").getDataSet();
+ PhysicalDataSet ds3 = createDataSetBuilder().code("ds3").getDataSet();
+ PhysicalDataSet ds4 = createDataSetBuilder().code("ds4").getDataSet();
+ prepareListDataSets(new Date(2010 - 1500), ds1, ds2, ds3, ds4);
+ MockContent fileContent1 =
+ prepareContentProvider(fileProvider, "ds1", ":0:0", "a/:0:0", "a/b:34:9",
+ "a/c:35:2", "b:33:3", "c/:0:0");
+ MockContent pathInfoContent1 =
+ prepareContentProvider(pathInfoProvider, "ds1", ":0:0", "a/:0:0", "a/b:34:7",
+ "a/c:42:2", "c:2:4");
+ MockContent fileContent2 = prepareContentProvider(fileProvider, "ds2");
+ MockContent pathInfoContent2 = prepareContentProvider(pathInfoProvider, "ds2", ":0:0");
+ MockContent fileContent3 =
+ prepareContentProvider(fileProvider, "ds3", ":0:0", "a/:0:0", "a/b:34:7",
+ "a/c:42:2", "c:2:4");
+ MockContent pathInfoContent3 =
+ prepareContentProvider(pathInfoProvider, "ds3", ":0:0", "a/:0:0", "a/b:34:9",
+ "a/c:35:2", "b:33:3", "c/:0:0");
+ MockContent fileContent4 = prepareContentProvider(fileProvider, "ds4", ":0:0");
+ MockContent pathInfoContent4 = prepareContentProvider(pathInfoProvider, "ds4");
+
+ task.execute();
+
+ assertEquals(
+ "ERROR NOTIFY.DataSetAndPathInfoDBConsistencyCheckTask - "
+ + "File system and path info DB consistency check report "
+ + "for all data sets since 1970-01-01 01:00:00\n\n"
+ + "Data sets checked:\n\nds1, ds2, ds3, ds4\n\n"
+ + "Differences found:\n\n"
+ + "Data set ds1:\n"
+ + "- 'a/b' CRC32 checksum in the file system = 00000009 but in the path info database = 00000007\n"
+ + "- 'a/c' size in the file system = 35 bytes but in the path info database = 42 bytes.\n"
+ + "- 'b' is on the file system but is not referenced in the path info database\n"
+ + "- 'c' is a directory in the file system but a file in the path info database\n\n"
+ + "Data set ds2:\n"
+ + "- exists in the path info database but does not exist in the file system\n\n"
+ + "Data set ds3:\n"
+ + "- 'a/b' CRC32 checksum in the file system = 00000007 but in the path info database = 00000009\n"
+ + "- 'a/c' size in the file system = 42 bytes but in the path info database = 35 bytes.\n"
+ + "- 'b' is referenced in the path info database but does not exist on the file system\n"
+ + "- 'c' is a directory in the path info database but a file in the file system\n\n"
+ + "Data set ds4:\n"
+ + "- exists in the file system but does not exist in the path info database",
+ logRecorder.getLogContent());
+ assertEquals(true, fileContent1.isClosed());
+ assertEquals(true, pathInfoContent1.isClosed());
+ assertEquals(true, fileContent2.isClosed());
+ assertEquals(true, pathInfoContent2.isClosed());
+ assertEquals(true, fileContent3.isClosed());
+ assertEquals(true, pathInfoContent3.isClosed());
+ assertEquals(true, fileContent4.isClosed());
+ assertEquals(true, pathInfoContent4.isClosed());
+ context.assertIsSatisfied();
+ }
+
+ private void prepareContentProvider(final IHierarchicalContentProvider provider,
+ final String dataSetCode, final Exception exception)
+ {
+ context.checking(new Expectations()
+ {
+ {
+ one(provider).asContent(dataSetCode);
+ will(throwException(exception));
+ }
+ });
+ }
+
+ private MockContent prepareContentProvider(final IHierarchicalContentProvider provider,
+ final String dataSetCode, final String... nodeDescriptions)
+ {
+ final MockContent content = new MockContent(nodeDescriptions);
+ context.checking(new Expectations()
+ {
+ {
+ one(provider).asContent(dataSetCode);
+ will(returnValue(content));
+ }
+ });
+ return content;
+ }
+
+ private void prepareListDataSets(final Date youngerThanDate, final PhysicalDataSet... dataSets)
+ {
+ final List<SimpleDataSetInformationDTO> translatedDataSets =
+ new ArrayList<SimpleDataSetInformationDTO>();
+ for (PhysicalDataSet physicalDataSet : dataSets)
+ {
+ translatedDataSets.add(SimpleDataSetHelper.translate(DataSetTranslator
+ .translateToDescription(physicalDataSet)));
+ }
+ context.checking(new Expectations()
+ {
+ {
+ one(service).listOldestPhysicalDataSets(youngerThanDate, Integer.MAX_VALUE);
+ will(returnValue(translatedDataSets));
+ }
+ });
+ }
+
+ private DataSetBuilder createDataSetBuilder()
+ {
+ return new DataSetBuilder().store(new DataStoreBuilder("DSS").getStore()).fileFormat("XML");
+ }
+
+}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetAndPathInfoDBConsistencyCheckProcessingPluginTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetAndPathInfoDBConsistencyCheckProcessingPluginTest.java
index bb5e9734a5fcce48e2d4234354ed83194342d68b..f6290b3133522685ce554d767c8dd0b0e5988951 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetAndPathInfoDBConsistencyCheckProcessingPluginTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetAndPathInfoDBConsistencyCheckProcessingPluginTest.java
@@ -21,7 +21,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.List;
-import java.util.Properties;
import org.jmock.Expectations;
import org.jmock.Mockery;
@@ -152,8 +151,8 @@ public class DataSetAndPathInfoDBConsistencyCheckProcessingPluginTest extends
pathInfoChildNode = context.mock(IHierarchicalContentNode.class, "pathInfoChildNode");
plugin =
- new DataSetAndPathInfoDBConsistencyCheckProcessingPlugin(new Properties(),
- workingDirectory, fileProvider, pathInfoProvider);
+ new DataSetAndPathInfoDBConsistencyCheckProcessingPlugin(fileProvider,
+ pathInfoProvider);
processingContext =
new DataSetProcessingContext(null, new MockDataSetDirectoryProvider(
workingDirectory, SHARE_ID), null, mailClient, USER_ID, USER_EMAIL);
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/MockContent.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/MockContent.java
new file mode 100644
index 0000000000000000000000000000000000000000..8cf8fdc8f281fa2b2cf9e56dc2d92430cc9a1ed9
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/MockContent.java
@@ -0,0 +1,124 @@
+/*
+ * Copyright 2013 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContent;
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContentNode;
+
+/**
+ * Mock content based on an array of textual content descriptions of the form:
+ * <path>:<size>:<checksum>. An empty string for <path> denotes root node. Directory paths end with
+ * '/'.
+ *
+ * @author Franz-Josef Elmer
+ */
+public class MockContent implements IHierarchicalContent
+{
+ private MockNode root;
+
+ private final Map<String, MockNode> nodes = new HashMap<String, MockNode>();
+
+ private boolean closed;
+
+ public MockContent(String... contentDescriptions)
+ {
+ for (String contentDescription : contentDescriptions)
+ {
+ String[] splittedDescription = contentDescription.split(":");
+ MockNode node = new MockNode();
+ String path = splittedDescription[0];
+ if (path.length() == 0)
+ {
+ root = node;
+ node.directory = true;
+ node.name = "";
+ node.relativePath = "";
+ } else
+ {
+ if (path.endsWith("/"))
+ {
+ path = path.substring(0, path.length() - 1);
+ node.directory = true;
+ } else
+ {
+ node.directory = false;
+ }
+ node.relativePath = path;
+ int lastIndexOfDelim = path.lastIndexOf('/');
+ MockNode parent = root;
+ if (lastIndexOfDelim >= 0)
+ {
+ String parentPath = path.substring(0, lastIndexOfDelim);
+ parent = nodes.get(parentPath);
+ }
+ parent.addNode(node);
+ nodes.put(path, node);
+ node.name = path.substring(lastIndexOfDelim + 1);
+ }
+ node.size = Long.parseLong(splittedDescription[1]);
+ node.checksum = (int) Long.parseLong(splittedDescription[2], 16);
+ }
+ }
+
+ @Override
+ public IHierarchicalContentNode getRootNode()
+ {
+ return root;
+ }
+
+ @Override
+ public IHierarchicalContentNode getNode(String relativePath)
+ throws IllegalArgumentException
+ {
+ return nodes.get(relativePath);
+ }
+
+ @Override
+ public IHierarchicalContentNode tryGetNode(String relativePath)
+ {
+ return nodes.get(relativePath);
+ }
+
+ @Override
+ public List<IHierarchicalContentNode> listMatchingNodes(String relativePathPattern)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<IHierarchicalContentNode> listMatchingNodes(String startingPath,
+ String fileNamePattern)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void close()
+ {
+ closed = true;
+ }
+
+ public boolean isClosed()
+ {
+ return closed;
+ }
+
+}
\ No newline at end of file
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/MockNode.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/MockNode.java
new file mode 100644
index 0000000000000000000000000000000000000000..227d1cdfbbd10d619cedd8f10f8f4df1c6291e1c
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/MockNode.java
@@ -0,0 +1,137 @@
+/*
+ * Copyright 2013 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard;
+
+import java.io.File;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+import ch.systemsx.cisd.base.exceptions.IOExceptionUnchecked;
+import ch.systemsx.cisd.base.io.IRandomAccessFile;
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContentNode;
+
+final class MockNode implements IHierarchicalContentNode
+{
+ private final List<IHierarchicalContentNode> children =
+ new ArrayList<IHierarchicalContentNode>();
+
+ String name;
+
+ String relativePath;
+
+ private IHierarchicalContentNode parent;
+
+ boolean directory;
+
+ long size;
+
+ int checksum;
+
+ void addNode(MockNode node)
+ {
+ node.parent = this;
+ children.add(node);
+ }
+
+ @Override
+ public String getName()
+ {
+ return name;
+ }
+
+ @Override
+ public String getRelativePath()
+ {
+ return relativePath;
+ }
+
+ @Override
+ public String getParentRelativePath()
+ {
+ return parent == null ? null : parent.getRelativePath();
+ }
+
+ @Override
+ public boolean exists()
+ {
+ return true;
+ }
+
+ @Override
+ public boolean isDirectory()
+ {
+ return directory;
+ }
+
+ @Override
+ public long getLastModified()
+ {
+ return 0;
+ }
+
+ @Override
+ public List<IHierarchicalContentNode> getChildNodes() throws UnsupportedOperationException
+ {
+ return children;
+ }
+
+ @Override
+ public File getFile() throws UnsupportedOperationException
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public File tryGetFile()
+ {
+ return null;
+ }
+
+ @Override
+ public long getFileLength() throws UnsupportedOperationException
+ {
+ return size;
+ }
+
+ @Override
+ public int getChecksumCRC32() throws UnsupportedOperationException
+ {
+ return checksum;
+ }
+
+ @Override
+ public boolean isChecksumCRC32Precalculated()
+ {
+ return true;
+ }
+
+ @Override
+ public IRandomAccessFile getFileContent() throws UnsupportedOperationException,
+ IOExceptionUnchecked
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public InputStream getInputStream() throws UnsupportedOperationException,
+ IOExceptionUnchecked
+ {
+ throw new UnsupportedOperationException();
+ }
+
+}
\ No newline at end of file
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiverTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiverTest.java
index 6c49eb627260fb03c925c90048a12695de509acb..769ac1841dbe7b58dd468447282cd6f7cf887122 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiverTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiverTest.java
@@ -19,13 +19,9 @@ package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard;
import static ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.AbstractArchiverProcessingPlugin.SHARE_FINDER_KEY;
import java.io.File;
-import java.io.InputStream;
import java.lang.reflect.Method;
-import java.util.ArrayList;
import java.util.Arrays;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.Properties;
import org.apache.log4j.Level;
@@ -39,15 +35,12 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import ch.rinn.restrictions.Friend;
-import ch.systemsx.cisd.base.exceptions.IOExceptionUnchecked;
-import ch.systemsx.cisd.base.io.IRandomAccessFile;
import ch.systemsx.cisd.base.tests.AbstractFileSystemTestCase;
import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.filesystem.BooleanStatus;
import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.common.logging.BufferedAppender;
import ch.systemsx.cisd.common.logging.LogInitializer;
-import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContent;
import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContentNode;
import ch.systemsx.cisd.openbis.dss.generic.shared.ArchiverTaskContext;
import ch.systemsx.cisd.openbis.dss.generic.shared.IConfigProvider;
@@ -556,200 +549,4 @@ public class RsyncArchiverTest extends AbstractFileSystemTestCase
RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2,
RsyncArchiver.ChecksumVerificationCondition.NO).toString());
}
-
- private static final class MockNode implements IHierarchicalContentNode
- {
- private final List<IHierarchicalContentNode> children =
- new ArrayList<IHierarchicalContentNode>();
-
- private String name;
-
- private String relativePath;
-
- private IHierarchicalContentNode parent;
-
- private boolean directory;
-
- private long size;
-
- private int checksum;
-
- void addNode(MockNode node)
- {
- node.parent = this;
- children.add(node);
- }
-
- @Override
- public String getName()
- {
- return name;
- }
-
- @Override
- public String getRelativePath()
- {
- return relativePath;
- }
-
- @Override
- public String getParentRelativePath()
- {
- return parent == null ? null : parent.getRelativePath();
- }
-
- @Override
- public boolean exists()
- {
- return true;
- }
-
- @Override
- public boolean isDirectory()
- {
- return directory;
- }
-
- @Override
- public long getLastModified()
- {
- return 0;
- }
-
- @Override
- public List<IHierarchicalContentNode> getChildNodes() throws UnsupportedOperationException
- {
- return children;
- }
-
- @Override
- public File getFile() throws UnsupportedOperationException
- {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public File tryGetFile()
- {
- return null;
- }
-
- @Override
- public long getFileLength() throws UnsupportedOperationException
- {
- return size;
- }
-
- @Override
- public int getChecksumCRC32() throws UnsupportedOperationException
- {
- return checksum;
- }
-
- @Override
- public boolean isChecksumCRC32Precalculated()
- {
- return true;
- }
-
- @Override
- public IRandomAccessFile getFileContent() throws UnsupportedOperationException,
- IOExceptionUnchecked
- {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public InputStream getInputStream() throws UnsupportedOperationException,
- IOExceptionUnchecked
- {
- throw new UnsupportedOperationException();
- }
-
- }
-
- private static final class MockContent implements IHierarchicalContent
- {
- private MockNode root;
-
- private final Map<String, MockNode> nodes = new HashMap<String, MockNode>();
-
- MockContent(String... contentDescriptions)
- {
- for (String contentDescription : contentDescriptions)
- {
- String[] splittedDescription = contentDescription.split(":");
- MockNode node = new MockNode();
- String path = splittedDescription[0];
- if (path.length() == 0)
- {
- root = node;
- node.directory = true;
- node.name = "";
- node.relativePath = "";
- } else
- {
- if (path.endsWith("/"))
- {
- path = path.substring(0, path.length() - 1);
- node.directory = true;
- } else
- {
- node.directory = false;
- }
- node.relativePath = path;
- int lastIndexOfDelim = path.lastIndexOf('/');
- MockNode parent = root;
- if (lastIndexOfDelim >= 0)
- {
- String parentPath = path.substring(0, lastIndexOfDelim);
- parent = nodes.get(parentPath);
- }
- parent.addNode(node);
- nodes.put(path, node);
- node.name = path.substring(lastIndexOfDelim + 1);
- }
- node.size = Long.parseLong(splittedDescription[1]);
- node.checksum = (int) Long.parseLong(splittedDescription[2], 16);
- }
- }
-
- @Override
- public IHierarchicalContentNode getRootNode()
- {
- return root;
- }
-
- @Override
- public IHierarchicalContentNode getNode(String relativePath)
- throws IllegalArgumentException
- {
- return nodes.get(relativePath);
- }
-
- @Override
- public IHierarchicalContentNode tryGetNode(String relativePath)
- {
- return nodes.get(relativePath);
- }
-
- @Override
- public List<IHierarchicalContentNode> listMatchingNodes(String relativePathPattern)
- {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public List<IHierarchicalContentNode> listMatchingNodes(String startingPath,
- String fileNamePattern)
- {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void close()
- {
- }
-
- }
}