From 3fbf856f3d4fff25a70cd351c7e5cbc20e4e58d2 Mon Sep 17 00:00:00 2001
From: pkupczyk <pkupczyk>
Date: Mon, 30 Jul 2012 08:53:15 +0000
Subject: [PATCH] SP-154 / BIS-38 - Improve checksum check for share moving
SVN: 26274
---
.../cisd/etlserver/plugins/DataSetMover.java | 12 +-
.../cisd/etlserver/plugins/IDataSetMover.java | 21 ++--
.../postregistration/EagerShufflingTask.java | 80 +++++++++----
.../HierarchicalContentChecksumProvider.java | 60 ++++++++++
.../dss/generic/shared/IChecksumProvider.java | 31 +++++
.../shared/utils/SegmentedStoreUtils.java | 53 +++++---
.../plugins/SimpleShufflingTest.java | 6 +-
.../EagerShufflingTaskTest.java | 113 +++++++++++-------
.../shared/utils/SegmentedStoreUtilsTest.java | 61 ++++++++--
9 files changed, 334 insertions(+), 103 deletions(-)
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/HierarchicalContentChecksumProvider.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IChecksumProvider.java
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/DataSetMover.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/DataSetMover.java
index 3a0b5f284dc..31be0c2a4c0 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/DataSetMover.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/DataSetMover.java
@@ -3,20 +3,22 @@ package ch.systemsx.cisd.etlserver.plugins;
import java.io.File;
import ch.systemsx.cisd.common.logging.ISimpleLogger;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IChecksumProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
import ch.systemsx.cisd.openbis.dss.generic.shared.IShareIdManager;
import ch.systemsx.cisd.openbis.dss.generic.shared.utils.SegmentedStoreUtils;
/**
* Implementation of {@link IDataSetMover}.
- *
+ *
* @author Franz-Josef Elmer
*/
public class DataSetMover implements IDataSetMover
{
private final IEncapsulatedOpenBISService service;
+
private final IShareIdManager manager;
-
+
public DataSetMover(IEncapsulatedOpenBISService service, IShareIdManager shareIdManager)
{
this.service = service;
@@ -25,9 +27,9 @@ public class DataSetMover implements IDataSetMover
@Override
public void moveDataSetToAnotherShare(File dataSetDirInStore, File share,
- ISimpleLogger logger)
+ IChecksumProvider checksumProvider, ISimpleLogger logger)
{
- SegmentedStoreUtils.moveDataSetToAnotherShare(dataSetDirInStore, share,
- service, manager, logger);
+ SegmentedStoreUtils.moveDataSetToAnotherShare(dataSetDirInStore, share, service, manager,
+ checksumProvider, logger);
}
}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/IDataSetMover.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/IDataSetMover.java
index 240dff83f48..ff24f2a3915 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/IDataSetMover.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/IDataSetMover.java
@@ -20,20 +20,25 @@ import java.io.File;
import java.util.Properties;
import ch.systemsx.cisd.common.logging.ISimpleLogger;
-
+import ch.systemsx.cisd.openbis.dss.generic.shared.IChecksumProvider;
/**
- * Strategy of moving a data set to another share. Implementations of this interface should
- * have a public constructor with an argument of type {@link Properties}.
- *
+ * Strategy of moving a data set to another share. Implementations of this interface should have a
+ * public constructor with an argument of type {@link Properties}.
+ *
* @author Franz-Josef Elmer
*/
public interface IDataSetMover
{
/**
- * Moves the specified data set to the specified share. The data set is folder in the store
- * its name is the data set code. The destination folder is <code>share</code>. Its name is
- * the share id. Share id and size will be updated on openBIS.
+ * Moves the specified data set to the specified share. The data set is folder in the store its
+ * name is the data set code. The destination folder is <code>share</code>. Its name is the
+ * share id. Share id and size will be updated on openBIS.
+ *
+ * @param checksumProvider provides checksums of the source data set files that will be compared
+ * with calculated checksums of the destination data set files. If checksumProvider
+ * is null, then checksum verification is not performed.
*/
- public void moveDataSetToAnotherShare(File dataSetDirInStore, File share, ISimpleLogger logger);
+ public void moveDataSetToAnotherShare(File dataSetDirInStore, File share,
+ IChecksumProvider checksumProvider, ISimpleLogger logger);
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/EagerShufflingTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/EagerShufflingTask.java
index 8c51edf17e9..44d2b767f7f 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/EagerShufflingTask.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/EagerShufflingTask.java
@@ -40,6 +40,8 @@ import ch.systemsx.cisd.common.utilities.PropertyParametersUtil;
import ch.systemsx.cisd.common.utilities.PropertyUtils;
import ch.systemsx.cisd.etlserver.plugins.DataSetMover;
import ch.systemsx.cisd.etlserver.plugins.IDataSetMover;
+import ch.systemsx.cisd.openbis.dss.generic.shared.HierarchicalContentChecksumProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IChecksumProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.IConfigProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
import ch.systemsx.cisd.openbis.dss.generic.shared.IShareFinder;
@@ -52,21 +54,30 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
/**
* Post registration task which move the data set to share which has enough space.
- *
+ *
* @author Franz-Josef Elmer
*/
public class EagerShufflingTask extends AbstractPostRegistrationTask
{
- @Private public static final String SHARE_FINDER_KEY = "share-finder";
- @Private public static final String FREE_SPACE_LIMIT_KEY = "free-space-limit-in-MB-triggering-notification";
- @Private public static final String STOP_ON_NO_SHARE_FOUND_KEY = "stop-on-no-share-found";
+ @Private
+ public static final String SHARE_FINDER_KEY = "share-finder";
+
+ @Private
+ public static final String FREE_SPACE_LIMIT_KEY =
+ "free-space-limit-in-MB-triggering-notification";
+
+ @Private
+ public static final String STOP_ON_NO_SHARE_FOUND_KEY = "stop-on-no-share-found";
+
+ @Private
+ public static final String VERIFY_CHECKSUM_KEY = "verify-checksum";
private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
EagerShufflingTask.class);
private static final Logger notificationLog = LogFactory.getLogger(LogCategory.NOTIFY,
EagerShufflingTask.class);
-
+
private static SimpleDataSetInformationDTO findDataSet(List<Share> shares, String dataSetCode)
{
for (Share share : shares)
@@ -89,10 +100,12 @@ public class EagerShufflingTask extends AbstractPostRegistrationTask
private final IDataSetMover dataSetMover;
+ private final IChecksumProvider checksumProvider;
+
private final ISimpleLogger logger;
private final ISimpleLogger notifyer;
-
+
private final File storeRoot;
private final String dataStoreCode;
@@ -100,29 +113,36 @@ public class EagerShufflingTask extends AbstractPostRegistrationTask
private final Set<String> incomingShares;
private IShareFinder finder;
-
+
private long freeSpaceLimitTriggeringNotification;
-
+
private boolean stopOnNoShareFound;
-
+
+ private boolean verifyChecksum;
+
public EagerShufflingTask(Properties properties, IEncapsulatedOpenBISService service)
{
this(properties, IncomingShareIdProvider.getIdsOfIncomingShares(), service, ServiceProvider
.getShareIdManager(), new SimpleFreeSpaceProvider(), new DataSetMover(service,
ServiceProvider.getShareIdManager()), ServiceProvider.getConfigProvider(),
- new Log4jSimpleLogger(operationLog), new Log4jSimpleLogger(notificationLog));
+ new HierarchicalContentChecksumProvider(
+ ServiceProvider.getHierarchicalContentProvider()), new Log4jSimpleLogger(
+ operationLog), new Log4jSimpleLogger(notificationLog));
}
- @Private public EagerShufflingTask(Properties properties, Set<String> incomingShares,
+ @Private
+ public EagerShufflingTask(Properties properties, Set<String> incomingShares,
IEncapsulatedOpenBISService service, IShareIdManager shareIdManager,
IFreeSpaceProvider freeSpaceProvider, IDataSetMover dataSetMover,
- IConfigProvider configProvider, ISimpleLogger logger, ISimpleLogger notifyer)
+ IConfigProvider configProvider, IChecksumProvider checksumProvider,
+ ISimpleLogger logger, ISimpleLogger notifyer)
{
super(properties, service);
this.incomingShares = incomingShares;
this.shareIdManager = shareIdManager;
this.freeSpaceProvider = freeSpaceProvider;
this.dataSetMover = dataSetMover;
+ this.checksumProvider = checksumProvider;
this.logger = logger;
this.notifyer = notifyer;
@@ -140,7 +160,9 @@ public class EagerShufflingTask extends AbstractPostRegistrationTask
finder = ClassUtils.create(IShareFinder.class, props.getProperty("class"), props);
freeSpaceLimitTriggeringNotification =
FileUtils.ONE_MB * PropertyUtils.getInt(properties, FREE_SPACE_LIMIT_KEY, 0);
- stopOnNoShareFound = PropertyUtils.getBoolean(properties, STOP_ON_NO_SHARE_FOUND_KEY, false);
+ stopOnNoShareFound =
+ PropertyUtils.getBoolean(properties, STOP_ON_NO_SHARE_FOUND_KEY, false);
+ verifyChecksum = PropertyUtils.getBoolean(properties, VERIFY_CHECKSUM_KEY, true);
}
@Override
@@ -149,6 +171,17 @@ public class EagerShufflingTask extends AbstractPostRegistrationTask
return true;
}
+ private IChecksumProvider getChecksumProvider()
+ {
+ if (verifyChecksum)
+ {
+ return checksumProvider;
+ } else
+ {
+ return null;
+ }
+ }
+
@Override
public IPostRegistrationTaskExecutor createExecutor(String dataSetCode, boolean container)
{
@@ -158,14 +191,15 @@ public class EagerShufflingTask extends AbstractPostRegistrationTask
}
return new Executor(dataSetCode);
}
-
+
private final class Executor implements IPostRegistrationTaskExecutor
{
private final String dataSetCode;
private SimpleDataSetInformationDTO dataSet;
+
private Share shareWithMostFreeOrNull;
-
+
Executor(String dataSetCode)
{
this.dataSetCode = dataSetCode;
@@ -175,8 +209,8 @@ public class EagerShufflingTask extends AbstractPostRegistrationTask
public ICleanupTask createCleanupTask()
{
List<Share> shares =
- SegmentedStoreUtils.getDataSetsPerShare(storeRoot, dataStoreCode, incomingShares,
- freeSpaceProvider, service, logger);
+ SegmentedStoreUtils.getDataSetsPerShare(storeRoot, dataStoreCode,
+ incomingShares, freeSpaceProvider, service, logger);
dataSet = findDataSet(shares, dataSetCode);
shareWithMostFreeOrNull = finder.tryToFindShare(dataSet, shares);
if (shareWithMostFreeOrNull == null)
@@ -192,7 +226,7 @@ public class EagerShufflingTask extends AbstractPostRegistrationTask
}
return new CleanupTask(dataSet, storeRoot, shareWithMostFreeOrNull.getShareId());
}
-
+
@Override
public void execute()
{
@@ -202,7 +236,8 @@ public class EagerShufflingTask extends AbstractPostRegistrationTask
File share = new File(storeRoot, shareIdManager.getShareId(dataSetCode));
dataSetMover.moveDataSetToAnotherShare(
new File(share, dataSet.getDataSetLocation()),
- shareWithMostFreeOrNull.getShare(), logger);
+ shareWithMostFreeOrNull.getShare(), getChecksumProvider(), logger);
+
String shareId = shareWithMostFreeOrNull.getShareId();
logger.log(LogLevel.INFO, "Data set " + dataSetCode
+ " successfully moved from share " + dataSet.getDataSetShareId() + " to "
@@ -221,16 +256,17 @@ public class EagerShufflingTask extends AbstractPostRegistrationTask
}
}
}
-
-
+
private static final class CleanupTask implements ICleanupTask
{
private static final long serialVersionUID = 1L;
private final SimpleDataSetInformationDTO dataSet;
+
private final File storeRoot;
+
private final String newShareId;
-
+
CleanupTask(SimpleDataSetInformationDTO dataSet, File storeRoot, String newShareId)
{
this.dataSet = dataSet;
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/HierarchicalContentChecksumProvider.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/HierarchicalContentChecksumProvider.java
new file mode 100644
index 00000000000..3b75976744e
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/HierarchicalContentChecksumProvider.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2012 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.shared;
+
+import java.io.IOException;
+
+import ch.systemsx.cisd.common.io.hierarchical_content.api.IHierarchicalContent;
+import ch.systemsx.cisd.common.io.hierarchical_content.api.IHierarchicalContentNode;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IChecksumProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
+
+/**
+ * Checksum provider that uses a hierarchical content provider to get checksums.
+ *
+ * @author pkupczyk
+ */
+public class HierarchicalContentChecksumProvider implements IChecksumProvider
+{
+
+ private IHierarchicalContentProvider hierarchicalContentProvider;
+
+ public HierarchicalContentChecksumProvider(
+ IHierarchicalContentProvider hierarchicalContentProvider)
+ {
+ this.hierarchicalContentProvider = hierarchicalContentProvider;
+ }
+
+ @Override
+ public long getChecksum(String dataSetCode, String relativePath) throws IOException
+ {
+ IHierarchicalContent content = null;
+
+ try
+ {
+ content = hierarchicalContentProvider.asContent(dataSetCode);
+ IHierarchicalContentNode node = content.getNode(relativePath);
+ return node.getChecksumCRC32();
+ } finally
+ {
+ if (content != null)
+ {
+ content.close();
+ }
+ }
+ }
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IChecksumProvider.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IChecksumProvider.java
new file mode 100644
index 00000000000..c26b460a590
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IChecksumProvider.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2012 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.shared;
+
+import java.io.IOException;
+
+/**
+ * Provides checksums of data set files.
+ *
+ * @author pkupczyk
+ */
+public interface IChecksumProvider
+{
+
+ public long getChecksum(String dataSetCode, String relativePath) throws IOException;
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java
index e040a2c66ae..0f6e3d15031 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java
@@ -45,6 +45,7 @@ import ch.systemsx.cisd.common.logging.LogLevel;
import ch.systemsx.cisd.common.utilities.ITimeProvider;
import ch.systemsx.cisd.common.utilities.SystemTimeProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.IDataSetDirectoryProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IChecksumProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
import ch.systemsx.cisd.openbis.dss.generic.shared.IShareIdManager;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ExternalData;
@@ -270,10 +271,11 @@ public class SegmentedStoreUtils
* </ol>
*
* @param service to access openBIS AS.
+ * @param checksumProvider
*/
public static void moveDataSetToAnotherShare(final File dataSetDirInStore, File share,
IEncapsulatedOpenBISService service, final IShareIdManager shareIdManager,
- final ISimpleLogger logger)
+ IChecksumProvider checksumProvider, final ISimpleLogger logger)
{
final String dataSetCode = dataSetDirInStore.getName();
ExternalData dataSet = service.tryGetDataSet(dataSetCode);
@@ -295,7 +297,9 @@ public class SegmentedStoreUtils
File dataSetDirInNewShare = new File(share, relativePath);
dataSetDirInNewShare.mkdirs();
copyToShare(dataSetDirInStore, dataSetDirInNewShare, logger);
- long size = assertEqualSizeAndChildren(dataSetDirInStore, dataSetDirInNewShare);
+ long size =
+ assertEqualSizeAndChildren(dataSetCode, dataSetDirInStore, dataSetDirInStore,
+ dataSetDirInNewShare, dataSetDirInNewShare, checksumProvider);
String shareId = share.getName();
service.updateShareIdAndSize(dataSetCode, shareId, size);
shareIdManager.setShareId(dataSetCode, shareId);
@@ -398,13 +402,15 @@ public class SegmentedStoreUtils
share.getPath(), (System.currentTimeMillis() - start) / 1000.0));
}
- private static long assertEqualSizeAndChildren(File source, File destination)
+ private static long assertEqualSizeAndChildren(String dataSetCode, File sourceRoot,
+ File source, File destinationRoot, File destination, IChecksumProvider checksumProvider)
{
assertSameName(source, destination);
if (source.isFile())
{
assertFile(destination);
- return assertSameSizeAndCheckSum(source, destination);
+ return assertSameSizeAndCheckSum(dataSetCode, sourceRoot, source, destinationRoot,
+ destination, checksumProvider);
} else
{
assertDirectory(destination);
@@ -414,7 +420,9 @@ public class SegmentedStoreUtils
long sum = 0;
for (int i = 0; i < sourceFiles.length; i++)
{
- sum += assertEqualSizeAndChildren(sourceFiles[i], destinationFiles[i]);
+ sum +=
+ assertEqualSizeAndChildren(dataSetCode, sourceRoot, sourceFiles[i],
+ destinationRoot, destinationFiles[i], checksumProvider);
}
return sum;
}
@@ -432,7 +440,8 @@ public class SegmentedStoreUtils
}
}
- private static long assertSameSizeAndCheckSum(File source, File destination)
+ private static long assertSameSizeAndCheckSum(String dataSetCode, File sourceRoot, File source,
+ File destinationRoot, File destination, IChecksumProvider checksumProvider)
{
long sourceSize = source.length();
long destinationSize = destination.length();
@@ -443,23 +452,37 @@ public class SegmentedStoreUtils
+ " but source file '" + source.getAbsolutePath() + "' has size " + sourceSize
+ ".");
}
- long sourceChecksum = calculateCRC(source);
- long destinationChecksum = calculateCRC(destination);
- if (sourceChecksum != destinationChecksum)
+
+ if (checksumProvider != null)
{
- throw new EnvironmentFailureException("Destination file '"
- + destination.getAbsolutePath() + "' has checksum " + destinationChecksum
- + " but source file '" + source.getAbsolutePath() + "' has checksum "
- + sourceChecksum + ".");
+ try
+ {
+ long sourceChecksum =
+ checksumProvider.getChecksum(dataSetCode,
+ FileUtilities.getRelativeFilePath(sourceRoot, source));
+ long destinationChecksum = calculateCRC(destination);
+
+ if (sourceChecksum != destinationChecksum)
+ {
+ throw new EnvironmentFailureException("Destination file '"
+ + destination.getAbsolutePath() + "' has checksum "
+ + destinationChecksum + " but source file '" + source.getAbsolutePath()
+ + "' has checksum " + sourceChecksum + ".");
+ }
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
}
+
return sourceSize;
}
- private static long calculateCRC(File file)
+ private static int calculateCRC(File file)
{
try
{
- return FileUtils.checksumCRC32(file);
+ return (int) FileUtils.checksumCRC32(file);
} catch (IOException ex)
{
throw CheckedExceptionTunnel.wrapIfNecessary(ex);
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/SimpleShufflingTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/SimpleShufflingTest.java
index 55ef886ac3e..376eaf9bf3d 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/SimpleShufflingTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/SimpleShufflingTest.java
@@ -162,7 +162,7 @@ public class SimpleShufflingTest extends AbstractFileSystemTestCase
});
eagerShufflingTask =
new EagerShufflingTask(properties, new HashSet<String>(Arrays.asList("1", "2")),
- service, shareIdManager, spaceProvider, dataSetMover, configProvider,
+ service, shareIdManager, spaceProvider, dataSetMover, configProvider, null,
logger, notifyer);
balancer = new SimpleShuffling(properties, eagerShufflingTask);
}
@@ -238,7 +238,7 @@ public class SimpleShufflingTest extends AbstractFileSystemTestCase
one(dataSetMover).moveDataSetToAnotherShare(
new File(share1.getShare(), STORE_PATH + "ds3"), share3.getShare(),
- logger);
+ null, logger);
one(logger).log(LogLevel.INFO,
"Data set ds3 successfully moved from share 1 to 3.");
@@ -248,7 +248,7 @@ public class SimpleShufflingTest extends AbstractFileSystemTestCase
one(dataSetMover).moveDataSetToAnotherShare(
new File(share1.getShare(), STORE_PATH + "ds2"), share4.getShare(),
- logger);
+ null, logger);
one(logger).log(LogLevel.INFO,
"Data set ds2 successfully moved from share 1 to 4.");
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/postregistration/EagerShufflingTaskTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/postregistration/EagerShufflingTaskTest.java
index bfca0ac9e37..d945feaaf3d 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/postregistration/EagerShufflingTaskTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/postregistration/EagerShufflingTaskTest.java
@@ -44,6 +44,7 @@ import ch.systemsx.cisd.common.logging.ISimpleLogger;
import ch.systemsx.cisd.common.logging.LogLevel;
import ch.systemsx.cisd.common.test.RecordingMatcher;
import ch.systemsx.cisd.etlserver.plugins.IDataSetMover;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IChecksumProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.IConfigProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
import ch.systemsx.cisd.openbis.dss.generic.shared.IShareIdManager;
@@ -51,19 +52,19 @@ import ch.systemsx.cisd.openbis.generic.shared.Constants;
import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
/**
- *
- *
* @author Franz-Josef Elmer
*/
-@Friend(toClasses=EagerShufflingTask.class)
+@Friend(toClasses = EagerShufflingTask.class)
public class EagerShufflingTaskTest extends AbstractFileSystemTestCase
{
private static final class MockFreeSpaceProvider implements IFreeSpaceProvider
{
private final List<String> shares = new ArrayList<String>();
+
private Integer[] freeSpaceValues;
+
private int index;
-
+
void setFreeSpaceValues(Integer... freeSpaceValues)
{
this.freeSpaceValues = freeSpaceValues;
@@ -76,26 +77,43 @@ public class EagerShufflingTaskTest extends AbstractFileSystemTestCase
return freeSpaceValues[index++ % freeSpaceValues.length];
}
}
-
+
private static final String SHARDING = "sharding/";
+
private static final String DATA_STORE_SERVER_CODE = "DSS";
+
private static final String DATA_SET_CODE1 = "ds-1";
+
private BufferedAppender logRecorder;
+
private Mockery context;
private IEncapsulatedOpenBISService service;
+
private IShareIdManager shareIdManager;
+
private MockFreeSpaceProvider freeSpaceProvider;
+
private IDataSetMover dataSetMover;
+ private IChecksumProvider checksumProvider;
+
private IConfigProvider configProvider;
+
private ISimpleLogger logger;
+
private ISimpleLogger notifyer;
+
private File store;
+
private File share1;
+
private File share2;
+
private File share3;
+
private File share4;
+
private File ds1File;
@BeforeMethod
@@ -108,6 +126,7 @@ public class EagerShufflingTaskTest extends AbstractFileSystemTestCase
freeSpaceProvider = new MockFreeSpaceProvider();
configProvider = context.mock(IConfigProvider.class);
dataSetMover = context.mock(IDataSetMover.class);
+ checksumProvider = context.mock(IChecksumProvider.class);
logger = context.mock(ISimpleLogger.class, "logger");
notifyer = context.mock(ISimpleLogger.class, "notifyer");
store = new File(workingDirectory.getAbsolutePath(), "store");
@@ -124,7 +143,7 @@ public class EagerShufflingTaskTest extends AbstractFileSystemTestCase
share4 = new File(store, "4");
share4.mkdir();
}
-
+
@AfterMethod(alwaysRun = true)
public void afterMethod()
{
@@ -138,7 +157,9 @@ public class EagerShufflingTaskTest extends AbstractFileSystemTestCase
public void testShufflingIntoAnExtensionShare()
{
prepareConfigProvider();
- EagerShufflingTask task = createTask();
+ Properties properties = createDefaultProperties();
+ properties.setProperty(EagerShufflingTask.VERIFY_CHECKSUM_KEY, "true");
+ EagerShufflingTask task = createTask(properties);
freeSpaceProvider.setFreeSpaceValues(200, 100, 300, 400, 400, 300);
prepareListDataSets();
prepareGetShareId();
@@ -146,14 +167,15 @@ public class EagerShufflingTaskTest extends AbstractFileSystemTestCase
context.checking(new Expectations()
{
{
- one(dataSetMover).moveDataSetToAnotherShare(ds1File, share4, logger);
+ one(dataSetMover).moveDataSetToAnotherShare(ds1File, share4, checksumProvider,
+ logger);
}
});
-
+
IPostRegistrationTaskExecutor executor = task.createExecutor(DATA_SET_CODE1, false);
executor.createCleanupTask();
executor.execute();
-
+
assertEquals("Data set ds-1 successfully moved from share 1 to 4.",
infoMessageMatcher.recordedObject());
assertEquals("[1, 2, 3, 4, 4, 4]", freeSpaceProvider.shares.toString());
@@ -172,7 +194,8 @@ public class EagerShufflingTaskTest extends AbstractFileSystemTestCase
context.checking(new Expectations()
{
{
- one(dataSetMover).moveDataSetToAnotherShare(ds1File, share2, logger);
+ one(dataSetMover).moveDataSetToAnotherShare(ds1File, share2, checksumProvider,
+ logger);
}
});
@@ -185,7 +208,7 @@ public class EagerShufflingTaskTest extends AbstractFileSystemTestCase
assertEquals("[1, 2, 3, 4, 2, 2]", freeSpaceProvider.shares.toString());
context.assertIsSatisfied();
}
-
+
@Test
public void testShufflingButNoShareFoundExceptTheOwnOne()
{
@@ -195,23 +218,24 @@ public class EagerShufflingTaskTest extends AbstractFileSystemTestCase
prepareListDataSets();
RecordingMatcher<String> logMessageMatcher = prepareLogging(LogLevel.WARN);
-
+
IPostRegistrationTaskExecutor executor = task.createExecutor(DATA_SET_CODE1, false);
executor.createCleanupTask();
executor.execute();
-
+
assertEquals("No share found for shuffling data set ds-1.",
logMessageMatcher.recordedObject());
assertEquals("[1, 2, 3, 4, 1, 2, 3, 4]", freeSpaceProvider.shares.toString());
context.assertIsSatisfied();
}
-
+
@Test
public void testShufflingWithNotification()
{
prepareConfigProvider();
Properties properties = createDefaultProperties();
properties.setProperty(EagerShufflingTask.FREE_SPACE_LIMIT_KEY, "1");
+ properties.setProperty(EagerShufflingTask.VERIFY_CHECKSUM_KEY, "false");
EagerShufflingTask task = createTask(properties);
freeSpaceProvider.setFreeSpaceValues(200, 1234, 10, 0, 1234, 900);
prepareListDataSets();
@@ -221,7 +245,7 @@ public class EagerShufflingTaskTest extends AbstractFileSystemTestCase
context.checking(new Expectations()
{
{
- one(dataSetMover).moveDataSetToAnotherShare(ds1File, share2, logger);
+ one(dataSetMover).moveDataSetToAnotherShare(ds1File, share2, null, logger);
one(notifyer).log(with(LogLevel.WARN), with(notificationRecorder));
}
});
@@ -232,8 +256,10 @@ public class EagerShufflingTaskTest extends AbstractFileSystemTestCase
assertEquals("Data set ds-1 successfully moved from share 1 to 2.",
infoMessageMatcher.recordedObject());
- assertEquals("After moving data set ds-1 to share 2 that share has only 900.00 KB free space. " +
- "It might be necessary to add a new share.", notificationRecorder.recordedObject());
+ assertEquals(
+ "After moving data set ds-1 to share 2 that share has only 900.00 KB free space. "
+ + "It might be necessary to add a new share.",
+ notificationRecorder.recordedObject());
assertEquals("[1, 2, 3, 4, 2, 2]", freeSpaceProvider.shares.toString());
context.assertIsSatisfied();
}
@@ -264,37 +290,38 @@ public class EagerShufflingTaskTest extends AbstractFileSystemTestCase
{
assertEquals("No share found for shuffling data set ds-1.", ex.getMessage());
}
-
- assertEquals("No share found for shuffling data set ds-1.", notificationRecorder.recordedObject());
+
+ assertEquals("No share found for shuffling data set ds-1.",
+ notificationRecorder.recordedObject());
assertEquals("[1, 2, 3, 4, 1, 2, 3, 4]", freeSpaceProvider.shares.toString());
context.assertIsSatisfied();
}
-
+
private RecordingMatcher<String> prepareLogging(final LogLevel level)
{
final RecordingMatcher<String> logMessageMatcher = new RecordingMatcher<String>();
context.checking(new Expectations()
- {
{
- one(logger).log(with(level), with(logMessageMatcher));
- }
- });
+ {
+ one(logger).log(with(level), with(logMessageMatcher));
+ }
+ });
return logMessageMatcher;
}
-
+
private void prepareListDataSets()
{
context.checking(new Expectations()
- {
{
- one(service).listDataSets();
- will(returnValue(Arrays.asList(dataSet("1", DATA_SET_CODE1))));
- one(logger).log(
- with(LogLevel.INFO),
- with(Matchers.startsWith("Obtained the list of all "
- + "datasets in all shares")));
- }
- });
+ {
+ one(service).listDataSets();
+ will(returnValue(Arrays.asList(dataSet("1", DATA_SET_CODE1))));
+ one(logger).log(
+ with(LogLevel.INFO),
+ with(Matchers.startsWith("Obtained the list of all "
+ + "datasets in all shares")));
+ }
+ });
}
private void prepareConfigProvider()
@@ -313,14 +340,14 @@ public class EagerShufflingTaskTest extends AbstractFileSystemTestCase
private void prepareGetShareId()
{
context.checking(new Expectations()
- {
{
- one(shareIdManager).getShareId(DATA_SET_CODE1);
- will(returnValue("1"));
- }
- });
+ {
+ one(shareIdManager).getShareId(DATA_SET_CODE1);
+ will(returnValue("1"));
+ }
+ });
}
-
+
private SimpleDataSetInformationDTO dataSet(String shareId, String dataSetCode)
{
SimpleDataSetInformationDTO dataSet = new SimpleDataSetInformationDTO();
@@ -350,7 +377,7 @@ public class EagerShufflingTaskTest extends AbstractFileSystemTestCase
{
return new EagerShufflingTask(properties,
new LinkedHashSet<String>(Arrays.asList("1", "2")), service, shareIdManager,
- freeSpaceProvider, dataSetMover, configProvider, logger, notifyer);
+ freeSpaceProvider, dataSetMover, configProvider, checksumProvider, logger, notifyer);
}
-
+
}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtilsTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtilsTest.java
index eeca7ee9207..a2cb81661b7 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtilsTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtilsTest.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import org.apache.commons.io.FileUtils;
import org.jmock.Expectations;
import org.jmock.Mockery;
import org.testng.annotations.AfterMethod;
@@ -32,12 +33,14 @@ import org.testng.annotations.Test;
import ch.rinn.restrictions.Friend;
import ch.systemsx.cisd.base.tests.AbstractFileSystemTestCase;
import ch.systemsx.cisd.common.concurrent.MessageChannel;
+import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.common.filesystem.HostAwareFile;
import ch.systemsx.cisd.common.filesystem.IFreeSpaceProvider;
import ch.systemsx.cisd.common.logging.MockLogger;
import ch.systemsx.cisd.common.test.RecordingMatcher;
import ch.systemsx.cisd.common.utilities.ITimeProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IChecksumProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.IDataSetDirectoryProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
import ch.systemsx.cisd.openbis.dss.generic.shared.IShareIdManager;
@@ -54,11 +57,13 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
public class SegmentedStoreUtilsTest extends AbstractFileSystemTestCase
{
private static final String MOVED = "Moved";
+
private static final String LOCK = "lock";
+
private static final String START_DELETION = "Start deletion";
private static final String DATA_STORE_CODE = "ds-code";
-
+
private Mockery context;
private IEncapsulatedOpenBISService service;
@@ -75,6 +80,8 @@ public class SegmentedStoreUtilsTest extends AbstractFileSystemTestCase
private IDataSetDirectoryProvider dataSetDirectoryProvider;
+ private IChecksumProvider checksumProvider;
+
private File store;
@BeforeMethod
@@ -87,6 +94,7 @@ public class SegmentedStoreUtilsTest extends AbstractFileSystemTestCase
timeProvider = context.mock(ITimeProvider.class);
datasetLocation = context.mock(IDatasetLocation.class);
dataSetDirectoryProvider = context.mock(IDataSetDirectoryProvider.class);
+ checksumProvider = context.mock(IChecksumProvider.class);
context.checking(new Expectations()
{
@@ -195,23 +203,26 @@ public class SegmentedStoreUtilsTest extends AbstractFileSystemTestCase
}
@Test
- public void testMoveDataSetToAnothetShareAndDelete()
+ public void testMoveDataSetToAnothetShareAndDelete() throws IOException
{
File share1 = new File(workingDirectory, "store/1");
File share1uuid01 = new File(share1, "uuid/01");
File ds2 = new File(share1uuid01, "0b/0c/ds-2/original");
ds2.mkdirs();
- FileUtilities.writeToFile(new File(ds2, "read.me"), "do nothing");
+ final File readmeFile = new File(ds2, "read.me");
+ FileUtilities.writeToFile(readmeFile, "do nothing");
final File dataSetDirInStore = new File(share1uuid01, "02/03/ds-1");
File original = new File(dataSetDirInStore, "original");
original.mkdirs();
- FileUtilities.writeToFile(new File(original, "hello.txt"), "hello world");
+ final File helloFile = new File(original, "hello.txt");
+ FileUtilities.writeToFile(helloFile, "hello world");
final File share2 = new File(workingDirectory, "store/2");
share2.mkdirs();
File share2uuid01 = new File(share2, "uuid/01");
File file = new File(share2uuid01, "22/33/orig");
file.mkdirs();
- FileUtilities.writeToFile(new File(file, "hi.txt"), "hi");
+ final File hiFile = new File(file, "hi.txt");
+ FileUtilities.writeToFile(hiFile, "hi");
context.checking(new Expectations()
{
{
@@ -230,6 +241,9 @@ public class SegmentedStoreUtilsTest extends AbstractFileSystemTestCase
one(dataSetDirectoryProvider).getDataSetDirectory(datasetLocation);
will(returnValue(new File(
"targets/unit-test-wd/ch.systemsx.cisd.openbis.dss.generic.shared.utils.SegmentedStoreUtilsTest/store/2/uuid/01/02/03/ds-1")));
+
+ one(checksumProvider).getChecksum("ds-1", "original/hello.txt");
+ will(returnValue(FileUtils.checksumCRC32(helloFile)));
}
});
assertEquals(true, dataSetDirInStore.exists());
@@ -252,7 +266,7 @@ public class SegmentedStoreUtilsTest extends AbstractFileSystemTestCase
moveChannel.send(LOCK);
deletionChannel.assertNextMessage(START_DELETION);
}
- }, log);
+ }, checksumProvider, log);
moveChannel.send(MOVED);
}
}).start();
@@ -332,7 +346,7 @@ public class SegmentedStoreUtilsTest extends AbstractFileSystemTestCase
assertFileNames(share2uuid01, "22");
SegmentedStoreUtils.moveDataSetToAnotherShare(dataSetDirInStore, share2, service,
- shareIdManager, log);
+ shareIdManager, null, log);
log.assertNextLogMessage("Start moving directory 'targets/unit-test-wd/ch.systemsx.cisd."
+ "openbis.dss.generic.shared.utils.SegmentedStoreUtilsTest/store/1/uuid/01/02/03/ds-1' "
@@ -354,6 +368,38 @@ public class SegmentedStoreUtilsTest extends AbstractFileSystemTestCase
log.assertNoMoreLogMessages();
}
+ @Test(groups = "slow", expectedExceptions = EnvironmentFailureException.class)
+ public void testMoveDataSetToAnotherShareWithDifferentChecksums() throws IOException
+ {
+ File share1 = new File(workingDirectory, "store/1");
+ File share1uuid01 = new File(share1, "uuid/01");
+ File dataSetDirInStore = new File(share1uuid01, "02/03/ds-1");
+ File original = new File(dataSetDirInStore, "original");
+ original.mkdirs();
+ final File helloFile = new File(original, "hello.txt");
+ FileUtilities.writeToFile(helloFile, "hello world");
+ File share2 = new File(workingDirectory, "store/2");
+
+ context.checking(new Expectations()
+ {
+ {
+ one(service).tryGetDataSet("ds-1");
+ will(returnValue(new DataSet()));
+
+ one(shareIdManager).lock("ds-1");
+ one(shareIdManager).releaseLock("ds-1");
+
+ one(checksumProvider).getChecksum("ds-1", "original/hello.txt");
+ will(returnValue(1L));
+ }
+ });
+
+ SegmentedStoreUtils.moveDataSetToAnotherShare(dataSetDirInStore, share2, service,
+ shareIdManager, checksumProvider, log);
+
+ fail();
+ }
+
@Test
public void testCleanupOld()
{
@@ -461,4 +507,5 @@ public class SegmentedStoreUtilsTest extends AbstractFileSystemTestCase
dataSet.setDataSetSize(size);
return dataSet;
}
+
}
--
GitLab