diff --git a/datastore_server/resource/test-data/HierarchicalStorageUpdaterTest/.gitignore b/datastore_server/resource/test-data/HierarchicalStorageUpdaterTest/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/HierarchicalStorageUpdater.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/HierarchicalStorageUpdater.java
index 65683af9e02fae8406b8ec2b9db8cbb2d645c278..841594cee555164807df3ff777ed45bd876dc1d5 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/HierarchicalStorageUpdater.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/HierarchicalStorageUpdater.java
@@ -62,8 +62,8 @@ public class HierarchicalStorageUpdater implements IDataStoreLockingMaintenanceT
private static final String REBUILDING_HIERARCHICAL_STORAGE = "Rebuilding hierarchical storage";
- private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
- HierarchicalStorageUpdater.class);
+ private static final Logger operationLog =
+ LogFactory.getLogger(LogCategory.OPERATION, HierarchicalStorageUpdater.class);
private static class LinkSourceDescriptor
{
@@ -232,8 +232,7 @@ public class HierarchicalStorageUpdater implements IDataStoreLockingMaintenanceT
LinkSourceDescriptor linkSourceDescriptor = getLinkSourceDescriptor(dataSetType);
File source = dataSetLocationRoot;
- if (linkSourceDescriptor != null)
- {
+ if (linkSourceDescriptor != null) {
String subPath = linkSourceDescriptor.getSubFolder();
if (StringUtils.isBlank(subPath) == false)
{
@@ -316,7 +315,7 @@ public class HierarchicalStorageUpdater implements IDataStoreLockingMaintenanceT
{
toDelete = parent;
parent = toDelete.getParentFile();
- delete(toDelete);
+ toDelete.delete();
} else
{
break;
@@ -325,39 +324,40 @@ public class HierarchicalStorageUpdater implements IDataStoreLockingMaintenanceT
}
}
- private static void deleteWithSymbolicLinks(File toDeleteParent)
+ private static void deleteWithSymbolicLinks(File toDelete)
{
- if (toDeleteParent.isDirectory() == false)
+ if (FileUtilities.isSymbolicLink(toDelete))
{
- operationLog
- .error("Directory structure is different than expected. File '"
- + toDeleteParent.getPath()
- + "' should be a directory. It will not be cleared.");
+ toDelete.delete();
return;
}
- for (File file : toDeleteParent.listFiles())
+
+ if (toDelete.isDirectory() == false)
+ {
+ operationLog.error("Directory structure is different than expected. File '"
+ + toDelete.getPath() + "' should be a directory. It will not be cleared.");
+ return;
+ }
+
+ for (File file : toDelete.listFiles())
{
// all these files should be symbolic links to a dataset directory.
// We cannot delete recursively here, it would remove the original files.
- boolean ok = delete(file);
+ if (false == FileUtilities.isSymbolicLink(file))
+ {
+ operationLog.error(file.getPath()
+ + " is not a symbolic link and will not be deleted.");
+ return;
+ }
+ boolean ok = file.delete();
if (ok == false)
{
operationLog.error("Cannot delete the file: " + file.getPath());
}
}
- delete(toDeleteParent);
- }
- private static boolean delete(File file)
- {
- if (FileUtilities.isSymbolicLink(file) || file.isDirectory())
- {
- return file.delete();
- } else
- {
- throw new IllegalStateException("Illegal attemp to delete a regular file: "
- + file.getPath());
- }
+ // delete the folder in the end
+ toDelete.delete();
}
/**
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/TemplateBasedLinkNamingStrategy.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/TemplateBasedLinkNamingStrategy.java
index ce5c247f5d472c19f57a709a0aaa730a812bdfd4..06ce34041fcb6a794ed17e2afcc9f6dcb07cccf5 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/TemplateBasedLinkNamingStrategy.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/TemplateBasedLinkNamingStrategy.java
@@ -20,11 +20,11 @@ import java.io.File;
import java.util.HashSet;
import java.util.Properties;
import java.util.Set;
-import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
import ch.rinn.restrictions.Private;
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.common.utilities.ExtendedProperties;
import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
@@ -43,8 +43,6 @@ public class TemplateBasedLinkNamingStrategy implements IHierarchicalStorageLink
private static final String NOT_DIRECTLY_CONNECTED = "NOT_DIRECTLY_CONNECTED";
- private static final String MATCH_ALL_FILE_NAMES = "([^/]*)";
-
private final String linkTemplate;
@@ -94,23 +92,10 @@ public class TemplateBasedLinkNamingStrategy implements IHierarchicalStorageLink
public Set<String> extractPaths(File root)
{
HashSet<String> set = new HashSet<String>();
- Pattern matchingFilesFilter = createMatchingFilesFilter(root);
- accumulatePaths(set, root, matchingFilesFilter, getNestedDirectoryLevels());
+ accumulateSymLinkPaths(set, root);
return set;
}
- private Pattern createMatchingFilesFilter(File root)
- {
- ExtendedProperties props = new ExtendedProperties();
- for (PathVariable var : PathVariable.values())
- {
- props.put(var.name(), MATCH_ALL_FILE_NAMES);
- }
-
- String subPathRegex = evaluateTemplate(props);
- return Pattern.compile(root.getAbsolutePath() + File.separator + subPathRegex);
- }
-
private String evaluateTemplate(ExtendedProperties props)
{
props.put("template", linkTemplate);
@@ -118,30 +103,21 @@ public class TemplateBasedLinkNamingStrategy implements IHierarchicalStorageLink
return props.getProperty("template");
}
- private int getNestedDirectoryLevels()
- {
- return StringUtils.countMatches(linkTemplate, File.separator);
- }
-
@Private
- static void accumulatePaths(HashSet<String> paths, File dir, Pattern matchingFilesFilter,
- int maxNestedLevel)
+ static void accumulateSymLinkPaths(HashSet<String> paths, File dir)
{
File[] children = dir.listFiles();
if (children != null)
{
for (File child : children)
{
- if (maxNestedLevel > 0)
- {
- accumulatePaths(paths, child, matchingFilesFilter, maxNestedLevel - 1);
- } else
+ if (FileUtilities.isSymbolicLink(child))
{
String absolutePath = child.getAbsolutePath();
- if (matchingFilesFilter.matcher(absolutePath).matches())
- {
- paths.add(absolutePath);
- }
+ paths.add(absolutePath);
+ } else if (child.isDirectory())
+ {
+ accumulateSymLinkPaths(paths, child);
}
}
}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/HierarchicalStorageUpdaterTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/HierarchicalStorageUpdaterTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..4b983848ea2dcadf5a9cc7db1a8527a345be1275
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/HierarchicalStorageUpdaterTest.java
@@ -0,0 +1,177 @@
+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.plugins;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.commons.io.FileUtils;
+import org.jmock.Expectations;
+import org.jmock.Mockery;
+import org.springframework.beans.factory.BeanFactory;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.base.tests.AbstractFileSystemTestCase;
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.utils.DssPropertyParametersUtil;
+import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
+
+/**
+ * @author Kaloyan Enimanev
+ */
+public class HierarchicalStorageUpdaterTest extends AbstractFileSystemTestCase
+{
+
+ private final static File STORE_ROOT_TEMPLATE = new File("./resource/test-data/"
+ + HierarchicalStorageUpdaterTest.class.getSimpleName() + "/store-root");
+
+ private final static String DATASET_TYPE = "dataset-type";
+
+ private IEncapsulatedOpenBISService openBISService;
+
+ private Mockery context;
+
+ public HierarchicalStorageUpdaterTest()
+ {
+ super(false);
+ }
+
+ @BeforeMethod
+ public void setUpMocks() throws Exception
+ {
+
+ prepareDirectoryStructures();
+
+ context = new Mockery();
+ openBISService = context.mock(IEncapsulatedOpenBISService.class);
+ final BeanFactory beanFactory = context.mock(BeanFactory.class);
+ ServiceProvider.setBeanFactory(beanFactory);
+
+ context.checking(new Expectations()
+ {
+ {
+ allowing(beanFactory).getBean("openBIS-service");
+ will(returnValue(openBISService));
+
+ allowing(openBISService).listDataSets();
+ will(returnValue(listDataSets()));
+ }
+ });
+ }
+
+ @Test
+ public void testDataIsNotDeletedAfterReconfig()
+ {
+
+ updater().execute();
+
+ // execute with different configuration and attempt to damage the data store
+ reconfiguredUpdater().execute();
+
+ assertDataStoreNotDamaged();
+
+ }
+
+ private void prepareDirectoryStructures() throws IOException
+ {
+ FileUtils.copyDirectory(STORE_ROOT_TEMPLATE, getStoreRoot());
+ getHierarchyRoot().mkdirs();
+ }
+
+ private File getStoreRoot()
+ {
+ return new File(workingDirectory, "store-root");
+ }
+
+ private File getHierarchyRoot()
+ {
+ return new File(workingDirectory, "hierarchy-root");
+ }
+
+ private void assertDataStoreNotDamaged()
+ {
+ long templateSize = FileUtils.sizeOfDirectory(STORE_ROOT_TEMPLATE);
+ long rootSize = FileUtils.sizeOfDirectory(getStoreRoot());
+
+ String errMessage =
+ String.format("The data-store root in '%s' has been damaged", getStoreRoot()
+ .getAbsolutePath());
+ assertEquals(errMessage, templateSize, rootSize);
+ }
+
+
+ private HierarchicalStorageUpdater updater()
+ {
+ return createUpdater(true);
+ }
+
+ private HierarchicalStorageUpdater reconfiguredUpdater()
+ {
+ return createUpdater(false);
+ }
+
+
+ private HierarchicalStorageUpdater createUpdater(boolean linkFromFirstChild)
+ {
+ final String pluginName = "hierarchical-storage-updater";
+ final String pluginPrefix = pluginName + ".";
+
+ System.setProperty(DssPropertyParametersUtil.OPENBIS_DSS_SYSTEM_PROPERTIES_PREFIX
+ + HierarchicalStorageUpdater.STOREROOT_DIR_KEY, getStoreRoot().getAbsolutePath());
+ System.setProperty(DssPropertyParametersUtil.OPENBIS_DSS_SYSTEM_PROPERTIES_PREFIX
+ + pluginPrefix
+ + HierarchicalStorageUpdater.HIERARCHY_ROOT_DIR_KEY, getHierarchyRoot()
+ .getAbsolutePath());
+
+ Properties properties = new Properties();
+ properties.put(HierarchicalStorageUpdater.LINK_SOURCE_SUBFOLDER + "."
+ + DATASET_TYPE, "original");
+ properties.put(HierarchicalStorageUpdater.LINK_FROM_FIRST_CHILD + "." + DATASET_TYPE, ""
+ + linkFromFirstChild);
+
+ HierarchicalStorageUpdater updater = new HierarchicalStorageUpdater();
+ updater.setUp(pluginName, properties);
+ return updater;
+ }
+
+ private List<SimpleDataSetInformationDTO> listDataSets()
+ {
+ final String share = "1";
+ final File shareRoot = new File(getStoreRoot(), share);
+ final List<SimpleDataSetInformationDTO> result = new ArrayList<SimpleDataSetInformationDTO>();
+ for (File directory : FileUtilities.listDirectories(shareRoot, false))
+ {
+ SimpleDataSetInformationDTO dataset = new SimpleDataSetInformationDTO();
+ result.add(dataset);
+ dataset.setDataSetType(DATASET_TYPE);
+ dataset.setDataSetCode(directory.getName());
+ dataset.setDataSetLocation(directory.getName());
+ dataset.setDataSetShareId(share);
+ dataset.setExperimentCode("experiment");
+ dataset.setGroupCode("space");
+ dataset.setProjectCode("project");
+ dataset.setSampleCode("sample");
+ }
+ return result;
+ }
+}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/TemplateBasedLinkNamingStrategyTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/TemplateBasedLinkNamingStrategyTest.java
index 33f26fac4921491298e16faedf683fef15824304..49bacbe10f8535a0340b3d48ae5db82e2369c29d 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/TemplateBasedLinkNamingStrategyTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/TemplateBasedLinkNamingStrategyTest.java
@@ -18,9 +18,6 @@ package ch.systemsx.cisd.etlserver.plugins;
import static ch.systemsx.cisd.etlserver.plugins.TemplateBasedLinkNamingStrategy.DEFAULT_LINK_TEMPLATE;
-import java.io.File;
-import java.util.Set;
-
import org.testng.annotations.Test;
import ch.systemsx.cisd.base.tests.AbstractFileSystemTestCase;
@@ -67,13 +64,6 @@ public class TemplateBasedLinkNamingStrategyTest extends AbstractFileSystemTestC
}
- @Test
- public void testExtractPathsFromFileSystem() throws Exception
- {
- assertFile(DATASET_PATH_DEFAULT).isExtractedWithTemplate(DEFAULT_LINK_TEMPLATE);
- assertFile(DATASET_PATH_LONG).isExtractedWithTemplate(LONG_LINK_TEMPLATE);
- }
-
private String createPathFromTemplate(String template)
{
SimpleDataSetInformationDTO dsInfo = createDataSetInfo();
@@ -94,33 +84,4 @@ public class TemplateBasedLinkNamingStrategyTest extends AbstractFileSystemTestC
dsInfo.setSampleCode(SAMPLE);
return dsInfo;
}
-
- private PathRecognizingAssertions assertFile(String fileName)
- {
- return new PathRecognizingAssertions(fileName);
- }
-
- private class PathRecognizingAssertions
- {
-
- private String fileName;
-
- public PathRecognizingAssertions(String fileName)
- {
- this.fileName = fileName;
- }
-
- private void isExtractedWithTemplate(String template)
- {
- File dataSetPath = new File(workingDirectory, fileName);
- dataSetPath.mkdirs();
-
- TemplateBasedLinkNamingStrategy strategy =
- new TemplateBasedLinkNamingStrategy(template);
- Set<String> paths = strategy.extractPaths(workingDirectory);
- assertTrue(paths.contains(dataSetPath.getAbsolutePath()));
- }
-
-
- }
}