diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/HierarchicalPath.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/HierarchicalPath.java new file mode 100644 index 0000000000000000000000000000000000000000..637067fcbf03fe51f041c060d87b204b6e0cc020 --- /dev/null +++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/HierarchicalPath.java @@ -0,0 +1,26 @@ +package ch.systemsx.cisd.etlserver.plugins; + +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ContainerDataSet; + +public class HierarchicalPath +{ + private final String path; + + private final ContainerDataSet container; + + public HierarchicalPath(String path, ContainerDataSet container) + { + this.path = path; + this.container = container; + } + + public String getPath() + { + return path; + } + + public ContainerDataSet getContainer() + { + return container; + } +} diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/HierarchicalStorageUpdater.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/HierarchicalStorageUpdater.java index 636fff82f49f83b5fd4c9683eca7bcc3f7da92a1..885eb0ed03dbb5eecfa10c1582251f6f8d289f9b 100644 --- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/HierarchicalStorageUpdater.java +++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/HierarchicalStorageUpdater.java @@ -17,6 +17,7 @@ package ch.systemsx.cisd.etlserver.plugins; import java.io.File; +import java.io.FileFilter; import java.io.IOException; import java.text.DateFormat; import java.text.ParseException; @@ -50,6 +51,7 @@ import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider; import ch.systemsx.cisd.openbis.dss.generic.shared.utils.DssPropertyParametersUtil; import ch.systemsx.cisd.openbis.dss.generic.shared.utils.MetaDataBuilder; import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ContainerDataSet; import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO; /** @@ -81,9 +83,9 @@ public class HierarchicalStorageUpdater implements IDataStoreLockingMaintenanceT public static final String LINK_DIRECTORY = "data"; private static final String META_DATA_TSV_FILE = "meta-data.tsv"; - + private static final String MODIFICATION_TIMESTAMP_FILE = "modification_timestamp"; - + private static final String REBUILDING_HIERARCHICAL_STORAGE = "Rebuilding hierarchical storage"; private static final Logger operationLog = @@ -208,6 +210,19 @@ public class HierarchicalStorageUpdater implements IDataStoreLockingMaintenanceT return result; } + private Set<String> startingWith(String prefix, Set<String> set) + { + Set<String> result = new HashSet<>(); + for (String s : set) + { + if (s.startsWith(prefix)) + { + result.add(s); + } + } + return result; + } + /** * Refreshes the hierarchy of the data inside hierarchical storage accordingly to the database content. */ @@ -216,19 +231,39 @@ public class HierarchicalStorageUpdater implements IDataStoreLockingMaintenanceT operationLog.info(REBUILDING_HIERARCHICAL_STORAGE); List<DataSetInformation> newLinkMappings = collectDataSet(); + // Includes files and all symbolic links. Does no include directories. Set<String> existingPaths = linkNamingStrategy.extractPaths(hierarchyRoot); for (DataSetInformation dataSetInformation : newLinkMappings) { String targetPath = dataSetInformation.targetFile.getAbsolutePath(); - if (existingPaths.contains(targetPath)) + + Set<String> existing = startingWith(targetPath, existingPaths); + + if (existing.size() > 0) { - existingPaths.remove(targetPath); + existingPaths.removeAll(existing); + if (dataSetInformation.containerDto != null) + { + File[] containerMetadata = dataSetInformation.targetFile.getParentFile().listFiles(new FileFilter() + { + @Override + public boolean accept(File file) + { + return file.isFile() && (FileUtilities.isSymbolicLink(file) == false); + } + }); + for (File cm : containerMetadata) + { + existingPaths.remove(cm.getAbsolutePath()); + } + } handleExistingEntry(dataSetInformation); } else { handleNonExistingEntry(dataSetInformation); } + } // by this time - only paths which should be deleted are left in the existingPaths @@ -253,9 +288,10 @@ public class HierarchicalStorageUpdater implements IDataStoreLockingMaintenanceT private void handleExistingEntry(DataSetInformation info) { String errorMsgLinksOnlyModeChanged = "The state of hierarchical store is corrupted or property '" + WITH_META_DATA + + "' or '" + TemplateBasedLinkNamingStrategy.COMPONENT_LINKS_TEMPLATE_PROP_NAME + "' has been modified after hierarchical store has been built. In this case the hierarchical store directory " + "should be deleted manually. It will be recreated after DSS start up."; - if (withMetaData) + if (withMetaData || info.containerDto != null) { if (FileUtilities.isSymbolicLink(info.targetFile)) { @@ -300,12 +336,27 @@ public class HierarchicalStorageUpdater implements IDataStoreLockingMaintenanceT File file = new File(info.targetFile, META_DATA_TSV_FILE); String content = MetaDataBuilder.createMetaData(info.dto); FileUtilities.writeToFile(file, content); + + if (info.containerDto != null) + { + file = new File(info.targetFile + "/..", META_DATA_TSV_FILE); + if (file.exists() == false) + { + content = MetaDataBuilder.createMetaData(info.containerDto); + FileUtilities.writeToFile(file, content); + } + } } private void createModificationDateFile(DataSetInformation info) { File file = new File(info.targetFile, MODIFICATION_TIMESTAMP_FILE); FileUtilities.writeToFile(file, dateFormat.format(info.dto.getModificationDate())); + if (info.containerDto != null) + { + file = new File(info.targetFile + "/..", MODIFICATION_TIMESTAMP_FILE); + FileUtilities.writeToFile(file, dateFormat.format(info.containerDto.getModificationDate())); + } } private Date getModificationDateFromFile(DataSetInformation info) @@ -331,6 +382,11 @@ public class HierarchicalStorageUpdater implements IDataStoreLockingMaintenanceT */ AbstractExternalData dto; + /** + * DTO of container. Might be null. + */ + ContainerDataSet containerDto; + /** * Path where the dataset metadata and link to store should be placed */ @@ -364,36 +420,44 @@ public class HierarchicalStorageUpdater implements IDataStoreLockingMaintenanceT private List<DataSetInformation> collectDataSet() { Collection<SimpleDataSetInformationDTO> dataSets = openBISService.listPhysicalDataSets(); + HashMap<String, AbstractExternalData> dataSetsByCode = getAbstractExternalDataByCode(dataSets); ArrayList<DataSetInformation> result = new ArrayList<DataSetInformation>(); for (SimpleDataSetInformationDTO dataSet : dataSets) { AbstractExternalData abstractData = dataSetsByCode.get(dataSet.getDataSetCode()); - File targetFile = - new File(hierarchyRoot, linkNamingStrategy.createHierarchicalPath(dataSet)); - File share = new File(storeRoot, dataSet.getDataSetShareId()); - File dataSetLocationRoot = new File(share, dataSet.getDataSetLocation()); - File linkSource = determineLinkSource(dataSetLocationRoot, dataSet.getDataSetType()); - - DataSetInformation info = new DataSetInformation(); - info.dto = abstractData; - info.linkSource = linkSource; - info.targetFile = targetFile; - - if (linkSource == null) - { + Set<HierarchicalPath> paths = linkNamingStrategy.createHierarchicalPaths(abstractData); - String logMessage = - String.format("Can not determine the link source file for data set '%s', " - + "dataSetType='%s'. Link creation will be skipped.", - dataSetLocationRoot, dataSet.getDataSetType()); - operationLog.warn(logMessage); - } - // we still want a directory with metadata if metadata option is specified - // if only links should be created we don't want to create broken links - if (withMetaData || linkSource != null) + for (HierarchicalPath hierarchicalPath : paths) { - result.add(info); + + File targetFile = + new File(hierarchyRoot, hierarchicalPath.getPath()); + File share = new File(storeRoot, dataSet.getDataSetShareId()); + File dataSetLocationRoot = new File(share, dataSet.getDataSetLocation()); + File linkSource = determineLinkSource(dataSetLocationRoot, dataSet.getDataSetType()); + + DataSetInformation info = new DataSetInformation(); + info.dto = abstractData; + info.linkSource = linkSource; + info.targetFile = targetFile; + info.containerDto = hierarchicalPath.getContainer(); + + if (linkSource == null) + { + + String logMessage = + String.format("Can not determine the link source file for data set '%s', " + + "dataSetType='%s'. Link creation will be skipped.", + dataSetLocationRoot, dataSet.getDataSetType()); + operationLog.warn(logMessage); + } + // we still want a directory with metadata if metadata option is specified + // if only links should be created we don't want to create broken links + if (withMetaData || linkSource != null) + { + result.add(info); + } } } return result; diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/IHierarchicalStorageLinkNamingStrategy.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/IHierarchicalStorageLinkNamingStrategy.java index a7e4e0283f595d4709e25776c67e0e5737d2d97d..1577d025b0b3bbd43c6cd9f1a50620e7f316cb99 100644 --- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/IHierarchicalStorageLinkNamingStrategy.java +++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/IHierarchicalStorageLinkNamingStrategy.java @@ -19,7 +19,7 @@ package ch.systemsx.cisd.etlserver.plugins; import java.io.File; import java.util.Set; -import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData; /** * A naming strategy for symbolic links in hiearchical storages. @@ -30,9 +30,10 @@ public interface IHierarchicalStorageLinkNamingStrategy { /** - * For a given {@link SimpleDataSetInformationDTO} creates relevant path part e.g. <code>Instance_AAA/Group_BBB/Project_CCC...</code> + * For a given {@link AbstractExternalData} creates relevant path part e.g. <code>Instance_AAA/Group_BBB/Project_CCC...</code> There can be + * multiple paths because dataset can be a component in multiple containers. */ - public String createHierarchicalPath(SimpleDataSetInformationDTO data); + public Set<HierarchicalPath> createHierarchicalPaths(AbstractExternalData data); /** * Returns all data set paths located under <code>root</code>. diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/TemplateBasedLinkNamingStrategy.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/TemplateBasedLinkNamingStrategy.java index 67b449b846bf516fa4f24f5ea91d441c01ba5092..bf61ed1e2580553467277f9fd2a63293c3f20be7 100644 --- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/TemplateBasedLinkNamingStrategy.java +++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/TemplateBasedLinkNamingStrategy.java @@ -17,6 +17,7 @@ package ch.systemsx.cisd.etlserver.plugins; import java.io.File; +import java.util.Collections; import java.util.HashSet; import java.util.Properties; import java.util.Set; @@ -25,9 +26,12 @@ import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import ch.rinn.restrictions.Private; +import ch.systemsx.cisd.common.filesystem.FileUtilities; import ch.systemsx.cisd.common.logging.LogCategory; import ch.systemsx.cisd.common.logging.LogFactory; import ch.systemsx.cisd.common.properties.ExtendedProperties; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ContainerDataSet; import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO; /** @@ -46,11 +50,15 @@ public class TemplateBasedLinkNamingStrategy implements IHierarchicalStorageLink private static final String LINKS_TEMPLATE_PROP_NAME = "template"; + public static final String COMPONENT_LINKS_TEMPLATE_PROP_NAME = "component-template"; + private static final String NOT_DIRECTLY_CONNECTED = "NOT_DIRECTLY_CONNECTED"; private final String linkTemplate; - public TemplateBasedLinkNamingStrategy(String template) + private final String componentLinkTemplate; + + public TemplateBasedLinkNamingStrategy(String template, String componentTemplate) { if (template == null) { @@ -59,12 +67,13 @@ public class TemplateBasedLinkNamingStrategy implements IHierarchicalStorageLink { this.linkTemplate = template; } + + this.componentLinkTemplate = componentTemplate; } public TemplateBasedLinkNamingStrategy(Properties configurationProperties) { - this(configurationProperties.getProperty(LINKS_TEMPLATE_PROP_NAME)); - + this(configurationProperties.getProperty(LINKS_TEMPLATE_PROP_NAME), configurationProperties.getProperty(COMPONENT_LINKS_TEMPLATE_PROP_NAME)); } /** @@ -73,22 +82,44 @@ public class TemplateBasedLinkNamingStrategy implements IHierarchicalStorageLink * @return Instance_AAA/Group_BBB/Project_CCC...</code> */ @Override - public String createHierarchicalPath(SimpleDataSetInformationDTO data) + public Set<HierarchicalPath> createHierarchicalPaths(AbstractExternalData data) { - ExtendedProperties props = new ExtendedProperties(); - for (PathVariable pathElement : PathVariable.values()) + if (data.getContainerDataSets().size() == 0 || componentLinkTemplate == null) { - String pathElementName = pathElement.name(); - String pathElementValue = pathElement.extractValueFromData(data); - if (pathElementValue == null) + ExtendedProperties props = new ExtendedProperties(); + for (PathVariable pathElement : PathVariable.values()) { - pathElementValue = StringUtils.EMPTY; + String pathElementName = pathElement.name(); + String pathElementValue = pathElement.extractValueFromData(data); + if (pathElementValue == null) + { + pathElementValue = StringUtils.EMPTY; + } + props.put(pathElementName, pathElementValue); } - props.put(pathElementName, pathElementValue); - } - return evaluateTemplate(props); + return Collections.singleton(new HierarchicalPath(evaluateTemplate(props, linkTemplate), null)); + } else + { + Set<HierarchicalPath> results = new HashSet<>(); + for (ContainerDataSet container : data.getContainerDataSets()) + { + ExtendedProperties props = new ExtendedProperties(); + for (PathVariable pathElement : PathVariable.values()) + { + String pathElementName = pathElement.name(); + String pathElementValue = pathElement.extractValueFromData(data, container); + if (pathElementValue == null) + { + pathElementValue = StringUtils.EMPTY; + } + props.put(pathElementName, pathElementValue); + } + results.add(new HierarchicalPath(evaluateTemplate(props, componentLinkTemplate), container)); + } + return results; + } } /** @@ -98,35 +129,32 @@ public class TemplateBasedLinkNamingStrategy implements IHierarchicalStorageLink public Set<String> extractPaths(File root) { HashSet<String> set = new HashSet<String>(); - accumulatePaths(set, root, 3); + accumulatePaths(set, root); return set; } - private String evaluateTemplate(ExtendedProperties props) + private String evaluateTemplate(ExtendedProperties props, String template) { - props.put("template", linkTemplate); + props.put("template", template); // this will evaluate and replace all variables in the value of the property return props.getProperty("template"); } @Private - static void accumulatePaths(HashSet<String> paths, File dir, int deepness) + static void accumulatePaths(HashSet<String> paths, File dir) { File[] children = dir.listFiles(); if (children != null) { for (File child : children) { - if (deepness == 0) + if (child.isDirectory() && (FileUtilities.isSymbolicLink(child) == false)) + { + accumulatePaths(paths, child); + } else { String absolutePath = child.getAbsolutePath(); paths.add(absolutePath); - } else if (child.isDirectory()) - { - accumulatePaths(paths, child, deepness - 1); - } else if (child.isFile()) - { - operationLog.warn("File in the Hierarchical store view at the unexpected depth " + child.getAbsolutePath()); } } } @@ -137,71 +165,112 @@ public class TemplateBasedLinkNamingStrategy implements IHierarchicalStorageLink dataSet { @Override - String extractValueFromData(SimpleDataSetInformationDTO data) + String extractValueFromData(AbstractExternalData data, ContainerDataSet container) { - return data.getDataSetCode(); + return data.getCode(); } - }, dataSetType { - @Override - String extractValueFromData(SimpleDataSetInformationDTO data) + String extractValueFromData(AbstractExternalData data, ContainerDataSet container) { - return data.getDataSetType(); + return data.getDataSetType().getCode(); } - }, sample { - @Override - String extractValueFromData(SimpleDataSetInformationDTO data) + String extractValueFromData(AbstractExternalData data, ContainerDataSet container) { return getPathElement(data.getSampleCode()); } - }, experiment { - @Override - String extractValueFromData(SimpleDataSetInformationDTO data) + String extractValueFromData(AbstractExternalData data, ContainerDataSet container) { - return getPathElement(data.getExperimentCode()); + return getPathElement(data.getExperiment().getCode()); } - }, project { @Override - String extractValueFromData(SimpleDataSetInformationDTO data) + String extractValueFromData(AbstractExternalData data, ContainerDataSet container) { - return getPathElement(data.getProjectCode()); + return getPathElement(data.getExperiment().getProject().getCode()); } }, space { + @Override + String extractValueFromData(AbstractExternalData data, ContainerDataSet container) + { + return data.getSpace().getCode(); + } + }, + + containerDataSet + { + @Override + String extractValueFromData(AbstractExternalData data, ContainerDataSet container) + { + if (container != null) + { + return container.getCode(); + } else + { + return null; + } + } + }, + containerDataSetType + { @Override - String extractValueFromData(SimpleDataSetInformationDTO data) + String extractValueFromData(AbstractExternalData data, ContainerDataSet container) { - return data.getSpaceCode(); + if (container != null) + { + return container.getDataSetType().getCode(); + } else + { + return null; + } + } + }, + + containerSample + { + @Override + String extractValueFromData(AbstractExternalData data, ContainerDataSet container) + { + if (container != null) + { + return getPathElement(container.getSampleCode()); + } else + { + return null; + } } }; - abstract String extractValueFromData(SimpleDataSetInformationDTO data); + String extractValueFromData(AbstractExternalData data) + { + return extractValueFromData(data, null); + } + + abstract String extractValueFromData(AbstractExternalData data, ContainerDataSet container); private static String getPathElement(String pathElement) { return pathElement == null ? NOT_DIRECTLY_CONNECTED : pathElement; } - } } \ No newline at end of file diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/HierarchicalStorageUpdaterTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/HierarchicalStorageUpdaterTest.java index 22e239de7ffea7f2245afa0fe931bba0ba2cf42d..8f68dfa890bac541f37b98495019a04378f799f7 100644 --- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/HierarchicalStorageUpdaterTest.java +++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/HierarchicalStorageUpdaterTest.java @@ -301,6 +301,7 @@ public class HierarchicalStorageUpdaterTest extends AbstractFileSystemTestCase space.setCode("space"); Project project = new Project(); project.setSpace(space); + project.setCode("project"); Experiment experiment = new Experiment(); experiment.setProject(project); experiment.setCode("experiment"); diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/TemplateBasedLinkNamingStrategyTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/TemplateBasedLinkNamingStrategyTest.java index 76f39617ca90dea18dcae58345002f79bb301e88..14c715f3a7be2cb72f04b002f34c4c95454f5834 100644 --- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/TemplateBasedLinkNamingStrategyTest.java +++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/TemplateBasedLinkNamingStrategyTest.java @@ -21,7 +21,13 @@ import static ch.systemsx.cisd.etlserver.plugins.TemplateBasedLinkNamingStrategy import org.testng.annotations.Test; import ch.systemsx.cisd.base.tests.AbstractFileSystemTestCase; -import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetType; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Experiment; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.PhysicalDataSet; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Project; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Sample; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Space; /** * Test cases for {@link TemplateBasedLinkNamingStrategy}. @@ -64,21 +70,35 @@ public class TemplateBasedLinkNamingStrategyTest extends AbstractFileSystemTestC private String createPathFromTemplate(String template) { - SimpleDataSetInformationDTO dsInfo = createDataSetInfo(); - return new TemplateBasedLinkNamingStrategy(template) - .createHierarchicalPath(dsInfo); + AbstractExternalData dsInfo = createDataSetInfo(); + return new TemplateBasedLinkNamingStrategy(template, null) + .createHierarchicalPaths(dsInfo).iterator().next().getPath(); } - private SimpleDataSetInformationDTO createDataSetInfo() + private AbstractExternalData createDataSetInfo() { - SimpleDataSetInformationDTO dsInfo = new SimpleDataSetInformationDTO(); - dsInfo.setDataSetCode(DATASET); - dsInfo.setDataSetLocation(LOCATION); - dsInfo.setDataSetType(TYPE); - dsInfo.setExperimentCode(EXPERIMENT); - dsInfo.setSpaceCode(GROUP); - dsInfo.setProjectCode(PROJECT); - dsInfo.setSampleCode(SAMPLE); + PhysicalDataSet dsInfo = new PhysicalDataSet(); + dsInfo.setCode(DATASET); + dsInfo.setLocation(LOCATION); + + DataSetType type = new DataSetType(); + type.setCode(TYPE); + dsInfo.setDataSetType(type); + + Space space = new Space(); + space.setCode(GROUP); + Project project = new Project(); + project.setCode(PROJECT); + project.setSpace(space); + Experiment experiment = new Experiment(); + experiment.setCode(EXPERIMENT); + experiment.setProject(project); + dsInfo.setExperiment(experiment); + + Sample sample = new Sample(); + sample.setCode(SAMPLE); + dsInfo.setSample(sample); + return dsInfo; } }