From 45b3d627c75aa897e58221c0c71cc41479b055fc Mon Sep 17 00:00:00 2001
From: kaloyane <kaloyane>
Date: Mon, 30 May 2011 10:02:34 +0000
Subject: [PATCH] LMS-2257: FTP server does not close IHierarchicalContent
handles
SVN: 21504
---
.../generic/server/ftp/FtpFileFactory.java | 78 +++++
...HierarchicalContentClosingInputStream.java | 52 ++++
.../resolver/FtpFileEvaluationContext.java | 106 +++++++
...ToFtpFileAdapter.java => FtpFileImpl.java} | 81 ++++--
.../resolver/IExperimentChildrenLister.java | 7 +
.../TemplateBasedDataSetResourceResolver.java | 271 ++++++++++--------
6 files changed, 453 insertions(+), 142 deletions(-)
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/FtpFileFactory.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/HierarchicalContentClosingInputStream.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/FtpFileEvaluationContext.java
rename datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/{HierarchicalContentToFtpFileAdapter.java => FtpFileImpl.java} (54%)
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/FtpFileFactory.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/FtpFileFactory.java
new file mode 100644
index 00000000000..f80cc1c6ec9
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/FtpFileFactory.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.server.ftp;
+
+import java.io.File;
+
+import org.apache.ftpserver.ftplet.FtpFile;
+
+import ch.systemsx.cisd.common.io.IHierarchicalContentNode;
+import ch.systemsx.cisd.common.io.IHierarchicalContentNodeFilter;
+import ch.systemsx.cisd.openbis.dss.generic.server.ftp.resolver.FtpFileImpl;
+
+/**
+ * A factory constructing {@link FtpFile} objects.
+ *
+ * @author Kaloyan Enimanev
+ */
+public class FtpFileFactory
+{
+
+ /**
+ * This is just a convenience method that retrieves all necessary data from an
+ * {@link IHierarchicalContentNode} and constructs an {@link FtpFile}.
+ * <p>
+ * The reference to {@link IHierarchicalContentNode} is *not* kept by the returned instance.
+ * Thus, callers are responsible for closing all resources associated with the
+ * {@link IHierarchicalContentNode} on their own.
+ */
+ public static FtpFile createFtpFile(String dataSetCode, String path,
+ IHierarchicalContentNode contentNode, IHierarchicalContentNodeFilter childrenFilter)
+ {
+ return new FtpFileImpl(dataSetCode, path, contentNode.getRelativePath(),
+ contentNode.isDirectory(), getSize(contentNode),
+ getLastModified(contentNode), childrenFilter);
+
+ }
+
+ private static long getSize(IHierarchicalContentNode contentNode)
+ {
+ if (contentNode.isDirectory())
+ {
+ return 0;
+ } else
+ {
+ return contentNode.getFileLength();
+ }
+ }
+
+ private static long getLastModified(IHierarchicalContentNode contentNode)
+ {
+ try
+ {
+ File file = contentNode.getFile();
+ if (file != null)
+ {
+ return file.lastModified();
+ }
+ } catch (UnsupportedOperationException uoe)
+ {
+ // ignore
+ }
+ return 0;
+ }
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/HierarchicalContentClosingInputStream.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/HierarchicalContentClosingInputStream.java
new file mode 100644
index 00000000000..d4e1c4d26a3
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/HierarchicalContentClosingInputStream.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.server.ftp;
+
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+import ch.systemsx.cisd.common.io.IHierarchicalContent;
+
+/**
+ * An {@link InputStream} implementation which closes an associated {@link IHierarchicalContent}
+ * together with an underlying target {@link InputStream}.
+ *
+ * @author Kaloyan Enimanev
+ */
+public class HierarchicalContentClosingInputStream extends FilterInputStream
+{
+ private final IHierarchicalContent hierarchicalContent;
+
+ public HierarchicalContentClosingInputStream(InputStream target,
+ IHierarchicalContent hierarchicalContent)
+ {
+ super(target);
+ this.hierarchicalContent = hierarchicalContent;
+ }
+
+ @Override
+ public void close() throws IOException
+ {
+ // no error can be throw here
+ hierarchicalContent.close();
+
+ // can throw IOException
+ super.close();
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/FtpFileEvaluationContext.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/FtpFileEvaluationContext.java
new file mode 100644
index 00000000000..02c0762b229
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/FtpFileEvaluationContext.java
@@ -0,0 +1,106 @@
+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.server.ftp.resolver;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+
+import ch.systemsx.cisd.common.io.IHierarchicalContent;
+import ch.systemsx.cisd.common.io.IHierarchicalContentNode;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ExternalData;
+
+/**
+ * An object holding templates evaluation result data.
+ *
+ * @author Kaloyan Enimanev
+ */
+public class FtpFileEvaluationContext
+{
+
+ static class EvaluatedElement
+ {
+ ExternalData dataSet;
+
+ // will only be filled when the ${fileName} variable
+ // is used in the template
+ String pathInDataSet = StringUtils.EMPTY;
+
+ String evaluatedTemplate;
+
+ IHierarchicalContentNode contentNode;
+ }
+
+ private Map<String /* dataset code */, IHierarchicalContent> contents =
+ new HashMap<String, IHierarchicalContent>();
+
+ private List<EvaluatedElement> evaluatedPaths = new ArrayList<EvaluatedElement>();
+
+ /**
+ * @return the evaluation result.
+ */
+ public List<EvaluatedElement> getEvalElements()
+ {
+ return Collections.unmodifiableList(evaluatedPaths);
+
+ }
+
+ /**
+ * Adds a collection of {@link EvaluatedElement} to the results.
+ */
+ public void addEvaluatedElements(Collection<EvaluatedElement> evaluatedPath)
+ {
+ evaluatedPaths.addAll(evaluatedPath);
+ }
+
+ public IHierarchicalContent getHierarchicalContent(String dataSetCode)
+ {
+ IHierarchicalContent result = contents.get(dataSetCode);
+ if (result == null)
+ {
+ result = createHierarchicalContent(dataSetCode);
+ contents.put(dataSetCode, result);
+ }
+ return result;
+ }
+
+ /**
+ * closes the evaluation context and frees all associated resources.
+ */
+ public void close()
+ {
+ for (IHierarchicalContent content : contents.values())
+ {
+ content.close();
+ }
+ contents.clear();
+ }
+
+ private IHierarchicalContent createHierarchicalContent(String code)
+ {
+ IHierarchicalContentProvider provider = ServiceProvider.getHierarchicalContentProvider();
+ return provider.asContent(code);
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/HierarchicalContentToFtpFileAdapter.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/FtpFileImpl.java
similarity index 54%
rename from datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/HierarchicalContentToFtpFileAdapter.java
rename to datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/FtpFileImpl.java
index d8587f6ecb4..38b6d4c3728 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/HierarchicalContentToFtpFileAdapter.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/FtpFileImpl.java
@@ -23,34 +23,58 @@ import java.util.List;
import org.apache.ftpserver.ftplet.FtpFile;
+import ch.systemsx.cisd.common.io.IHierarchicalContent;
import ch.systemsx.cisd.common.io.IHierarchicalContentNode;
import ch.systemsx.cisd.common.io.IHierarchicalContentNodeFilter;
import ch.systemsx.cisd.openbis.dss.generic.server.ftp.FtpConstants;
+import ch.systemsx.cisd.openbis.dss.generic.server.ftp.FtpFileFactory;
+import ch.systemsx.cisd.openbis.dss.generic.server.ftp.HierarchicalContentClosingInputStream;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
/**
- * An {@link FtpFile} implementation adapting an underlying {@link IHierarchicalContentNode}.
+ * An {@link FtpFile} implementation which lazily creates and uses {@link IHierarchicalContent} when
+ * needed.
* <p>
- * The resources represented by {@link HierarchicalContentToFtpFileAdapter} exist in the data store.
+ * The resources represented by {@link FtpFileImpl} exist in the data store.
*
* @author Kaloyan Enimanev
*/
-public class HierarchicalContentToFtpFileAdapter extends AbstractFtpFile
+public class FtpFileImpl extends AbstractFtpFile
{
- private final IHierarchicalContentNode contentNode;
+ private final String dataSetCode;
+
+ private final String pathInDataSet;
+
+ private final boolean isDirectory;
+
+ private final long size;
+
+ private final long lastModified;
private final IHierarchicalContentNodeFilter childrenFilter;
- public HierarchicalContentToFtpFileAdapter(String path, IHierarchicalContentNode contentNode,
+ public FtpFileImpl(String dataSetCode, String path, String pathInDataSet, boolean isDirectory,
+ long size,
+ long lastModified,
IHierarchicalContentNodeFilter childrenFilter)
{
super(path);
- this.contentNode = contentNode;
+ this.dataSetCode = dataSetCode;
+ this.pathInDataSet = pathInDataSet;
+ this.isDirectory = isDirectory;
+ this.size = size;
+ this.lastModified = lastModified;
this.childrenFilter = childrenFilter;
}
public InputStream createInputStream(long offset) throws IOException
{
- InputStream result = contentNode.getInputStream();
+ IHierarchicalContent content = createHierarchicalContent();
+ IHierarchicalContentNode contentNode = getContentNodeForThisFile(content);
+
+ InputStream result =
+ new HierarchicalContentClosingInputStream(contentNode.getInputStream(), content);
+
if (offset > 0)
{
result.skip(offset);
@@ -60,13 +84,7 @@ public class HierarchicalContentToFtpFileAdapter extends AbstractFtpFile
public long getLastModified()
{
- try
- {
- return contentNode.getFile().lastModified();
- } catch (UnsupportedOperationException uoe)
- {
- return 0;
- }
+ return lastModified;
}
@@ -74,14 +92,14 @@ public class HierarchicalContentToFtpFileAdapter extends AbstractFtpFile
{
if (isFile())
{
- return contentNode.getFileLength();
+ return size;
}
return 0;
}
public boolean isDirectory()
{
- return contentNode.isDirectory();
+ return isDirectory;
}
public boolean isFile()
@@ -92,30 +110,45 @@ public class HierarchicalContentToFtpFileAdapter extends AbstractFtpFile
@Override
public List<org.apache.ftpserver.ftplet.FtpFile> unsafeListFiles()
{
- if (isDirectory())
+ if (isFile())
{
+ throw new UnsupportedOperationException();
+ }
+
+ IHierarchicalContent content = createHierarchicalContent();
+ try
+ {
+ IHierarchicalContentNode contentNode = getContentNodeForThisFile(content);
List<IHierarchicalContentNode> children = contentNode.getChildNodes();
List<org.apache.ftpserver.ftplet.FtpFile> result =
new ArrayList<org.apache.ftpserver.ftplet.FtpFile>();
for (IHierarchicalContentNode childNode : children)
- {
+ {
if (childrenFilter.accept(childNode))
{
String childPath =
absolutePath + FtpConstants.FILE_SEPARATOR + childNode.getName();
- HierarchicalContentToFtpFileAdapter childFile =
- new HierarchicalContentToFtpFileAdapter(childPath, childNode,
+ FtpFile childFile =
+ FtpFileFactory.createFtpFile(dataSetCode, childPath, childNode,
childrenFilter);
result.add(childFile);
}
- }
+ }
return result;
-
- } else
+ } finally
{
- throw new UnsupportedOperationException();
+ content.close();
}
}
+ private IHierarchicalContent createHierarchicalContent()
+ {
+ return ServiceProvider.getHierarchicalContentProvider().asContent(dataSetCode);
+ }
+
+ private IHierarchicalContentNode getContentNodeForThisFile(IHierarchicalContent content)
+ {
+ return content.getNode(pathInDataSet);
+ }
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/IExperimentChildrenLister.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/IExperimentChildrenLister.java
index 9bb58c54550..6a4afb02d71 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/IExperimentChildrenLister.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/IExperimentChildrenLister.java
@@ -24,10 +24,17 @@ import ch.systemsx.cisd.openbis.dss.generic.server.ftp.FtpPathResolverContext;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Experiment;
/**
+ * An interface decoupling the resolver implementations for experiments and data sets.
+ *
* @author Kaloyan Enimanev
*/
public interface IExperimentChildrenLister
{
+ /**
+ * Lists the children {@link FtpFile} objects in an experiment.
+ *
+ * @param parentPath the FTP path representing the experiment.
+ */
List<FtpFile> listExperimentChildrenPaths(Experiment experiment, String parentPath,
FtpPathResolverContext context);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/TemplateBasedDataSetResourceResolver.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/TemplateBasedDataSetResourceResolver.java
index fd0f26da13d..d70d236bab4 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/TemplateBasedDataSetResourceResolver.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/TemplateBasedDataSetResourceResolver.java
@@ -37,11 +37,11 @@ import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.utilities.Template;
import ch.systemsx.cisd.openbis.dss.generic.server.ftp.FtpConstants;
+import ch.systemsx.cisd.openbis.dss.generic.server.ftp.FtpFileFactory;
import ch.systemsx.cisd.openbis.dss.generic.server.ftp.FtpPathResolverContext;
import ch.systemsx.cisd.openbis.dss.generic.server.ftp.FtpServerConfig;
import ch.systemsx.cisd.openbis.dss.generic.server.ftp.IFtpPathResolver;
-import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
-import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
+import ch.systemsx.cisd.openbis.dss.generic.server.ftp.resolver.FtpFileEvaluationContext.EvaluatedElement;
import ch.systemsx.cisd.openbis.generic.shared.IETLLIMSService;
import ch.systemsx.cisd.openbis.generic.shared.basic.TechId;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Experiment;
@@ -51,9 +51,10 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.ExperimentIdentifi
/**
* Resolves paths like
- * /<space-code>/<project-code>/<experiment-code>/<dataset-template>[/<sub-path>]*
+ * "/<space-code>/<project-code>/<experiment-code>/<dataset-template>[/<sub-path>]*" to
+ * {@link FtpFile} objects.
* <p>
- * Subpaths are resolved as a relative paths starting from the root of a dataset.
+ * Subpaths are resolved as relative paths starting from the root of a dataset.
* <p>
*
* @author Kaloyan Enimanev
@@ -92,19 +93,6 @@ public class TemplateBasedDataSetResourceResolver implements IFtpPathResolver,
IHierarchicalContentNodeFilter fileFilter = IHierarchicalContentNodeFilter.MATCH_ALL;
}
- private static class EvaluatedDataSetPath
- {
- ExternalData dataSet;
-
- // will only be filled when the ${fileName} variable
- // is used in the template
- String fileName = StringUtils.EMPTY;
-
- String evaluatedTemplate;
-
- IHierarchicalContentNode contentNode;
- }
-
/**
* a template, that can contain special variables.
*
@@ -125,35 +113,6 @@ public class TemplateBasedDataSetResourceResolver implements IFtpPathResolver,
}
- private Map<String, DataSetTypeConfig> initializeDataSetTypeConfigs(
- FtpServerConfig ftpServerConfig)
- {
- Map<String, DataSetTypeConfig> result = new HashMap<String, DataSetTypeConfig>();
- Map<String, String> fileListSubPaths = ftpServerConfig.getFileListSubPaths();
- Map<String, String> fileListFilters = ftpServerConfig.getFileListFilters();
-
- for (Entry<String, String> subPathEntry : fileListSubPaths.entrySet())
- {
- DataSetTypeConfig dsConfig = new DataSetTypeConfig();
- dsConfig.fileListSubPath = subPathEntry.getValue();
- result.put(subPathEntry.getKey(), dsConfig);
- }
-
- for (Entry<String, String> filterEntry : fileListFilters.entrySet())
- {
- String dataSetType = filterEntry.getKey();
- String fileFilterPattern = filterEntry.getValue();
- DataSetTypeConfig dsConfig = result.get(dataSetType);
- if (dsConfig == null)
- {
- dsConfig = new DataSetTypeConfig();
- }
- dsConfig.fileFilter = createFilter(fileFilterPattern);
- result.put(dataSetType, dsConfig);
- }
- return result;
- }
-
/**
* @return <code>true</code> for paths containing at least 4 nested directory levels.
*/
@@ -165,75 +124,68 @@ public class TemplateBasedDataSetResourceResolver implements IFtpPathResolver,
public FtpFile resolve(String path, final FtpPathResolverContext resolverContext)
{
- IETLLIMSService service = resolverContext.getService();
- String sessionToken = resolverContext.getSessionToken();
-
- EvaluatedDataSetPath evaluationResult =
- tryExtractDataSetAndFileName(path, service, sessionToken);
- if (evaluationResult == null)
+ String experimentId = extractExperimentIdFromPath(path);
+ FtpFileEvaluationContext evalContext =
+ evaluateExperimentDataSets(experimentId, resolverContext);
+ if (evalContext == null)
{
return null;
}
- String nestedSubPath = extractNestedSubPath(path);
- String relativePath = evaluationResult.fileName;
- if (false == StringUtils.isBlank(nestedSubPath))
+ try
{
- if (StringUtils.isBlank(relativePath))
- {
- relativePath = nestedSubPath;
- } else
- {
- relativePath += FtpConstants.FILE_SEPARATOR + nestedSubPath;
- }
+ return extractMatchingFileOrNull(path, experimentId, evalContext);
+ } finally
+ {
+ evalContext.close();
}
+ }
+
+ private FtpFile extractMatchingFileOrNull(String path, String experimentId,
+ FtpFileEvaluationContext evalContext)
+ {
+ EvaluatedElement matchingElement =
+ tryFindMatchingEvalElement(path, experimentId, evalContext);
- IHierarchicalContentNodeFilter fileFilter = getFileFilter(evaluationResult.dataSet);
- IHierarchicalContentProvider provider = ServiceProvider.getHierarchicalContentProvider();
- IHierarchicalContent content = provider.asContent(evaluationResult.dataSet.getCode());
- // FIXME use content.close() - otherwise data set will be locked until timeout
- IHierarchicalContentNode contentNode = content.getNode(relativePath);
+ String pathInDataSet = extractPathInDataSet(path);
+ String hierarchicalNodePath =
+ constructHierarchicalNodePath(matchingElement.pathInDataSet, pathInDataSet);
+
+ ExternalData dataSet = matchingElement.dataSet;
+ IHierarchicalContentNodeFilter fileFilter = getFileFilter(dataSet);
+ IHierarchicalContent content = evalContext.getHierarchicalContent(dataSet.getCode());
+ IHierarchicalContentNode contentNode = content.getNode(hierarchicalNodePath);
if (fileFilter.accept(contentNode))
{
- return new HierarchicalContentToFtpFileAdapter(path, contentNode, fileFilter);
+ return FtpFileFactory.createFtpFile(dataSet.getCode(), path, contentNode, fileFilter);
} else
{
return null;
}
}
- private EvaluatedDataSetPath tryExtractDataSetAndFileName(String path, IETLLIMSService service,
- String sessionToken)
+ private EvaluatedElement tryFindMatchingEvalElement(String path, String experimentId,
+ FtpFileEvaluationContext evalContext)
{
- String experimentId = extractExperimentIdentifier(path);
- Experiment experiment = tryExtractExperiment(experimentId, service, sessionToken);
- if (experiment == null)
- {
- // cannot resolve an existing experiment from the specified path
- return null;
- }
- List<ExternalData> dataSets =
- service.listDataSetsByExperimentID(sessionToken, new TechId(experiment));
String pathWithEndSlash = path + FtpConstants.FILE_SEPARATOR;
-
- for (EvaluatedDataSetPath evaluatedPath : evaluateDataSetPaths(dataSets))
+ for (EvaluatedElement evalElement : evalContext.getEvalElements())
{
String fullEvaluatedPath =
- experimentId + FtpConstants.FILE_SEPARATOR + evaluatedPath.evaluatedTemplate
+ experimentId + FtpConstants.FILE_SEPARATOR + evalElement.evaluatedTemplate
+ FtpConstants.FILE_SEPARATOR;
if (pathWithEndSlash.startsWith(fullEvaluatedPath))
{
- return evaluatedPath;
+ return evalElement;
}
}
-
return null;
}
/**
+ * @param path the path we are trying to resolve
* @return the nested path within the data set (i.e. under the dataset directory level)
*/
- private String extractNestedSubPath(String path)
+ private String extractPathInDataSet(String path)
{
String[] levels = StringUtils.split(path, FtpConstants.FILE_SEPARATOR);
if (levels.length > 4)
@@ -245,7 +197,23 @@ public class TemplateBasedDataSetResourceResolver implements IFtpPathResolver,
}
}
- private Experiment tryExtractExperiment(String experimentId, IETLLIMSService service,
+ private String constructHierarchicalNodePath(String relativePath, String pathInDataSet)
+ {
+ String result = relativePath;
+ if (false == StringUtils.isBlank(pathInDataSet))
+ {
+ if (StringUtils.isBlank(relativePath))
+ {
+ result = pathInDataSet;
+ } else
+ {
+ result += FtpConstants.FILE_SEPARATOR + pathInDataSet;
+ }
+ }
+ return result;
+ }
+
+ private Experiment tryGetExperiment(String experimentId, IETLLIMSService service,
String sessionToken)
{
ExperimentIdentifier experimentIdentifier =
@@ -263,7 +231,7 @@ public class TemplateBasedDataSetResourceResolver implements IFtpPathResolver,
return result;
}
- private String extractExperimentIdentifier(String path)
+ private String extractExperimentIdFromPath(String path)
{
String[] levels = StringUtils.split(path, FtpConstants.FILE_SEPARATOR);
String experimentId =
@@ -272,43 +240,84 @@ public class TemplateBasedDataSetResourceResolver implements IFtpPathResolver,
return experimentId;
}
- public List<FtpFile> listExperimentChildrenPaths(Experiment experiment, String parentPath,
- FtpPathResolverContext context)
+ /**
+ * @see IExperimentChildrenLister
+ */
+ public List<FtpFile> listExperimentChildrenPaths(Experiment experiment,
+ final String parentPath, FtpPathResolverContext context)
{
IETLLIMSService service = context.getService();
String sessionToken = context.getSessionToken();
- List<FtpFile> result = new ArrayList<FtpFile>();
List<ExternalData> dataSets =
service.listDataSetsByExperimentID(sessionToken, new TechId(experiment));
- for (EvaluatedDataSetPath evaluationResult : evaluateDataSetPaths(dataSets))
+
+ FtpFileEvaluationContext evalContext = evaluateDataSetPaths(dataSets);
+ try
+ {
+ return createFtpFilesFromEvaluationResult(parentPath, evalContext);
+ } finally
+ {
+ evalContext.close();
+ }
+ }
+
+ private List<FtpFile> createFtpFilesFromEvaluationResult(final String parentPath,
+ FtpFileEvaluationContext evalResult)
+ {
+ ArrayList<FtpFile> result = new ArrayList<FtpFile>();
+ for (EvaluatedElement evalElement : evalResult.getEvalElements())
{
- IHierarchicalContentNodeFilter fileFilter = getFileFilter(evaluationResult.dataSet);
- if (fileFilter.accept(evaluationResult.contentNode))
+ IHierarchicalContentNodeFilter fileFilter = getFileFilter(evalElement.dataSet);
+ if (fileFilter.accept(evalElement.contentNode))
{
String childPath =
- parentPath + FtpConstants.FILE_SEPARATOR
- + evaluationResult.evaluatedTemplate;
+ parentPath + FtpConstants.FILE_SEPARATOR + evalElement.evaluatedTemplate;
FtpFile childFtpFile =
- new HierarchicalContentToFtpFileAdapter(childPath,
- evaluationResult.contentNode, fileFilter);
+ FtpFileFactory.createFtpFile(evalElement.dataSet.getCode(), childPath,
+ evalElement.contentNode, fileFilter);
result.add(childFtpFile);
}
}
+
return result;
}
- private List<EvaluatedDataSetPath> evaluateDataSetPaths(List<ExternalData> dataSets)
+ private FtpFileEvaluationContext evaluateExperimentDataSets(String experimentId,
+ FtpPathResolverContext resolverContext)
+ {
+ IETLLIMSService service = resolverContext.getService();
+ String sessionToken = resolverContext.getSessionToken();
+
+ Experiment experiment = tryGetExperiment(experimentId, service, sessionToken);
+ if (experiment == null)
+ {
+ // cannot resolve an existing experiment from the specified path
+ return null;
+ }
+ List<ExternalData> dataSets =
+ service.listDataSetsByExperimentID(sessionToken, new TechId(experiment));
+
+ return evaluateDataSetPaths(dataSets);
+ }
+
+ private FtpFileEvaluationContext evaluateDataSetPaths(List<ExternalData> dataSets)
{
- List<EvaluatedDataSetPath> result = new ArrayList<EvaluatedDataSetPath>();
+ FtpFileEvaluationContext evalContext = new FtpFileEvaluationContext();
for (int disambiguationIdx = 0; disambiguationIdx < dataSets.size(); disambiguationIdx++)
{
ExternalData dataSet = dataSets.get(disambiguationIdx);
try
{
- List<EvaluatedDataSetPath> paths = evaluateDataSetPaths(dataSet, disambiguationIdx);
- result.addAll(paths);
+ IHierarchicalContent hierarchicalContent =
+ evalContext.getHierarchicalContent(dataSet.getCode());
+ IHierarchicalContentNode rootNode =
+ getDataSetFileListRoot(dataSet, hierarchicalContent);
+ List<EvaluatedElement> paths =
+ evaluateDataSetPaths(dataSet, rootNode, disambiguationIdx);
+
+ evalContext.addEvaluatedElements(paths);
} catch (Throwable t)
{
operationLog.warn("Failed to evaluate data set paths for dataset "
@@ -316,41 +325,38 @@ public class TemplateBasedDataSetResourceResolver implements IFtpPathResolver,
}
}
- return result;
+ return evalContext;
}
- private List<EvaluatedDataSetPath> evaluateDataSetPaths(ExternalData dataSet,
- int disambiguationIndex)
+ private List<EvaluatedElement> evaluateDataSetPaths(ExternalData dataSet,
+ IHierarchicalContentNode rootNode, int disambiguationIndex)
{
- List<EvaluatedDataSetPath> result = new ArrayList<EvaluatedDataSetPath>();
-
- IHierarchicalContentNode rootNode = getDataSetFileListRoot(dataSet);
+ List<EvaluatedElement> result = new ArrayList<EvaluatedElement>();
String disambiguationVar = computeDisambiguation(disambiguationIndex);
for (IHierarchicalContentNode fileNode : getFileNamesRequiredByTemplate(rootNode))
{
- EvaluatedDataSetPath evaluatedPath = new EvaluatedDataSetPath();
- evaluatedPath.dataSet = dataSet;
- evaluatedPath.fileName = fileNode.getRelativePath();
- evaluatedPath.evaluatedTemplate =
+ EvaluatedElement evalElement = new EvaluatedElement();
+ evalElement.dataSet = dataSet;
+ evalElement.pathInDataSet = fileNode.getRelativePath();
+ evalElement.evaluatedTemplate =
evaluateTemplate(dataSet, fileNode.getName(), disambiguationVar);
- evaluatedPath.contentNode = fileNode;
- result.add(evaluatedPath);
+ evalElement.contentNode = fileNode;
+ result.add(evalElement);
}
return result;
}
- private IHierarchicalContentNode getDataSetFileListRoot(ExternalData dataSet)
+ private IHierarchicalContentNode getDataSetFileListRoot(ExternalData dataSet,
+ IHierarchicalContent hierachicalContent)
{
- IHierarchicalContentProvider provider = ServiceProvider.getHierarchicalContentProvider();
- IHierarchicalContent content = provider.asContent(dataSet.getCode());
String fileListSubPathOrNull = getFileListSubPath(dataSet);
if (false == StringUtils.isBlank(fileListSubPathOrNull))
{
List<IHierarchicalContentNode> matchingNodes =
- content.listMatchingNodes(fileListSubPathOrNull);
+ hierachicalContent.listMatchingNodes(fileListSubPathOrNull);
if (false == matchingNodes.isEmpty())
{
if (matchingNodes.size() == 1)
@@ -373,7 +379,7 @@ public class TemplateBasedDataSetResourceResolver implements IFtpPathResolver,
operationLog.warn(message);
}
}
- return content.getRootNode();
+ return hierachicalContent.getRootNode();
}
/**
@@ -434,6 +440,35 @@ public class TemplateBasedDataSetResourceResolver implements IFtpPathResolver,
return parsedTemplate.getPlaceholderNames().contains(variableName);
}
+ private Map<String, DataSetTypeConfig> initializeDataSetTypeConfigs(
+ FtpServerConfig ftpServerConfig)
+ {
+ Map<String, DataSetTypeConfig> result = new HashMap<String, DataSetTypeConfig>();
+ Map<String, String> fileListSubPaths = ftpServerConfig.getFileListSubPaths();
+ Map<String, String> fileListFilters = ftpServerConfig.getFileListFilters();
+
+ for (Entry<String, String> subPathEntry : fileListSubPaths.entrySet())
+ {
+ DataSetTypeConfig dsConfig = new DataSetTypeConfig();
+ dsConfig.fileListSubPath = subPathEntry.getValue();
+ result.put(subPathEntry.getKey(), dsConfig);
+ }
+
+ for (Entry<String, String> filterEntry : fileListFilters.entrySet())
+ {
+ String dataSetType = filterEntry.getKey();
+ String fileFilterPattern = filterEntry.getValue();
+ DataSetTypeConfig dsConfig = result.get(dataSetType);
+ if (dsConfig == null)
+ {
+ dsConfig = new DataSetTypeConfig();
+ }
+ dsConfig.fileFilter = createFilter(fileFilterPattern);
+ result.put(dataSetType, dsConfig);
+ }
+ return result;
+ }
+
private IHierarchicalContentNodeFilter createFilter(final String fileFilterPattern)
{
return new IHierarchicalContentNodeFilter()
--
GitLab