From 93414e94f3958361058a852d187cb66f5241eb1c Mon Sep 17 00:00:00 2001
From: felmer <felmer>
Date: Tue, 4 Oct 2011 13:53:02 +0000
Subject: [PATCH] LMS-2558 implemented and tested
SVN: 23177
---
.../server/EncapsulatedOpenBISService.java | 22 +-
.../generic/server/ftp/DSSFileSystemView.java | 12 +-
.../generic/server/ftp/FtpFileFactory.java | 8 +-
.../server/ftp/FtpPathResolverContext.java | 11 +-
.../dss/generic/server/ftp/FtpServer.java | 11 +-
.../generic/server/ftp/FtpServerConfig.java | 13 +-
.../server/ftp/resolver/AbstractFtpFile.java | 3 +-
.../resolver/FtpFileEvaluationContext.java | 11 +-
.../server/ftp/resolver/FtpFileImpl.java | 21 +-
.../TemplateBasedDataSetResourceResolver.java | 269 +++++++--
.../dss/generic/shared/ServiceProvider.java | 6 +
.../source/java/dssApplicationContext.xml | 7 +
.../server/ftp/FtpServerConfigBuilder.java | 11 +-
...plateBasedDataSetResourceResolverTest.java | 518 ++++++++++--------
14 files changed, 623 insertions(+), 300 deletions(-)
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/EncapsulatedOpenBISService.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/EncapsulatedOpenBISService.java
index 6ee9df81701..35ef23f3489 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/EncapsulatedOpenBISService.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/EncapsulatedOpenBISService.java
@@ -24,6 +24,7 @@ import org.apache.commons.lang.time.DateUtils;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.FactoryBean;
+import ch.systemsx.cisd.common.api.client.ServiceFinder;
import ch.systemsx.cisd.common.exceptions.UserFailureException;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
@@ -34,6 +35,7 @@ import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
import ch.systemsx.cisd.openbis.generic.shared.IETLLIMSService;
import ch.systemsx.cisd.openbis.generic.shared.ResourceNames;
+import ch.systemsx.cisd.openbis.generic.shared.api.v1.IGeneralInformationService;
import ch.systemsx.cisd.openbis.generic.shared.api.v1.OpenBisServiceFactory;
import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.SearchCriteria;
import ch.systemsx.cisd.openbis.generic.shared.basic.TechId;
@@ -104,9 +106,27 @@ public final class EncapsulatedOpenBISService implements IEncapsulatedOpenBISSer
{
return factory.createService();
}
- return factory.createService(Integer.parseInt(timeout) * DateUtils.MILLIS_PER_MINUTE);
+ return factory.createService(normalizeTimeout(timeout));
}
+ /**
+ * Creates a remote version of {@link IGeneralInformationService} for specified URL and
+ * time out (in minutes).
+ */
+ public static IGeneralInformationService createGeneralInformationService(String openBISURL,
+ String timeout)
+ {
+ ServiceFinder generalInformationServiceFinder =
+ new ServiceFinder("openbis", IGeneralInformationService.SERVICE_URL);
+ return generalInformationServiceFinder.createService(IGeneralInformationService.class,
+ openBISURL, normalizeTimeout(timeout));
+ }
+
+ private static long normalizeTimeout(String timeout)
+ {
+ return Integer.parseInt(timeout) * DateUtils.MILLIS_PER_MINUTE;
+ }
+
public EncapsulatedOpenBISService(IETLLIMSService service, OpenBISSessionHolder sessionHolder)
{
this(service, sessionHolder, null);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/DSSFileSystemView.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/DSSFileSystemView.java
index a622aac2e94..709360c9636 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/DSSFileSystemView.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/DSSFileSystemView.java
@@ -25,6 +25,7 @@ import ch.systemsx.cisd.cifex.client.application.utils.StringUtils;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.openbis.generic.shared.IETLLIMSService;
+import ch.systemsx.cisd.openbis.generic.shared.api.v1.IGeneralInformationService;
/**
* A central class that manages the movement of a user up and down the exposed hierarchical
@@ -41,15 +42,19 @@ public class DSSFileSystemView implements FileSystemView
private final IETLLIMSService service;
+ private final IGeneralInformationService generalInfoService;
+
private FtpFile workingDirectory;
private final IFtpPathResolverRegistry pathResolverRegistry;
DSSFileSystemView(String sessionToken, IETLLIMSService service,
+ IGeneralInformationService generalInfoService,
IFtpPathResolverRegistry pathResolverRegistry) throws FtpException
{
this.sessionToken = sessionToken;
this.service = service;
+ this.generalInfoService = generalInfoService;
this.pathResolverRegistry = pathResolverRegistry;
this.workingDirectory = getHomeDirectory();
}
@@ -82,15 +87,16 @@ public class DSSFileSystemView implements FileSystemView
try
{
FtpPathResolverContext context =
- new FtpPathResolverContext(sessionToken, service, pathResolverRegistry);
+ new FtpPathResolverContext(sessionToken, service, generalInfoService,
+ pathResolverRegistry);
return pathResolverRegistry.tryResolve(normalizedPath, context);
} catch (RuntimeException rex)
{
String message =
- String.format("Error while resolving FTP path '%s' : '%s", path,
+ String.format("Error while resolving FTP path '%s' : %s", path,
rex.getMessage());
operationLog.error(message);
- return null;
+ throw new FtpException(message, rex);
}
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/FtpFileFactory.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/FtpFileFactory.java
index ef4f53367d2..aa8f4ed3822 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/FtpFileFactory.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/FtpFileFactory.java
@@ -21,6 +21,7 @@ import java.io.File;
import org.apache.ftpserver.ftplet.FtpFile;
import ch.systemsx.cisd.common.io.hierarchical_content.IHierarchicalContentNodeFilter;
+import ch.systemsx.cisd.common.io.hierarchical_content.api.IHierarchicalContent;
import ch.systemsx.cisd.common.io.hierarchical_content.api.IHierarchicalContentNode;
import ch.systemsx.cisd.openbis.dss.generic.server.ftp.resolver.FtpFileImpl;
@@ -41,11 +42,12 @@ public class FtpFileFactory
* {@link IHierarchicalContentNode} on their own.
*/
public static FtpFile createFtpFile(String dataSetCode, String path,
- IHierarchicalContentNode contentNode, IHierarchicalContentNodeFilter childrenFilter)
+ IHierarchicalContentNode contentNode, IHierarchicalContent content,
+ IHierarchicalContentNodeFilter childrenFilter)
{
return new FtpFileImpl(dataSetCode, path, contentNode.getRelativePath(),
- contentNode.isDirectory(), getSize(contentNode),
- getLastModified(contentNode), childrenFilter);
+ contentNode.isDirectory(), getSize(contentNode), getLastModified(contentNode),
+ content, childrenFilter);
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/FtpPathResolverContext.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/FtpPathResolverContext.java
index 465e4cce345..17d97ea2384 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/FtpPathResolverContext.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/FtpPathResolverContext.java
@@ -17,6 +17,7 @@
package ch.systemsx.cisd.openbis.dss.generic.server.ftp;
import ch.systemsx.cisd.openbis.generic.shared.IETLLIMSService;
+import ch.systemsx.cisd.openbis.generic.shared.api.v1.IGeneralInformationService;
/**
* An object holding all necessary context information for ftp path resolution.
@@ -30,13 +31,16 @@ public class FtpPathResolverContext
private final IETLLIMSService service;
+ private final IGeneralInformationService generalInfoService;
+
private final IFtpPathResolverRegistry resolverRegistry;
- public FtpPathResolverContext(String sessionToken, IETLLIMSService service,
+ public FtpPathResolverContext(String sessionToken, IETLLIMSService service, IGeneralInformationService generalInfoService,
IFtpPathResolverRegistry resolverRegistry)
{
this.sessionToken = sessionToken;
this.service = service;
+ this.generalInfoService = generalInfoService;
this.resolverRegistry = resolverRegistry;
}
@@ -50,6 +54,11 @@ public class FtpPathResolverContext
return service;
}
+ public IGeneralInformationService getGeneralInfoService()
+ {
+ return generalInfoService;
+ }
+
public IFtpPathResolverRegistry getResolverRegistry()
{
return resolverRegistry;
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/FtpServer.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/FtpServer.java
index 2ee047d6906..963b13e8840 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/FtpServer.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/FtpServer.java
@@ -33,6 +33,7 @@ import org.apache.log4j.Logger;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.openbis.generic.shared.IETLLIMSService;
+import ch.systemsx.cisd.openbis.generic.shared.api.v1.IGeneralInformationService;
/**
* Controls the lifecycle of an FTP server built into DSS.
@@ -54,10 +55,13 @@ public class FtpServer implements FileSystemFactory
private org.apache.ftpserver.FtpServer server;
- public FtpServer(IETLLIMSService openBisService, UserManager userManager, Properties configProps)
- throws Exception
+ private final IGeneralInformationService generalInfoService;
+
+ public FtpServer(IETLLIMSService openBisService, IGeneralInformationService generalInfoService,
+ UserManager userManager, Properties configProps) throws Exception
{
this.openBisService = openBisService;
+ this.generalInfoService = generalInfoService;
this.userManager = userManager;
this.config = new FtpServerConfig(configProps);
this.pathResolverRegistry = new FtpPathResolverRegistry(config);
@@ -128,7 +132,8 @@ public class FtpServer implements FileSystemFactory
if (user instanceof FtpUser)
{
String sessionToken = ((FtpUser) user).getSessionToken();
- return new DSSFileSystemView(sessionToken, openBisService, pathResolverRegistry);
+ return new DSSFileSystemView(sessionToken, openBisService, generalInfoService,
+ pathResolverRegistry);
} else
{
throw new FtpException("Unsupported user type.");
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/FtpServerConfig.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/FtpServerConfig.java
index 2347066e464..a77ead83ca3 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/FtpServerConfig.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/FtpServerConfig.java
@@ -29,6 +29,7 @@ import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.utilities.ExtendedProperties;
import ch.systemsx.cisd.common.utilities.PropertyUtils;
+import ch.systemsx.cisd.common.utilities.Template;
import ch.systemsx.cisd.openbis.dss.generic.server.ConfigParameters;
/**
@@ -62,6 +63,8 @@ public class FtpServerConfig
final static String ACTIVE_PORT_KEY = PREFIX + "activemode.port";
final static String PASSIVE_MODE_PORT_RANGE_KEY = PREFIX + "passivemode.port.range";
+
+ final static String SHOW_PARENTS_AND_CHILDREN_KEY = PREFIX + "show-parents-and-children";
private static final int DEFAULT_PORT = 2121;
@@ -107,6 +110,8 @@ public class FtpServerConfig
private Map<String /* dataset type */, String /* filter pattern */> fileListFilters =
new HashMap<String, String>();
+ private boolean showParentsAndChildren;
+
public FtpServerConfig(Properties props) {
this.startServer = PropertyUtils.getBoolean(props, ENABLE_KEY, false);
if (startServer)
@@ -131,7 +136,8 @@ public class FtpServerConfig
maxThreads = PropertyUtils.getPosInt(props, MAX_THREADS_KEY, DEFAULT_MAX_THREADS);
dataSetDisplayTemplate =
PropertyUtils.getProperty(props, DATASET_DISPLAY_TEMPLATE_KEY, DEFAULT_DATASET_TEMPLATE);
-
+ showParentsAndChildren = PropertyUtils.getBoolean(props, SHOW_PARENTS_AND_CHILDREN_KEY, false);
+
ExtendedProperties fileListSubPathProps =
ExtendedProperties.getSubset(props, DATASET_FILELIST_SUBPATH_KEY, true);
for (Object key : fileListSubPathProps.keySet())
@@ -211,6 +217,11 @@ public class FtpServerConfig
return dataSetDisplayTemplate;
}
+ public boolean isShowParentsAndChildren()
+ {
+ return showParentsAndChildren;
+ }
+
public Map<String, String> getFileListSubPaths()
{
return Collections.unmodifiableMap(fileListSubPaths);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/AbstractFtpFile.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/AbstractFtpFile.java
index 8f4896b2b48..2987e797d93 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/AbstractFtpFile.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/AbstractFtpFile.java
@@ -19,7 +19,6 @@ package ch.systemsx.cisd.openbis.dss.generic.server.ftp.resolver;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
-import java.util.Collections;
import java.util.List;
import org.apache.ftpserver.ftplet.FtpFile;
@@ -60,7 +59,7 @@ public abstract class AbstractFtpFile implements FtpFile
} catch (RuntimeException rex)
{
operationLog.error("Error while listing files for FTP :" + rex.getMessage(), rex);
- return Collections.emptyList();
+ throw rex;
}
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/FtpFileEvaluationContext.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/FtpFileEvaluationContext.java
index 35363311043..e112df1a69b 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/FtpFileEvaluationContext.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/FtpFileEvaluationContext.java
@@ -28,7 +28,6 @@ import org.apache.commons.lang.StringUtils;
import ch.systemsx.cisd.common.io.hierarchical_content.api.IHierarchicalContent;
import ch.systemsx.cisd.common.io.hierarchical_content.api.IHierarchicalContentNode;
import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
-import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ExternalData;
/**
@@ -55,7 +54,14 @@ public class FtpFileEvaluationContext
private Map<String /* dataset code */, IHierarchicalContent> contents =
new HashMap<String, IHierarchicalContent>();
+ private IHierarchicalContentProvider contentProvider;
+
private List<EvaluatedElement> evaluatedPaths = new ArrayList<EvaluatedElement>();
+
+ FtpFileEvaluationContext(IHierarchicalContentProvider contentProvider)
+ {
+ this.contentProvider = contentProvider;
+ }
/**
* @return the evaluation result.
@@ -100,8 +106,7 @@ public class FtpFileEvaluationContext
private IHierarchicalContent createHierarchicalContent(ExternalData dataSet)
{
- IHierarchicalContentProvider provider = ServiceProvider.getHierarchicalContentProvider();
- return provider.asContent(dataSet);
+ return contentProvider.asContent(dataSet);
}
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/FtpFileImpl.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/FtpFileImpl.java
index 13ce96597c6..3283f9613dc 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/FtpFileImpl.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/FtpFileImpl.java
@@ -53,8 +53,10 @@ public class FtpFileImpl extends AbstractFtpFile
private final IHierarchicalContentNodeFilter childrenFilter;
+ private IHierarchicalContent content;
+
public FtpFileImpl(String dataSetCode, String path, String pathInDataSet, boolean isDirectory,
- long size, long lastModified, IHierarchicalContentNodeFilter childrenFilter)
+ long size, long lastModified, IHierarchicalContent content, IHierarchicalContentNodeFilter childrenFilter)
{
super(path);
this.dataSetCode = dataSetCode;
@@ -62,15 +64,15 @@ public class FtpFileImpl extends AbstractFtpFile
this.isDirectory = isDirectory;
this.size = size;
this.lastModified = lastModified;
+ this.content = content;
this.childrenFilter = childrenFilter;
}
public InputStream createInputStream(long offset) throws IOException
{
- IHierarchicalContent content = createHierarchicalContent();
try
{
- IHierarchicalContentNode contentNode = getContentNodeForThisFile(content);
+ IHierarchicalContentNode contentNode = getContentNodeForThisFile();
InputStream result =
HierarchicalContentUtils.getInputStreamAutoClosingContent(contentNode, content);
@@ -122,10 +124,9 @@ public class FtpFileImpl extends AbstractFtpFile
throw new UnsupportedOperationException();
}
- IHierarchicalContent content = createHierarchicalContent();
try
{
- IHierarchicalContentNode contentNode = getContentNodeForThisFile(content);
+ IHierarchicalContentNode contentNode = getContentNodeForThisFile();
List<IHierarchicalContentNode> children = contentNode.getChildNodes();
List<org.apache.ftpserver.ftplet.FtpFile> result =
new ArrayList<org.apache.ftpserver.ftplet.FtpFile>();
@@ -137,7 +138,7 @@ public class FtpFileImpl extends AbstractFtpFile
String childPath =
absolutePath + FtpConstants.FILE_SEPARATOR + childNode.getName();
FtpFile childFile =
- FtpFileFactory.createFtpFile(dataSetCode, childPath, childNode,
+ FtpFileFactory.createFtpFile(dataSetCode, childPath, childNode, content,
childrenFilter);
result.add(childFile);
}
@@ -151,10 +152,14 @@ public class FtpFileImpl extends AbstractFtpFile
private IHierarchicalContent createHierarchicalContent()
{
- return ServiceProvider.getHierarchicalContentProvider().asContent(dataSetCode);
+ if (content == null)
+ {
+ content = ServiceProvider.getHierarchicalContentProvider().asContent(dataSetCode);
+ }
+ return content;
}
- private IHierarchicalContentNode getContentNodeForThisFile(IHierarchicalContent content)
+ private IHierarchicalContentNode getContentNodeForThisFile()
{
return content.getNode(pathInDataSet);
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/TemplateBasedDataSetResourceResolver.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/TemplateBasedDataSetResourceResolver.java
index ae253df0aa7..8cf26609e20 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/TemplateBasedDataSetResourceResolver.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/TemplateBasedDataSetResourceResolver.java
@@ -17,6 +17,7 @@
package ch.systemsx.cisd.openbis.dss.generic.server.ftp.resolver;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
@@ -30,6 +31,8 @@ import org.apache.commons.lang.time.DateFormatUtils;
import org.apache.ftpserver.ftplet.FtpFile;
import org.apache.log4j.Logger;
+import ch.rinn.restrictions.Private;
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
import ch.systemsx.cisd.common.io.hierarchical_content.IHierarchicalContentNodeFilter;
import ch.systemsx.cisd.common.io.hierarchical_content.api.IHierarchicalContent;
import ch.systemsx.cisd.common.io.hierarchical_content.api.IHierarchicalContentNode;
@@ -42,7 +45,11 @@ import ch.systemsx.cisd.openbis.dss.generic.server.ftp.FtpPathResolverContext;
import ch.systemsx.cisd.openbis.dss.generic.server.ftp.FtpServerConfig;
import ch.systemsx.cisd.openbis.dss.generic.server.ftp.IFtpPathResolver;
import ch.systemsx.cisd.openbis.dss.generic.server.ftp.resolver.FtpFileEvaluationContext.EvaluatedElement;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
import ch.systemsx.cisd.openbis.generic.shared.IETLLIMSService;
+import ch.systemsx.cisd.openbis.generic.shared.api.v1.IGeneralInformationService;
+import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.DataSet;
import ch.systemsx.cisd.openbis.generic.shared.basic.TechId;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Experiment;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ExternalData;
@@ -51,8 +58,10 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.ExperimentIdentifi
/**
* Resolves paths like
- * "/<space-code>/<project-code>/<experiment-code>/<dataset-template>[/<sub-path>]*" to
- * {@link FtpFile} objects.
+ * <pre>
+ * /<space-code>/<project-code>/<experiment-code>/<dataset-template>[/[PARENT-<dataset-template>|CHILD-<dataset-template>|<sub-path>]]*
+ * </pre>
+ * to {@link FtpFile} objects.
* <p>
* Subpaths are resolved as relative paths starting from the root of a dataset.
* <p>
@@ -74,8 +83,87 @@ public class TemplateBasedDataSetResourceResolver implements IFtpPathResolver,
private static final String DATA_SET_DATE_FORMAT = "yyyy-MM-dd-HH-mm";
+ private static final String PARENT_PREFIX = "PARENT-";
+
+ private static final String CHILD_PREFIX = "CHILD-";
+
private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
TemplateBasedDataSetResourceResolver.class);
+
+ private final class DataSetFtpFolder extends AbstractFtpFolder
+ {
+ private final ExternalData dataSet;
+
+ private final FtpPathResolverContext resolverContext;
+
+ private DataSetFtpFolder(String absolutePath, ExternalData dataSet,
+ FtpPathResolverContext resolverContext)
+ {
+ super(absolutePath);
+ this.dataSet = dataSet;
+ this.resolverContext = resolverContext;
+ }
+
+ @Override
+ public List<FtpFile> unsafeListFiles() throws RuntimeException
+ {
+ List<FtpFile> result = new ArrayList<FtpFile>();
+ if (showParentsAndChildren)
+ {
+ IGeneralInformationService generalInfoService =
+ resolverContext.getGeneralInfoService();
+ String sessionToken = resolverContext.getSessionToken();
+ List<DataSet> dataSetsWithMetaData =
+ generalInfoService.getDataSetMetaData(sessionToken,
+ Arrays.asList(dataSet.getCode()));
+
+ DataSet dataSetWithMetaData = dataSetsWithMetaData.get(0);
+ addNodesOfType(PARENT_PREFIX, result, dataSetWithMetaData.getParentCodes());
+ addNodesOfType(CHILD_PREFIX, result, dataSetWithMetaData.getChildrenCodes());
+ }
+ FtpFileEvaluationContext evalContext = createFtpFileEvaluationContext();
+ try
+ {
+ IHierarchicalContent hierarchicalContent =
+ evalContext.getHierarchicalContent(dataSet);
+ IHierarchicalContentNode rootNode =
+ getDataSetFileListRoot(dataSet, hierarchicalContent);
+ List<IHierarchicalContentNode> childNodes = rootNode.getChildNodes();
+ for (IHierarchicalContentNode childNode : childNodes)
+ {
+ IHierarchicalContentNodeFilter fileFilter = getFileFilter(dataSet);
+ if (fileFilter.accept(childNode))
+ {
+ result.add(FtpFileFactory.createFtpFile(dataSet.getCode(), absolutePath
+ + FtpConstants.FILE_SEPARATOR + childNode.getName(), childNode,
+ hierarchicalContent, fileFilter));
+ }
+ }
+ } finally
+ {
+ evalContext.close();
+ }
+ return result;
+ }
+
+ private void addNodesOfType(String prefix, List<FtpFile> result, List<String> dataSetCodes)
+ {
+ if (dataSetCodes.isEmpty())
+ {
+ return;
+ }
+ String sessionToken = resolverContext.getSessionToken();
+ List<ExternalData> dataSets =
+ resolverContext.getService().listDataSetsByCode(sessionToken, dataSetCodes);
+ for (int i = 0; i < dataSets.size(); i++)
+ {
+ ExternalData ds = dataSets.get(i);
+ String folderName = prefix + evaluateTemplate(ds, null, computeDisambiguation(i));
+ result.add(new DataSetFtpFolder(absolutePath + FtpConstants.FILE_SEPARATOR
+ + folderName, ds, resolverContext));
+ }
+ }
+ }
/**
* helper class holding the configuration properties for a data set type.
@@ -99,18 +187,36 @@ public class TemplateBasedDataSetResourceResolver implements IFtpPathResolver,
* @see #evaluateTemplate(ExternalData, String) to find out what variables are understood and
* interpreted.
*/
- private final String template;
+ private final Template template;
private final Map<String /* dataset type */, DataSetTypeConfig> dataSetTypeConfigs;
private final DataSetTypeConfig defaultDSTypeConfig;
+ private boolean showParentsAndChildren;
+
+ private boolean fileNamePresent;
+
+ private IHierarchicalContentProvider contentProvider;
+
public TemplateBasedDataSetResourceResolver(FtpServerConfig ftpServerConfig)
{
- this.template = ftpServerConfig.getDataSetDisplayTemplate();
+ this.template = new Template(ftpServerConfig.getDataSetDisplayTemplate());
+ showParentsAndChildren = ftpServerConfig.isShowParentsAndChildren();
+ fileNamePresent = template.getPlaceholderNames().contains(FILE_NAME_VARNAME);
+ if (fileNamePresent && showParentsAndChildren)
+ {
+ throw new ConfigurationFailureException(
+ "Template contains file name variable and "
+ + "the flag to show parents/children data sets is set.");
+ }
this.dataSetTypeConfigs = initializeDataSetTypeConfigs(ftpServerConfig);
this.defaultDSTypeConfig = new DataSetTypeConfig();
-
+ }
+
+ void setContentProvider(IHierarchicalContentProvider contentProvider)
+ {
+ this.contentProvider = contentProvider;
}
/**
@@ -122,23 +228,95 @@ public class TemplateBasedDataSetResourceResolver implements IFtpPathResolver,
return nestedLevels >= 4;
}
- public FtpFile resolve(String path, final FtpPathResolverContext resolverContext)
+ public FtpFile resolve(final String path, final FtpPathResolverContext resolverContext)
{
String experimentId = extractExperimentIdFromPath(path);
- FtpFileEvaluationContext evalContext =
- evaluateExperimentDataSets(experimentId, resolverContext);
- if (evalContext == null)
+ IETLLIMSService service = resolverContext.getService();
+ String sessionToken = resolverContext.getSessionToken();
+
+ Experiment experiment = tryGetExperiment(experimentId, service, sessionToken);
+ if (experiment == null)
{
- return null;
+ throw new IllegalArgumentException("Unknown experiment '" + experimentId + "'.");
}
+ List<ExternalData> dataSets =
+ service.listDataSetsByExperimentID(sessionToken, new TechId(experiment));
+ if (fileNamePresent)
+ {
+ FtpFileEvaluationContext evalContext = evaluateDataSetPaths(dataSets);
+
+ try
+ {
+ return extractMatchingFileOrNull(path, experimentId, evalContext);
+ } finally
+ {
+ evalContext.close();
+ }
+ }
+ return resolve(path, resolverContext, dataSets);
+ }
- try
+ private FtpFile resolve(final String path, final FtpPathResolverContext resolverContext,
+ List<ExternalData> dataSets)
+ {
+ String[] pathElements =
+ StringUtils.splitByWholeSeparatorPreserveAllTokens(path,
+ FtpConstants.FILE_SEPARATOR);
+ FtpFile result = null;
+ for (int i = 4; i < pathElements.length; i++)
{
- return extractMatchingFileOrNull(path, experimentId, evalContext);
- } finally
+ String dataSetPathElement = pathElements[i];
+ if (result == null)
+ {
+ ExternalData dataSet = tryToFindDataSet(dataSets, dataSetPathElement);
+ if (dataSet == null)
+ {
+ throw createException(dataSetPathElement);
+ }
+ String subPath =
+ StringUtils.join(pathElements, FtpConstants.FILE_SEPARATOR, 0, i + 1);
+ result = new DataSetFtpFolder(subPath, dataSet, resolverContext);
+ } else
+ {
+ List<FtpFile> files = result.listFiles();
+ FtpFile matchingFile = null;
+ for (FtpFile file : files)
+ {
+ if (dataSetPathElement.equals(file.getName()))
+ {
+ matchingFile = file;
+ break;
+ }
+ }
+ if (matchingFile == null)
+ {
+ throw createException(dataSetPathElement);
+ }
+ result = matchingFile;
+ }
+ }
+ return result;
+ }
+
+ private IllegalArgumentException createException(String dataSetPathElement)
+ {
+ return new IllegalArgumentException("No match found for path element '"
+ + dataSetPathElement + "'.");
+ }
+
+ private ExternalData tryToFindDataSet(List<ExternalData> dataSets, String dataSetPathElement)
+ {
+ for (int disambiguationIdx = 0; disambiguationIdx < dataSets.size(); disambiguationIdx++)
{
- evalContext.close();
+ String disambiguationVar = computeDisambiguation(disambiguationIdx);
+ ExternalData dataSet = dataSets.get(disambiguationIdx);
+ String pathElement = evaluateTemplate(dataSet, null, disambiguationVar);
+ if (dataSetPathElement.equals(pathElement))
+ {
+ return dataSet;
+ }
}
+ return null;
}
private FtpFile extractMatchingFileOrNull(String path, String experimentId,
@@ -157,7 +335,8 @@ public class TemplateBasedDataSetResourceResolver implements IFtpPathResolver,
IHierarchicalContentNode contentNode = content.getNode(hierarchicalNodePath);
if (fileFilter.accept(contentNode))
{
- return FtpFileFactory.createFtpFile(dataSet.getCode(), path, contentNode, fileFilter);
+ return FtpFileFactory.createFtpFile(dataSet.getCode(), path, contentNode, content,
+ fileFilter);
} else
{
return null;
@@ -273,9 +452,14 @@ public class TemplateBasedDataSetResourceResolver implements IFtpPathResolver,
{
String childPath =
parentPath + FtpConstants.FILE_SEPARATOR + evalElement.evaluatedTemplate;
+ String dataSetCode = evalElement.dataSet.getCode();
+ IHierarchicalContentProvider getContentProvider =
+ getContentProvider();
+
FtpFile childFtpFile =
- FtpFileFactory.createFtpFile(evalElement.dataSet.getCode(), childPath,
- evalElement.contentNode, fileFilter);
+ FtpFileFactory.createFtpFile(dataSetCode, childPath,
+ evalElement.contentNode, getContentProvider.asContent(dataSetCode),
+ fileFilter);
result.add(childFtpFile);
}
}
@@ -283,27 +467,9 @@ public class TemplateBasedDataSetResourceResolver implements IFtpPathResolver,
return result;
}
- private FtpFileEvaluationContext evaluateExperimentDataSets(String experimentId,
- FtpPathResolverContext resolverContext)
- {
- IETLLIMSService service = resolverContext.getService();
- String sessionToken = resolverContext.getSessionToken();
-
- Experiment experiment = tryGetExperiment(experimentId, service, sessionToken);
- if (experiment == null)
- {
- // cannot resolve an existing experiment from the specified path
- return null;
- }
- List<ExternalData> dataSets =
- service.listDataSetsByExperimentID(sessionToken, new TechId(experiment));
-
- return evaluateDataSetPaths(dataSets);
- }
-
private FtpFileEvaluationContext evaluateDataSetPaths(List<ExternalData> dataSets)
{
- FtpFileEvaluationContext evalContext = new FtpFileEvaluationContext();
+ FtpFileEvaluationContext evalContext = createFtpFileEvaluationContext();
for (int disambiguationIdx = 0; disambiguationIdx < dataSets.size(); disambiguationIdx++)
{
@@ -397,7 +563,7 @@ public class TemplateBasedDataSetResourceResolver implements IFtpPathResolver,
*/
private List<IHierarchicalContentNode> getFileNamesRequiredByTemplate(IHierarchicalContentNode rootNode)
{
- if (isVariablePresentInTemplate(FILE_NAME_VARNAME))
+ if (fileNamePresent)
{
if (rootNode.isDirectory())
{
@@ -409,7 +575,7 @@ public class TemplateBasedDataSetResourceResolver implements IFtpPathResolver,
private String evaluateTemplate(ExternalData dataSet, String fileName, String disambiguation)
{
- Template eval = new Template(template);
+ Template eval = template.createFreshCopy();
eval.attemptToBind(DATA_SET_CODE_VARNAME, dataSet.getCode());
eval.attemptToBind(DATA_SET_TYPE_VARNAME, dataSet.getDataSetType().getCode());
String dataSetDate = extractDateValue(dataSet.getRegistrationDate());
@@ -424,22 +590,13 @@ public class TemplateBasedDataSetResourceResolver implements IFtpPathResolver,
}
/**
- * formats a date as it will appear when a {@link #DATA_SET_DATE_VARNAME} is replaced.
+ * Formats a date as it will appear after template evaluation.
*/
- private String extractDateValue(Date dataSetDate)
+ @Private static String extractDateValue(Date dataSetDate)
{
return DateFormatUtils.format(dataSetDate, DATA_SET_DATE_FORMAT);
}
- /**
- * @return true if the specified variable is present in the template.
- */
- private boolean isVariablePresentInTemplate(String variableName)
- {
- Template parsedTemplate = new Template(template);
- return parsedTemplate.getPlaceholderNames().contains(variableName);
- }
-
private Map<String, DataSetTypeConfig> initializeDataSetTypeConfigs(
FtpServerConfig ftpServerConfig)
{
@@ -511,4 +668,18 @@ public class TemplateBasedDataSetResourceResolver implements IFtpPathResolver,
return getDataSetTypeConfig(dataSet).fileListSubPath;
}
+ private FtpFileEvaluationContext createFtpFileEvaluationContext()
+ {
+ return new FtpFileEvaluationContext(getContentProvider());
+ }
+
+ private IHierarchicalContentProvider getContentProvider()
+ {
+ if (contentProvider == null)
+ {
+ contentProvider = ServiceProvider.getHierarchicalContentProvider();
+ }
+ return contentProvider;
+ }
+
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/ServiceProvider.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/ServiceProvider.java
index 49ffe07a842..5956f81078d 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/ServiceProvider.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/ServiceProvider.java
@@ -28,6 +28,7 @@ import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.openbis.dss.generic.shared.api.internal.v1.IDataSourceQueryService;
import ch.systemsx.cisd.openbis.dss.generic.shared.api.internal.v1.ISearchService;
+import ch.systemsx.cisd.openbis.generic.shared.api.v1.IGeneralInformationService;
/**
* Provider of remote service onto openBIS.
@@ -100,6 +101,11 @@ public class ServiceProvider
return ((IEncapsulatedOpenBISService) getApplicationContext().getBean("openBIS-service"));
}
+ public static IGeneralInformationService getGeneralInformationService()
+ {
+ return ((IGeneralInformationService) getApplicationContext().getBean("general-information-service"));
+ }
+
public static ISearchService getSearchService()
{
return ((ISearchService) getApplicationContext().getBean("search-service"));
diff --git a/datastore_server/source/java/dssApplicationContext.xml b/datastore_server/source/java/dssApplicationContext.xml
index 2c07fc869d6..e3486d1d30e 100644
--- a/datastore_server/source/java/dssApplicationContext.xml
+++ b/datastore_server/source/java/dssApplicationContext.xml
@@ -33,6 +33,12 @@
<constructor-arg value="${server-timeout-in-minutes}"/>
</bean>
+ <bean id="general-information-service" class="ch.systemsx.cisd.openbis.dss.generic.server.EncapsulatedOpenBISService"
+ factory-method="createGeneralInformationService">
+ <constructor-arg value="${server-url}"/>
+ <constructor-arg value="${server-timeout-in-minutes}"/>
+ </bean>
+
<bean id="sessionHolder" class="ch.systemsx.cisd.openbis.dss.generic.server.openbisauth.OpenBISSessionHolder">
<property name="dataStoreCode" value="${data-store-server-code}"/>
</bean>
@@ -168,6 +174,7 @@
-->
<bean id="ftp-server" class="ch.systemsx.cisd.openbis.dss.generic.server.ftp.FtpServer" destroy-method="stop">
<constructor-arg ref="etl-lims-service"/>
+ <constructor-arg ref="general-information-service"/>
<constructor-arg ref="adapted-ftp-user-manager"/>
<constructor-arg ref="configProperties" />
</bean>
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/FtpServerConfigBuilder.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/FtpServerConfigBuilder.java
index c9157046a84..582856a4a89 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/FtpServerConfigBuilder.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/FtpServerConfigBuilder.java
@@ -43,13 +43,20 @@ public class FtpServerConfigBuilder
props.put(FtpServerConfig.USE_SSL_KEY, String.valueOf(useSSL));
}
- public FtpServerConfigBuilder withTemplate(String template)
+ public FtpServerConfigBuilder showParentsAndChildren()
{
- props.setProperty(FtpServerConfig.DATASET_DISPLAY_TEMPLATE_KEY, template);
+ props.setProperty(FtpServerConfig.SHOW_PARENTS_AND_CHILDREN_KEY, Boolean.TRUE.toString());
return this;
}
+ public FtpServerConfigBuilder withTemplate(String template)
+ {
+ props.setProperty(FtpServerConfig.DATASET_DISPLAY_TEMPLATE_KEY, template);
+ return this;
+
+ }
+
public FtpServerConfigBuilder withFileListFilter(String dataSetType, String filterPattern)
{
String key = FtpServerConfig.DATASET_FILELIST_FILTER_KEY + dataSetType;
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/TemplateBasedDataSetResourceResolverTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/TemplateBasedDataSetResourceResolverTest.java
index 3304ad35ba2..8afae30cdfb 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/TemplateBasedDataSetResourceResolverTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/ftp/resolver/TemplateBasedDataSetResourceResolverTest.java
@@ -17,45 +17,92 @@
package ch.systemsx.cisd.openbis.dss.generic.server.ftp.resolver;
import java.io.ByteArrayInputStream;
+import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
+import java.util.EnumSet;
import java.util.List;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.io.IOUtils;
import org.apache.ftpserver.ftplet.FtpFile;
import org.jmock.Expectations;
-import org.springframework.beans.factory.BeanFactory;
-import org.testng.AssertJUnit;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
+import ch.rinn.restrictions.Friend;
+import ch.systemsx.cisd.base.tests.AbstractFileSystemTestCase;
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
+import ch.systemsx.cisd.common.io.hierarchical_content.DefaultFileBasedHierarchicalContentFactory;
import ch.systemsx.cisd.common.io.hierarchical_content.api.IHierarchicalContent;
import ch.systemsx.cisd.common.io.hierarchical_content.api.IHierarchicalContentNode;
import ch.systemsx.cisd.common.test.TrackingMockery;
+import ch.systemsx.cisd.common.utilities.IDelegatedAction;
import ch.systemsx.cisd.openbis.dss.generic.server.ftp.FtpConstants;
import ch.systemsx.cisd.openbis.dss.generic.server.ftp.FtpPathResolverContext;
import ch.systemsx.cisd.openbis.dss.generic.server.ftp.FtpServerConfig;
import ch.systemsx.cisd.openbis.dss.generic.server.ftp.FtpServerConfigBuilder;
import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
-import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProviderTestWrapper;
+import ch.systemsx.cisd.openbis.generic.server.api.v1.Translator;
import ch.systemsx.cisd.openbis.generic.shared.IETLLIMSService;
+import ch.systemsx.cisd.openbis.generic.shared.api.v1.IGeneralInformationService;
+import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.DataSet.Connections;
import ch.systemsx.cisd.openbis.generic.shared.basic.TechId;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetType;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSet;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Experiment;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ExternalData;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IDatasetLocation;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.builders.DataSetBuilder;
import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.ExperimentIdentifier;
import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.ExperimentIdentifierFactory;
/**
* @author Kaloyan Enimanev
*/
-public class TemplateBasedDataSetResourceResolverTest extends AssertJUnit
+@Friend(toClasses=TemplateBasedDataSetResourceResolver.class)
+public class TemplateBasedDataSetResourceResolverTest extends AbstractFileSystemTestCase
{
+ private static final class SimpleFileContentProvider implements IHierarchicalContentProvider
+ {
+ private final File root;
+
+ SimpleFileContentProvider(File root)
+ {
+ this.root = root;
+ }
+
+ public IHierarchicalContent asContent(ExternalData dataSet)
+ {
+ return asContent((IDatasetLocation) dataSet.tryGetAsDataSet());
+ }
+
+ public IHierarchicalContent asContent(IDatasetLocation datasetLocation)
+ {
+ String dataSetCode = datasetLocation.getDataSetCode();
+ return asContent(dataSetCode);
+ }
+
+ public IHierarchicalContent asContent(String dataSetCode)
+ {
+ return asContent(new File(root, dataSetCode));
+ }
+
+ public IHierarchicalContent asContent(File datasetDirectory)
+ {
+ return new DefaultFileBasedHierarchicalContentFactory().asHierarchicalContent(
+ datasetDirectory, IDelegatedAction.DO_NOTHING);
+ }
+ }
+
+ private static final Date REGISTRATION_DATE = new Date(42);
+
+ private static final String RENDERED_REGISTRATION_DATE = TemplateBasedDataSetResourceResolver
+ .extractDateValue(REGISTRATION_DATE);
private static final String SESSION_TOKEN = "token";
@@ -72,6 +119,9 @@ public class TemplateBasedDataSetResourceResolverTest extends AssertJUnit
private static final String TEMPLATE_WITH_FILENAMES =
"DS-${dataSetType}-${fileName}-${disambiguation}";
+ private static final String BIG_TEMPLATE =
+ "DS-${dataSetType}-${dataSetCode}-${dataSetDate}-${disambiguation}";
+
private TrackingMockery context;
private IETLLIMSService service;
@@ -83,6 +133,15 @@ public class TemplateBasedDataSetResourceResolverTest extends AssertJUnit
private Experiment experiment;
+ private IGeneralInformationService generalInfoService;
+
+ private SimpleFileContentProvider simpleFileContentProvider;
+
+ private DataSet ds1;
+
+ private DataSet ds2;
+
+ @Override
@BeforeMethod
public void setUp()
{
@@ -92,12 +151,15 @@ public class TemplateBasedDataSetResourceResolverTest extends AssertJUnit
context = new TrackingMockery();
service = context.mock(IETLLIMSService.class);
+ generalInfoService = context.mock(IGeneralInformationService.class);
- final BeanFactory beanFactory = context.mock(BeanFactory.class);
- ServiceProviderTestWrapper.setApplicationContext(beanFactory);
- hierarchicalContentProvider = ServiceProviderTestWrapper.mock(context, IHierarchicalContentProvider.class);
+ hierarchicalContentProvider = context.mock(IHierarchicalContentProvider.class);
+ File root = new File(workingDirectory, "data-sets");
+ root.mkdirs();
+ simpleFileContentProvider = new SimpleFileContentProvider(root);
+
- resolverContext = new FtpPathResolverContext(SESSION_TOKEN, service, null);
+ resolverContext = new FtpPathResolverContext(SESSION_TOKEN, service, generalInfoService, null);
context.checking(new Expectations()
{
{
@@ -107,12 +169,32 @@ public class TemplateBasedDataSetResourceResolverTest extends AssertJUnit
will(returnValue(experiment));
}
});
+
+ ds1 =
+ new DataSetBuilder().experiment(experiment).code("ds1").type(DS_TYPE1)
+ .registrationDate(REGISTRATION_DATE).getDataSet();
+ File ds1Root = new File(root, ds1.getCode());
+ File ds1Original = new File(ds1Root, "original");
+ ds1Original.mkdirs();
+ FileUtilities.writeToFile(new File(ds1Original, "hello.txt"), "hello world");
+ FileUtilities.writeToFile(new File(ds1Original, "abc.txt"), "abcdefghijklmnopqrstuvwxyz");
+ FileUtilities.writeToFile(new File(ds1Original, "some.properties"), "a = alpha\nb = bets");
+ ds2 =
+ new DataSetBuilder().experiment(experiment).code("ds2").type(DS_TYPE2)
+ .registrationDate(REGISTRATION_DATE).getDataSet();
+ File ds2Root = new File(root, ds2.getCode());
+ File ds2Original = new File(ds2Root, "original2");
+ ds2Original.mkdirs();
+ FileUtilities.writeToFile(new File(ds2Original, "hello.txt"), "hello world");
+ File dataFolder = new File(ds2Original, "data");
+ dataFolder.mkdirs();
+ FileUtilities.writeToFile(new File(dataFolder, "a1.tsv"), "t\tlevel\n1.34\t2\n");
+ FileUtilities.writeToFile(new File(dataFolder, "a2.tsv"), "t\tlevel\n2.53\t3\n");
}
@AfterMethod(alwaysRun = true)
public void tearDown(Method m)
{
- ServiceProviderTestWrapper.restoreApplicationContext();
try
{
context.assertIsSatisfied();
@@ -121,97 +203,121 @@ public class TemplateBasedDataSetResourceResolverTest extends AssertJUnit
throw new Error(m.getName() + "() : ", t);
}
}
-
+
@Test
- public void testResolveSimpleTemplate()
+ public void testInvalidConfig()
{
FtpServerConfig config =
- new FtpServerConfigBuilder().withTemplate(SIMPLE_TEMPLATE).getConfig();
+ new FtpServerConfigBuilder().withTemplate(TEMPLATE_WITH_FILENAMES).showParentsAndChildren().getConfig();
+ try
+ {
+ new TemplateBasedDataSetResourceResolver(config);
+ fail("ConfigurationFailureException expected");
+ } catch (ConfigurationFailureException ex)
+ {
+ assertEquals("Template contains file name variable and the flag " +
+ "to show parents/children data sets is set.", ex.getMessage());
+ }
+ }
+
+ @Test
+ public void testWithParentsTopLevel()
+ {
+ FtpServerConfig config =
+ new FtpServerConfigBuilder().withTemplate(BIG_TEMPLATE).showParentsAndChildren().getConfig();
resolver = new TemplateBasedDataSetResourceResolver(config);
-
- final String dataSetCode = "dataSetCode";
-
- String path = EXP_ID + FtpConstants.FILE_SEPARATOR + dataSetCode;
-
- List<ExternalData> dataSets =
- Arrays.asList(createDataSet("randomCode", "randomType"),
- createDataSet(dataSetCode, DS_TYPE1),
- createDataSet("randomCode2", "randomType2"));
-
+ resolver.setContentProvider(simpleFileContentProvider);
+
+ ds1.setParents(Arrays.<ExternalData>asList(ds2));
+ final List<ExternalData> dataSets = Arrays.<ExternalData>asList(ds1);
+
prepareExperimentListExpectations(dataSets);
-
- context.checking(new Expectations()
- {
- {
- IHierarchicalContent content = getHierarchicalContentMock(dataSetCode);
- IHierarchicalContentNode rootNode = getHierarchicalRootNodeMock(dataSetCode);
-
- one(content).getNode(StringUtils.EMPTY);
- will(returnValue(rootNode));
-
- exactly(2).of(rootNode).isDirectory();
- will(returnValue(true));
-
- one(rootNode).getFile();
- will(throwException(new UnsupportedOperationException()));
-
- }
- });
-
+ prepareGetDataSetMetaData(ds1);
+ prepareListDataSetsByCode(ds2);
+
+ String dataSetPathElement = "DS-DS_TYPE1-ds1-" + RENDERED_REGISTRATION_DATE + "-A";
+ String path = EXP_ID + FtpConstants.FILE_SEPARATOR + dataSetPathElement;
FtpFile ftpFile = resolver.resolve(path, resolverContext);
-
- assertNotNull(ftpFile);
- assertEquals(dataSetCode, ftpFile.getName());
-
+
+ assertEquals(dataSetPathElement, ftpFile.getName());
+ assertEquals(path, ftpFile.getAbsolutePath());
+ assertEquals(true, ftpFile.isDirectory());
+ List<FtpFile> files = ftpFile.listFiles();
+ assertEquals("PARENT-DS-DS_TYPE2-ds2-" + RENDERED_REGISTRATION_DATE + "-A", files.get(0).getName());
+ assertEquals(true, files.get(0).isDirectory());
+ assertEquals("original", files.get(1).getName());
+ assertEquals(true, files.get(1).isDirectory());
+ assertEquals(2, files.size());
}
-
+
@Test
- public void testResolveNestedFilesWithSimpleTemplate()
+ public void testChildOfParent()
{
FtpServerConfig config =
- new FtpServerConfigBuilder().withTemplate(SIMPLE_TEMPLATE).getConfig();
+ new FtpServerConfigBuilder().withTemplate(BIG_TEMPLATE).showParentsAndChildren().getConfig();
resolver = new TemplateBasedDataSetResourceResolver(config);
-
- final String dataSetCode = "dataSetCode";
- final String subPath = "level1/level2/fileName.txt";
-
+ resolver.setContentProvider(simpleFileContentProvider);
+
+ ds1.setParents(Arrays.<ExternalData>asList(ds2));
+ ds2.setChildren(Arrays.<ExternalData>asList(ds1));
+ final List<ExternalData> dataSets = Arrays.<ExternalData>asList(ds1);
+
+ prepareExperimentListExpectations(dataSets);
+ prepareGetDataSetMetaData(ds1);
+ prepareListDataSetsByCode(ds2);
+ prepareGetDataSetMetaData(ds2);
+ prepareListDataSetsByCode(ds1);
+
+ String dataSetPathElement = "DS-DS_TYPE1-ds1-" + RENDERED_REGISTRATION_DATE + "-A";
+ String ds2AsParent = "PARENT-DS-DS_TYPE2-ds2-" + RENDERED_REGISTRATION_DATE + "-A";
String path =
- EXP_ID + FtpConstants.FILE_SEPARATOR + dataSetCode + FtpConstants.FILE_SEPARATOR
- + subPath;
+ EXP_ID + FtpConstants.FILE_SEPARATOR + dataSetPathElement
+ + FtpConstants.FILE_SEPARATOR + ds2AsParent;
+ FtpFile ftpFile = resolver.resolve(path, resolverContext);
+
+ assertEquals(ds2AsParent, ftpFile.getName());
+ assertEquals(path, ftpFile.getAbsolutePath());
+ assertEquals(true, ftpFile.isDirectory());
+ List<FtpFile> files = ftpFile.listFiles();
+ assertEquals("CHILD-DS-DS_TYPE1-ds1-" + RENDERED_REGISTRATION_DATE + "-A", files.get(0).getName());
+ assertEquals(true, files.get(0).isDirectory());
+ assertEquals("original2", files.get(1).getName());
+ assertEquals(true, files.get(1).isDirectory());
+ assertEquals(2, files.size());
+ }
+
+ @Test
+ public void testResolveNestedFilesWithSimpleTemplate() throws IOException
+ {
+ FtpServerConfig config =
+ new FtpServerConfigBuilder().withTemplate(SIMPLE_TEMPLATE).showParentsAndChildren()
+ .getConfig();
+ resolver = new TemplateBasedDataSetResourceResolver(config);
+ resolver.setContentProvider(simpleFileContentProvider);
- List<ExternalData> dataSets = Arrays.asList(createDataSet(dataSetCode, DS_TYPE1));
+ ds1.setParents(Arrays.<ExternalData> asList(ds2));
+ ds2.setChildren(Arrays.<ExternalData> asList(ds1));
+ final List<ExternalData> dataSets = Arrays.<ExternalData> asList(ds1);
prepareExperimentListExpectations(dataSets);
+ prepareGetDataSetMetaData(ds1);
+ prepareListDataSetsByCode(ds2);
+ prepareGetDataSetMetaData(ds2);
+ prepareListDataSetsByCode(ds1);
- context.checking(new Expectations()
- {
- {
- IHierarchicalContent content = getHierarchicalContentMock(dataSetCode);
- IHierarchicalContentNode mockNode =
- context.mock(IHierarchicalContentNode.class);
-
- allowing(content).getNode(subPath);
- will(returnValue(mockNode));
-
- one(mockNode).getRelativePath();
- will(returnValue(subPath));
-
- exactly(2).of(mockNode).isDirectory();
- will(returnValue(false));
-
- one(mockNode).getFileLength();
- will(returnValue(2L));
-
- one(mockNode).getFile();
- will(returnValue(null));
- }
- });
-
+ String path =
+ EXP_ID + FtpConstants.FILE_SEPARATOR + "ds1" + FtpConstants.FILE_SEPARATOR
+ + "PARENT-ds2" + FtpConstants.FILE_SEPARATOR + "original2"
+ + FtpConstants.FILE_SEPARATOR + "data" + FtpConstants.FILE_SEPARATOR
+ + "a1.tsv";
FtpFile ftpFile = resolver.resolve(path, resolverContext);
- assertNotNull(ftpFile);
- assertEquals("fileName.txt", ftpFile.getName());
- assertTrue(ftpFile.isFile());
+ assertEquals("a1.tsv", ftpFile.getName());
+ assertEquals(path, ftpFile.getAbsolutePath());
+ assertEquals(true, ftpFile.isFile());
+ InputStream fileContent = ftpFile.createInputStream(0);
+ assertEquals("[t\tlevel, 1.34\t2]", IOUtils.readLines(fileContent).toString());
+ fileContent.close();
}
@Test
@@ -220,57 +326,67 @@ public class TemplateBasedDataSetResourceResolverTest extends AssertJUnit
FtpServerConfig config =
new FtpServerConfigBuilder().withTemplate(SIMPLE_TEMPLATE).getConfig();
resolver = new TemplateBasedDataSetResourceResolver(config);
+ resolver.setContentProvider(hierarchicalContentProvider);
- final String dataSetCode = "dataSetCode";
final String subPath = "fileName.txt";
String path =
- EXP_ID + FtpConstants.FILE_SEPARATOR + dataSetCode + FtpConstants.FILE_SEPARATOR
+ EXP_ID + FtpConstants.FILE_SEPARATOR + ds1.getCode() + FtpConstants.FILE_SEPARATOR
+ subPath;
- List<ExternalData> dataSets = Arrays.asList(createDataSet(dataSetCode, DS_TYPE1));
+ List<ExternalData> dataSets = Arrays.<ExternalData>asList(ds1);
prepareExperimentListExpectations(dataSets);
context.checking(new Expectations()
{
{
- IHierarchicalContent content = getHierarchicalContentMock(dataSetCode);
+ IHierarchicalContent content = context.mock(IHierarchicalContent.class, ds1.getCode());
- one(hierarchicalContentProvider).asContent(dataSetCode);
+ one(hierarchicalContentProvider).asContent((ExternalData) ds1);
will(returnValue(content));
- one(content).close();
-
- IHierarchicalContentNode mockNode =
- context.mock(IHierarchicalContentNode.class);
-
- ByteArrayInputStream is = new ByteArrayInputStream(new byte[] {});
-
- allowing(content).getNode(subPath);
- will(returnValue(mockNode));
-
- one(mockNode).getRelativePath();
+
+ IHierarchicalContentNode rootNode =
+ context.mock(IHierarchicalContentNode.class, "root");
+ IHierarchicalContentNode fileNode =
+ context.mock(IHierarchicalContentNode.class, "file");
+
+ one(content).getRootNode();
+ will(returnValue(rootNode));
+
+ one(rootNode).getChildNodes();
+ will(returnValue(Arrays.asList(fileNode)));
+
+ one(fileNode).getName();
will(returnValue(subPath));
-
- exactly(2).of(mockNode).isDirectory();
+
+ one(fileNode).getRelativePath();
+ will(returnValue(subPath));
+
+ exactly(2).of(fileNode).isDirectory();
will(returnValue(false));
-
- one(mockNode).getFileLength();
+
+ one(fileNode).getFileLength();
will(returnValue(2L));
-
- one(mockNode).getFile();
+
+ one(fileNode).getFile();
will(returnValue(null));
-
- one(mockNode).getInputStream();
+
+ one(fileNode).getInputStream();
+ ByteArrayInputStream is = new ByteArrayInputStream(new byte[] {});
will(returnValue(is));
-
+
+ allowing(content).getNode(subPath);
+ will(returnValue(fileNode));
+
+ atLeast(1).of(content).close();
}
});
FtpFile ftpFile = resolver.resolve(path, resolverContext);
assertNotNull(ftpFile);
- assertEquals("fileName.txt", ftpFile.getName());
+ assertEquals(subPath, ftpFile.getName());
assertTrue(ftpFile.isFile());
InputStream fileContent = ftpFile.createInputStream(0);
// this call will also close the IHierarchicalContent
@@ -278,59 +394,33 @@ public class TemplateBasedDataSetResourceResolverTest extends AssertJUnit
}
@Test
- public void testFileFilters()
+ public void testSubPathAndFileFilters()
{
- String filterPattern = "[^.]*\\.txt";
FtpServerConfig config =
new FtpServerConfigBuilder().withTemplate(TEMPLATE_WITH_FILENAMES)
- .withFileListFilter(DS_TYPE1, filterPattern).getConfig();
+ .withFileListSubPath(DS_TYPE1, "orig[^/]*")
+ .withFileListFilter(DS_TYPE1, "[^.]*\\.txt").getConfig();
resolver = new TemplateBasedDataSetResourceResolver(config);
-
- final String dataSetCode = "dataSetCode";
- final String dataSetCode2 = "dataSetCode2";
+ resolver.setContentProvider(simpleFileContentProvider);
List<ExternalData> dataSets =
- Arrays.asList(createDataSet(dataSetCode, DS_TYPE1),
- createDataSet(dataSetCode2, DS_TYPE2));
+ Arrays.<ExternalData>asList(ds1, ds2);
prepareExperimentListExpectations(dataSets);
- context.checking(new Expectations()
- {
- {
- IHierarchicalContentNode rootNode = getHierarchicalRootNodeMock(dataSetCode);
- allowing(rootNode).isDirectory();
- will(returnValue(true));
-
- allowing(rootNode).getChildNodes();
- List<IHierarchicalContentNode> children =
- createFileNodeMocks("file.exe", "file.txt", "file");
- will(returnValue(children));
-
- IHierarchicalContentNode rootNode2 = getHierarchicalRootNodeMock(dataSetCode2);
- allowing(rootNode2).isDirectory();
- will(returnValue(true));
-
- allowing(rootNode2).getChildNodes();
- List<IHierarchicalContentNode> children2 =
- createFileNodeMocks("file.jar", "file.sh");
- will(returnValue(children2));
- }
- });
-
List<FtpFile> files =
resolver.listExperimentChildrenPaths(experiment, EXP_ID, resolverContext);
assertNotNull(files);
- assertEquals(3, files.size());
-
+ assertEquals("DS-DS_TYPE1-abc.txt-A", files.get(0).getName());
assertTrue(files.get(0).isFile());
- assertEquals("DS-DS_TYPE1-file.txt-A", files.get(0).getName());
+ assertEquals(26, files.get(0).getSize());
+ assertEquals("DS-DS_TYPE1-hello.txt-A", files.get(1).getName());
assertTrue(files.get(1).isFile());
- assertEquals("DS-DS_TYPE2-file.jar-B", files.get(1).getName());
- assertTrue(files.get(2).isFile());
- assertEquals("DS-DS_TYPE2-file.sh-B", files.get(2).getName());
-
+ assertEquals(11, files.get(1).getSize());
+ assertEquals("DS-DS_TYPE2-original2-B", files.get(2).getName());
+ assertTrue(files.get(2).isDirectory());
+ assertEquals(3, files.size());
}
@@ -343,45 +433,72 @@ public class TemplateBasedDataSetResourceResolverTest extends AssertJUnit
new TechId(experiment));
will(returnValue(dataSets));
- for (ExternalData dataSet : dataSets)
- {
- String mockName = getHierarchicalContentMockName(dataSet.getCode());
- IHierarchicalContent content =
- context.mock(IHierarchicalContent.class, mockName);
- one(hierarchicalContentProvider).asContent(dataSet);
- will(returnValue(content));
- one(content).close();
-
- String rootMockName = getRootNodeMockName(dataSet.getCode());
- IHierarchicalContentNode rootNode =
- context.mock(IHierarchicalContentNode.class, rootMockName);
- allowing(content).getRootNode();
- will(returnValue(rootNode));
-
- allowing(rootNode).getRelativePath();
- will(returnValue(StringUtils.EMPTY));
-
- allowing(rootNode).getName();
- will(returnValue(StringUtils.EMPTY));
- }
+// for (ExternalData dataSet : dataSets)
+// {
+// String mockName = getHierarchicalContentMockName(dataSet.getCode());
+// IHierarchicalContent content =
+// context.mock(IHierarchicalContent.class, mockName);
+// one(hierarchicalContentProvider).asContent(dataSet);
+// will(returnValue(content));
+// one(content).close();
+//
+// String rootMockName = getRootNodeMockName(dataSet.getCode());
+// IHierarchicalContentNode rootNode =
+// context.mock(IHierarchicalContentNode.class, rootMockName);
+// allowing(content).getRootNode();
+// will(returnValue(rootNode));
+//
+// allowing(rootNode).getRelativePath();
+// will(returnValue(StringUtils.EMPTY));
+//
+// allowing(rootNode).getName();
+// will(returnValue(StringUtils.EMPTY));
+// }
}
});
}
- private String getHierarchicalContentMockName(String dataSetCode)
+ private void prepareGetDataSetMetaData(final ExternalData... dataSets)
{
- return dataSetCode;
+ final List<String> codes = extractCodes(dataSets);
+ final List<ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.DataSet> translateDataSets =
+ Translator.translate(Arrays.asList(dataSets),
+ EnumSet.of(Connections.PARENTS, Connections.CHILDREN));
+ context.checking(new Expectations()
+ {
+ {
+ one(generalInfoService).getDataSetMetaData(SESSION_TOKEN, codes);
+ will(returnValue(translateDataSets));
+ }
+ });
}
- private String getRootNodeMockName(String dataSetCode)
+ private void prepareListDataSetsByCode(final ExternalData... dataSets)
+ {
+ final List<String> codes = extractCodes(dataSets);
+ context.checking(new Expectations()
+ {
+ {
+ one(service).listDataSetsByCode(SESSION_TOKEN, codes);
+ will(returnValue(Arrays.asList(dataSets)));
+ }
+ });
+
+ }
+
+ private List<String> extractCodes(final ExternalData... dataSets)
{
- return dataSetCode + "-rootNode";
+ final List<String> codes = new ArrayList<String>();
+ for (ExternalData dataSet : dataSets)
+ {
+ codes.add(dataSet.getCode());
+ }
+ return codes;
}
-
- private IHierarchicalContentNode getHierarchicalRootNodeMock(String dataSetCode)
+
+ private String getHierarchicalContentMockName(String dataSetCode)
{
- String mockName = getRootNodeMockName(dataSetCode);
- return context.getMock(mockName, IHierarchicalContentNode.class);
+ return dataSetCode;
}
protected IHierarchicalContent getHierarchicalContentMock(String dataSetCode)
@@ -390,51 +507,4 @@ public class TemplateBasedDataSetResourceResolverTest extends AssertJUnit
return context.getMock(mockName, IHierarchicalContent.class);
}
- private ExternalData createDataSet(String dataSetCode, String dataSetType)
- {
- return createDataSet(dataSetCode, dataSetType, new Date());
- }
-
- private ExternalData createDataSet(String dataSetCode, String dataSetType, Date registrationDate)
- {
- ExternalData result = new ExternalData();
- result.setCode(dataSetCode);
- DataSetType type = new DataSetType(dataSetType);
- result.setDataSetType(type);
- result.setRegistrationDate(registrationDate);
- return result;
- }
-
- private List<IHierarchicalContentNode> createFileNodeMocks(final String... fileNames)
- {
- final List<IHierarchicalContentNode> result = new ArrayList<IHierarchicalContentNode>();
-
- context.checking(new Expectations()
- {
- {
- for (String fileName : fileNames)
- {
- IHierarchicalContentNode mockNode =
- context.mock(IHierarchicalContentNode.class, fileName);
- result.add(mockNode);
-
- allowing(mockNode).getFileLength();
- will(returnValue(10L));
-
- allowing(mockNode).getFile();
- will(throwException(new UnsupportedOperationException()));
-
- allowing(mockNode).getName();
- will(returnValue(fileName));
-
- allowing(mockNode).isDirectory();
- will(returnValue(false));
-
- allowing(mockNode).getRelativePath();
- will(returnValue(fileName));
- }
- }
- });
- return result;
- }
}
--
GitLab