LMS-2286 handle uploading to cifex even for container data sets....

LMS-2286 handle uploading to cifex even for container data sets. IHierachicalContentNode by method getLastModified().

SVN: 21759

common/source/java/ch/systemsx/cisd/common/io/DefaultFileBasedHierarchicalContentNode.java

@@ -83,6 +83,11 @@ class DefaultFileBasedHierarchicalContentNode extends AbstractHierarchicalConten
         return file.isDirectory();
     }
 
+    public long getLastModified()
+    {
+        return file.lastModified();
+    }
+
     @Override
     public List<IHierarchicalContentNode> doGetChildNodes()
     {

common/source/java/ch/systemsx/cisd/common/io/HDF5ContainerBasedHierarchicalContentNode.java

@@ -187,6 +187,11 @@ public class HDF5ContainerBasedHierarchicalContentNode extends
             return true;
         }
 
+        public long getLastModified()
+        {
+            return file.lastModified();
+        }
+
         public File getFile() throws UnsupportedOperationException
         {
             throw new UnsupportedOperationException("This is not a normal directory node.");
@@ -267,6 +272,11 @@ public class HDF5ContainerBasedHierarchicalContentNode extends
             return false;
         }
 
+        public long getLastModified()
+        {
+            return file.lastModified();
+        }
+
         public File getFile() throws UnsupportedOperationException
         {
             throw new UnsupportedOperationException("This is not a normal file node.");

common/source/java/ch/systemsx/cisd/common/io/IHierarchicalContentNode.java

@@ -51,6 +51,14 @@ public interface IHierarchicalContentNode
      * otherwise.
      */
     boolean isDirectory();
+    
+    /**
+     * Returns the time this node or the persistent object containing this node has been modified.
+     * 
+     * @return A long value representing the time of last modification, measured in milliseconds
+     *         since the epoch (00:00:00 GMT, January 1, 1970).
+     */
+    long getLastModified();
 
     /**
      * List of child nodes of this node.

common/source/java/ch/systemsx/cisd/common/io/VirtualHierarchicalContent.java

@@ -391,6 +391,11 @@ class VirtualHierarchicalContent implements IHierarchicalContent
             return lastNode().isDirectory();
         }
 
+        public long getLastModified()
+        {
+            return lastNode().getLastModified();
+        }
+
         public List<IHierarchicalContentNode> getChildNodes() throws UnsupportedOperationException
         {
             IVirtualNodeListMerger listMerger = nodeMergerFactory.createNodeListMerger();

common/sourceTest/java/ch/systemsx/cisd/common/io/AbstractHierarchicalContentNodeTest.java

@@ -152,6 +152,11 @@ public class AbstractHierarchicalContentNodeTest extends AssertJUnit
                     return false;
                 }
 
+                public long getLastModified()
+                {
+                    return 0;
+                }
+
                 @Override
                 protected InputStream doGetInputStream()
                 {
@@ -209,6 +214,11 @@ public class AbstractHierarchicalContentNodeTest extends AssertJUnit
                     return false;
                 }
 
+                public long getLastModified()
+                {
+                    return 0;
+                }
+
                 @Override
                 protected InputStream doGetInputStream()
                 {

common/sourceTest/java/ch/systemsx/cisd/common/io/DefaultFileBasedHierarchicalContentTest.java

@@ -491,8 +491,9 @@ public class DefaultFileBasedHierarchicalContentTest extends AbstractFileSystemT
             assertEquals("This is not a normal file node.", ex.getMessage());
         }
         // file info access
-        assertEquals(expectedFile.getName(), fileNode.getName());
-        assertEquals(expectedFile.length(), fileNode.getFileLength());
+        assertEquals("File: " + expectedFile, expectedFile.getName(), fileNode.getName());
+        assertEquals("File: " + expectedFile, expectedFile.length(), fileNode.getFileLength());
+        assertEquals("File: " + expectedFile, expectedFile.lastModified(), fileNode.getLastModified());
 
         final String expectedFileData = expectedFile.getName() + " data";
         // check random access to file content
@@ -571,6 +572,11 @@ public class DefaultFileBasedHierarchicalContentTest extends AbstractFileSystemT
                     return file.exists();
                 }
 
+                public long getLastModified()
+                {
+                    return file.lastModified();
+                }
+
                 public long getFileLength() throws UnsupportedOperationException
                 {
                     throw new UnsupportedOperationException(METHOD_NOT_IMPLEMENTED);

common/sourceTest/java/ch/systemsx/cisd/common/io/VirtualNodeTest.java

@@ -271,6 +271,23 @@ public class VirtualNodeTest extends AbstractFileSystemTestCase
 
         context.assertIsSatisfied();
     }
+    
+    @Test
+    public void testLastModified()
+    {
+        final IHierarchicalContentNode virtualNode = createVirtualNode();
+        context.checking(new Expectations()
+            {
+                {
+                    one(node1).getLastModified();
+                    will(returnValue(42L));
+                }
+            });
+        
+        assertEquals(42, virtualNode.getLastModified());
+        
+        context.assertIsSatisfied();
+    }
 
     @Test
     public void testGetChildNodes()

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/UploadingCommand.java

@@ -18,7 +18,6 @@ package ch.systemsx.cisd.openbis.dss.generic.server;
 
 import java.io.ByteArrayInputStream;
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -40,6 +39,8 @@ import ch.systemsx.cisd.cifex.rpc.client.ICIFEXComponent;
 import ch.systemsx.cisd.cifex.rpc.client.ICIFEXUploader;
 import ch.systemsx.cisd.cifex.rpc.client.gui.IProgressListener;
 import ch.systemsx.cisd.cifex.shared.basic.Constants;
+import ch.systemsx.cisd.common.io.IHierarchicalContent;
+import ch.systemsx.cisd.common.io.IHierarchicalContentNode;
 import ch.systemsx.cisd.common.logging.LogCategory;
 import ch.systemsx.cisd.common.logging.LogFactory;
 import ch.systemsx.cisd.common.mail.IMailClient;
@@ -48,7 +49,8 @@ import ch.systemsx.cisd.common.mail.MailClientParameters;
 import ch.systemsx.cisd.common.types.BooleanOrUnknown;
 import ch.systemsx.cisd.common.utilities.TokenGenerator;
 import ch.systemsx.cisd.openbis.dss.generic.shared.IDataSetDirectoryProvider;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSet;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
 import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Experiment;
 import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ExternalData;
 import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IEntityProperty;
@@ -57,8 +59,6 @@ import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Project;
 import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Sample;
 import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Space;
 import ch.systemsx.cisd.openbis.generic.shared.dto.DataSetUploadContext;
-import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
-import ch.systemsx.cisd.openbis.generic.shared.translator.DataSetTranslator;
 
 import de.schlichtherle.util.zip.ZipEntry;
 import de.schlichtherle.util.zip.ZipOutputStream;
@@ -285,6 +285,8 @@ class UploadingCommand implements IDataSetCommand
     @Private
     boolean deleteAfterUploading = true;
 
+    @Private transient IHierarchicalContentProvider hierarchicalContentProvider;
+
     UploadingCommand(ICIFEXRPCServiceFactory cifexServiceFactory,
             MailClientParameters mailClientParameters, List<ExternalData> dataSets,
             DataSetUploadContext context, String cifexAdminUserOrNull,
@@ -412,37 +414,34 @@ class UploadingCommand implements IDataSetCommand
             zipOutputStream = new ZipOutputStream(outputStream);
             for (ExternalData externalData : dataSets)
             {
-                DataSet dataSet = externalData.tryGetAsDataSet();
-                assert dataSet != null : "container datasets are currently not supported by DSS client";
-
-                DatasetDescription dataSetDescription =
-                        DataSetTranslator.translateToDescription(dataSet);
-                File dataSetFile = dataSetDirectoryProvider.getDataSetDirectory(dataSetDescription);
-                if (dataSetFile.exists() == false)
-                {
-                    notificationLog.error("Data set '" + dataSetFile + "' does not exist.");
-                    return false;
-                }
-                String newRootPath = createRootPath(dataSet);
+                String newRootPath = createRootPath(externalData) + "/";
                 try
                 {
-                    addEntry(zipOutputStream, newRootPath + "/meta-data.tsv",
+                    addEntry(zipOutputStream, newRootPath + "meta-data.tsv",
                             System.currentTimeMillis(),
-                            new ByteArrayInputStream(createMetaData(dataSet).getBytes()));
+                            new ByteArrayInputStream(createMetaData(externalData).getBytes()));
                 } catch (IOException ex)
                 {
                     notificationLog.error(
-                            "Couldn't add meta date for data set '" + dataSet.getCode()
+                            "Couldn't add meta date for data set '" + externalData.getCode()
                                     + "' to zip file.", ex);
                     return false;
                 }
+                IHierarchicalContent root = null;
+                try
+                {
+                    root = getHierarchicalContentProvider().asContent(externalData.getCode());
+                } catch (Exception ex)
+                {
+                    notificationLog.error("Data set " + externalData.getCode() + " does not exist.", ex);
+                    return false;
+                }
                 try
                 {
-                    addTo(zipOutputStream, dataSetFile.getCanonicalPath().length(), newRootPath,
-                            dataSetFile);
+                    addTo(zipOutputStream, newRootPath, root.getRootNode());
                 } catch (IOException ex)
                 {
-                    notificationLog.error("Couldn't add data set '" + dataSetFile
+                    notificationLog.error("Couldn't add data set '" + externalData.getCode()
                             + "' to zip file.", ex);
                     return false;
                 }
@@ -466,8 +465,35 @@ class UploadingCommand implements IDataSetCommand
             }
         }
     }
+    
+    private IHierarchicalContentProvider getHierarchicalContentProvider()
+    {
+        if (hierarchicalContentProvider == null)
+        {
+            hierarchicalContentProvider = ServiceProvider.getHierarchicalContentProvider();
+        }
+        return hierarchicalContentProvider;
+    }
+    
+    private void addTo(ZipOutputStream zipOutputStream, String newRootPath,
+            IHierarchicalContentNode node) throws IOException
+    {
+        if (node.isDirectory())
+        {
+            List<IHierarchicalContentNode> childNodes = node.getChildNodes();
+            for (IHierarchicalContentNode childNode : childNodes)
+            {
+                addTo(zipOutputStream, newRootPath, childNode);
+            }
+        } else
+        {
+            addEntry(zipOutputStream, newRootPath + node.getRelativePath(), node.getLastModified(),
+                    node.getInputStream());
+        }
+    }
 
-    private String createMetaData(DataSet dataSet)
+    @SuppressWarnings("deprecation")
+    private String createMetaData(ExternalData dataSet)
     {
         MetaDataBuilder builder = new MetaDataBuilder();
         builder.dataSet("code", dataSet.getCode());
@@ -525,24 +551,6 @@ class UploadingCommand implements IDataSetCommand
                 + "/" + (sample == null ? "" : sample.getCode() + "/") + dataSet.getCode();
     }
 
-    private void addTo(ZipOutputStream zipOutputStream, int oldRootPathLength, String newRootPath,
-            File file) throws IOException
-    {
-        if (file.isFile())
-        {
-            String zipEntryPath =
-                    newRootPath + file.getCanonicalPath().substring(oldRootPathLength);
-            addEntry(zipOutputStream, zipEntryPath, file.lastModified(), new FileInputStream(file));
-        } else
-        {
-            File[] files = file.listFiles();
-            for (File childFile : files)
-            {
-                addTo(zipOutputStream, oldRootPathLength, newRootPath, childFile);
-            }
-        }
-    }
-
     private void addEntry(ZipOutputStream zipOutputStream, String zipEntryPath, long lastModified,
             InputStream in) throws IOException
     {

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/content/PathInfoProviderBasedHierarchicalContent.java

@@ -207,6 +207,11 @@ class PathInfoProviderBasedHierarchicalContent implements IHierarchicalContent
             return pathInfo.isDirectory();
         }
 
+        public long getLastModified()
+        {
+            return System.currentTimeMillis();
+        }
+
         @Override
         protected String doGetRelativePath()
         {

datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/UploadingCommandTest.java

@@ -47,9 +47,13 @@ import ch.systemsx.cisd.cifex.rpc.client.ICIFEXUploader;
 import ch.systemsx.cisd.cifex.rpc.client.gui.IProgressListener;
 import ch.systemsx.cisd.common.exceptions.AuthorizationFailureException;
 import ch.systemsx.cisd.common.filesystem.FileUtilities;
+import ch.systemsx.cisd.common.io.DefaultFileBasedHierarchicalContentFactory;
+import ch.systemsx.cisd.common.io.IHierarchicalContent;
 import ch.systemsx.cisd.common.logging.BufferedAppender;
 import ch.systemsx.cisd.common.mail.MailClientParameters;
+import ch.systemsx.cisd.common.utilities.IDelegatedAction;
 import ch.systemsx.cisd.openbis.dss.generic.shared.IDataSetDirectoryProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
 import ch.systemsx.cisd.openbis.dss.generic.shared.IShareIdManager;
 import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ExternalData;
 import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IDatasetLocation;
@@ -74,11 +78,48 @@ import ch.systemsx.cisd.openbis.generic.shared.translator.ExperimentTranslator;
 @Friend(toClasses = UploadingCommand.class)
 public class UploadingCommandTest extends AssertJUnit
 {
+
     private static final File TEST_FOLDER = new File("targets/upload-test");
 
     private static final File STORE = new File(TEST_FOLDER, "store");
 
     private static final String SHARE_ID = "share-id";
+    
+    private static final String LOCATION_PREFIX = "ds";
+
+    private static final IHierarchicalContentProvider HIERARCHICAL_CONTENT_PROVIDER =
+            new IHierarchicalContentProvider()
+                {
+                    private DefaultFileBasedHierarchicalContentFactory hierarchicalContentFactory =
+                            new DefaultFileBasedHierarchicalContentFactory();
+
+                    public IHierarchicalContent asContent(File datasetDirectory)
+                    {
+                        return hierarchicalContentFactory.asHierarchicalContent(datasetDirectory,
+                                IDelegatedAction.DO_NOTHING);
+                    }
+
+                    public IHierarchicalContent asContent(IDatasetLocation datasetLocation)
+                    {
+                        return getContent(datasetLocation.getDataSetLocation());
+                    }
+
+                    public IHierarchicalContent getContent(String location)
+                    {
+                        return asContent(new File(new File(STORE, SHARE_ID), location));
+                    }
+
+                    public IHierarchicalContent asContent(ExternalData dataSet)
+                    {
+                        return getContent(dataSet.getCode());
+                    }
+
+                    public IHierarchicalContent asContent(String dataSetCode)
+                            throws IllegalArgumentException
+                    {
+                        return getContent(LOCATION_PREFIX + dataSetCode);
+                    }
+                };
 
     private static final class MockDataSetDirectoryProvider implements IDataSetDirectoryProvider
     {
@@ -110,9 +151,9 @@ public class UploadingCommandTest extends AssertJUnit
 
     private static final File EMAILS = new File(TEST_FOLDER, "emails");
 
-    private static final String LOCATION1 = "ds1";
+    private static final String LOCATION1 = LOCATION_PREFIX + "1";
 
-    private static final String LOCATION2 = "ds2";
+    private static final String LOCATION2 = LOCATION_PREFIX + "2";
 
     private static final String SESSION_TOKEN = "session42";
 
@@ -183,33 +224,36 @@ public class UploadingCommandTest extends AssertJUnit
         createTestData(LOCATION1);
         ds2 = createTestData(LOCATION2);
         ExternalData dataSet1 =
-                DataSetTranslator.translate(createDataSet("1", LOCATION1), "?",
+                DataSetTranslator.translate(createDataSet("1"), "?",
                         ExperimentTranslator.LoadableFields.PROPERTIES);
         System.out.println("ds1:" + dataSet1.getExperiment().getProperties());
         ExternalData dataSet2 =
-                DataSetTranslator.translate(createDataSet("2", LOCATION2), "?",
+                DataSetTranslator.translate(createDataSet("2"), "?",
                         ExperimentTranslator.LoadableFields.PROPERTIES);
         dataSets = Arrays.<ExternalData> asList(dataSet1, dataSet2);
         command =
                 new UploadingCommand(factory, mailClientParameters, dataSets, uploadContext, null,
                         null);
+        command.hierarchicalContentProvider = HIERARCHICAL_CONTENT_PROVIDER;
         commandAdminSession =
                 new UploadingCommand(factory, mailClientParameters, dataSets,
                         uploadContextNoPasswordAuthenticated, "admin", "admpwd");
+        commandAdminSession.hierarchicalContentProvider = HIERARCHICAL_CONTENT_PROVIDER;
         commandAdminSessionNotAuthenticated =
                 new UploadingCommand(factory, mailClientParameters, dataSets,
                         uploadContextNoPasswordNotAuthenticated, "admin", "admpwd");
+        commandAdminSessionNotAuthenticated.hierarchicalContentProvider = HIERARCHICAL_CONTENT_PROVIDER;
         command.deleteAfterUploading = false;
         commandAdminSession.deleteAfterUploading = false;
         commandAdminSessionNotAuthenticated.deleteAfterUploading = false;
     }
 
-    private ExternalDataPE createDataSet(String code, String location)
+    private ExternalDataPE createDataSet(String code)
     {
         ExternalDataPE externalData = new ExternalDataPE();
         externalData.setCode(code);
         externalData.setShareId(SHARE_ID);
-        externalData.setLocation(location);
+        externalData.setLocation(LOCATION_PREFIX + code);
         externalData.setDerived(true); // measured == (derived == false)
         DataSetTypePE dataSetTypePE = new DataSetTypePE();
         dataSetTypePE.setCode("D");
@@ -283,7 +327,8 @@ public class UploadingCommandTest extends AssertJUnit
 
     private String getNormalizedLogContent()
     {
-        return logRecorder.getLogContent().replaceAll(" [^ ]*\\.zip", " <zipfile>");
+        return logRecorder.getLogContent().replaceAll(" [^ ]*\\.zip", " <zipfile>")
+                .replaceAll("\n\ta.*\\)", "").replaceAll("[^:]*" + TEST_FOLDER.getPath(), "");
     }
 
     @AfterMethod
@@ -438,8 +483,9 @@ public class UploadingCommandTest extends AssertJUnit
         command.execute(directoryProvider);
 
         checkEmail("Couldn't create zip file");
-        assertEquals("ERROR NOTIFY.UploadingCommand"
-                + " - Data set 'targets/upload-test/store/share-id/ds2' does not exist."
+        assertEquals("ERROR NOTIFY.UploadingCommand - Data set 2 does not exist."
+                + OSUtilities.LINE_SEPARATOR
+                + "java.lang.IllegalArgumentException:/store/share-id/ds2 doesn't exist"
                 + OSUtilities.LINE_SEPARATOR + INFO_MAIL_PREFIX
                 + "Sending message from 'a@bc.de' to recipients '[user@bc.de]'",
                 getNormalizedLogContent());

openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/DataSetTable.java

@@ -23,7 +23,6 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -129,7 +128,7 @@ public final class DataSetTable extends AbstractDataSetBusinessObject implements
         List<String> notAvailableDatasets = new ArrayList<String>();
         for (DataPE dataSet : datasets)
         {
-            if (dataSet.isAvailable() == false)
+            if (dataSet.isExternalData() && dataSet.isAvailable() == false)
             {
                 notAvailableDatasets.add(dataSet.getCode());
             }
@@ -327,21 +326,20 @@ public final class DataSetTable extends AbstractDataSetBusinessObject implements
     public String uploadLoadedDataSetsToCIFEX(DataSetUploadContext uploadContext)
     {
         assertDatasetsAreAvailable(dataSets);
-        Map<DataStorePE, List<ExternalDataPE>> map = groupExternalDataByDataStores();
+        Map<DataStorePE, List<DataPE>> map = groupDataByDataStores();
         assertDataSetsAreKnown(map);
-        assertNoContainerDataSets(map);
         uploadContext.setUserEMail(session.getPrincipal().getEmail());
         uploadContext.setSessionUserID(session.getUserName());
         if (StringUtils.isBlank(uploadContext.getComment()))
         {
             uploadContext.setComment(createUploadComment(dataSets));
         }
-        List<ExternalDataPE> dataSetsWithUnknownDSS = new ArrayList<ExternalDataPE>();
-        for (Map.Entry<DataStorePE, List<ExternalDataPE>> entry : map.entrySet())
+        List<DataPE> dataSetsWithUnknownDSS = new ArrayList<DataPE>();
+        for (Map.Entry<DataStorePE, List<DataPE>> entry : map.entrySet())
         {
             DataStorePE dataStore = entry.getKey();
-            List<ExternalDataPE> externalDatas = entry.getValue();
-            for (ExternalDataPE dataSet : externalDatas)
+            List<DataPE> dataSetsOfStore = entry.getValue();
+            for (DataPE dataSet : dataSetsOfStore)
             {
                 HibernateUtils.initialize(dataSet.getParents());
                 HibernateUtils.initialize(dataSet.getProperties());
@@ -359,35 +357,35 @@ public final class DataSetTable extends AbstractDataSetBusinessObject implements
             }
             if (StringUtils.isBlank(dataStore.getRemoteUrl()))
             {
-                dataSetsWithUnknownDSS.addAll(externalDatas);
+                dataSetsWithUnknownDSS.addAll(dataSetsOfStore);
             } else
             {
-                uploadDataSetsToCIFEX(dataStore, externalDatas, uploadContext);
+                uploadDataSetsToCIFEX(dataStore, dataSetsOfStore, uploadContext);
             }
         }
         StringBuilder builder = new StringBuilder();
         if (dataSetsWithUnknownDSS.isEmpty() == false)
         {
             builder.append("The following data sets couldn't been uploaded because of unkown data store:");
-            for (ExternalDataPE externalDataPE : dataSetsWithUnknownDSS)
+            for (DataPE dataSet : dataSetsWithUnknownDSS)
             {
-                builder.append(' ').append(externalDataPE.getCode());
+                builder.append(' ').append(dataSet.getCode());
             }
         }
         return builder.toString();
     }
 
-    private void assertDataSetsAreKnown(Map<DataStorePE, List<ExternalDataPE>> map)
+    private <D extends DataPE> void assertDataSetsAreKnown(Map<DataStorePE, List<D>> map)
     {
         // Set<String> knownLocations = new LinkedHashSet<String>();
         List<String> unknownDataSets = new ArrayList<String>();
-        for (Map.Entry<DataStorePE, List<ExternalDataPE>> entry : map.entrySet())
+        for (Map.Entry<DataStorePE, List<D>> entry : map.entrySet())
         {
             DataStorePE dataStore = entry.getKey();
-            List<ExternalDataPE> externalDatas = entry.getValue();
+            List<ExternalDataPE> externalDatas = filterRealDataSets(entry.getValue());
             Set<String> knownLocations =
                     getKnownDataSets(dataStore, createDatasetDescriptions(externalDatas));
-            for (ExternalDataPE dataSet : entry.getValue())
+            for (ExternalDataPE dataSet : externalDatas)
             {
                 if (dataSet.getStatus() == DataSetArchivingStatus.ARCHIVED)
                 {
@@ -407,34 +405,44 @@ public final class DataSetTable extends AbstractDataSetBusinessObject implements
                             + unknownDataSets);
         }
     }
-
-    private void assertNoContainerDataSets(Map<DataStorePE, List<ExternalDataPE>> map)
+    
+    private List<ExternalDataPE> filterRealDataSets(List<? extends DataPE> mixedDataSets)
     {
-        Set<String> containerDataSets = new LinkedHashSet<String>();
-        for (Map.Entry<DataStorePE, List<ExternalDataPE>> entry : map.entrySet())
+        List<ExternalDataPE> realDataSets = new ArrayList<ExternalDataPE>();
+        for (DataPE dataSet : mixedDataSets)
         {
-            for (ExternalDataPE dataSet : entry.getValue())
+            if (dataSet instanceof ExternalDataPE)
             {
-                if (dataSet.isContainer())
-                {
-                    containerDataSets.add(dataSet.getCode());
-                }
+                realDataSets.add((ExternalDataPE) dataSet);
             }
         }
+        return realDataSets;
+    }
 
-        if (false == containerDataSets.isEmpty())
+    /** groups data sets by data stores */
+    private Map<DataStorePE, List<DataPE>> groupDataByDataStores()
+    {
+        Map<DataStorePE, List<DataPE>> map =
+                new LinkedHashMap<DataStorePE, List<DataPE>>();
+        for (DataPE dataSet : dataSets)
         {
-            throw new UserFailureException(
-                    "The following data sets are container data sets and cannot be uploaded to CIFEX. "
-                            + containerDataSets);
+            DataStorePE dataStore = dataSet.getDataStore();
+            List<DataPE> list = map.get(dataStore);
+            if (list == null)
+            {
+                list = new ArrayList<DataPE>();
+                map.put(dataStore, list);
+            }
+            list.add(dataSet);
         }
+        return map;
     }
 
-    /** groups data sets by data stores filtering out virtual data sets */
+    /** groups data sets by data stores filtering out container data sets */
     private Map<DataStorePE, List<ExternalDataPE>> groupExternalDataByDataStores()
     {
         Map<DataStorePE, List<ExternalDataPE>> map =
-                new LinkedHashMap<DataStorePE, List<ExternalDataPE>>();
+            new LinkedHashMap<DataStorePE, List<ExternalDataPE>>();
         for (DataPE dataSet : dataSets)
         {
             if (dataSet.isExternalData())
@@ -451,7 +459,7 @@ public final class DataSetTable extends AbstractDataSetBusinessObject implements
         }
         return map;
     }
-
+    
     /** groups all data sets (both virtual and non-virtual) by data stores */
     private Map<DataStorePE, List<DataPE>> groupDataSetsByDataStores()
     {
@@ -470,7 +478,7 @@ public final class DataSetTable extends AbstractDataSetBusinessObject implements
         return map;
     }
 
-    private void uploadDataSetsToCIFEX(DataStorePE dataStore, List<ExternalDataPE> list,
+    private void uploadDataSetsToCIFEX(DataStorePE dataStore, List<DataPE> list,
             DataSetUploadContext context)
     {
         IDataStoreService service = dssFactory.create(dataStore.getRemoteUrl());