LMS-1219 Extend FlowLineFeeder

SVN: 12876

deep_sequencing_unit/source/java/ch/ethz/bsse/cisd/dsu/dss/FlowLineFeeder.java

@@ -19,7 +19,9 @@ package ch.ethz.bsse.cisd.dsu.dss;
 import java.io.File;
 import java.text.MessageFormat;
 import java.util.ArrayList;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Properties;
 
 import ch.systemsx.cisd.common.TimingParameters;
@@ -27,11 +29,20 @@ import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
 import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
 import ch.systemsx.cisd.common.filesystem.FastRecursiveHardLinkMaker;
 import ch.systemsx.cisd.common.filesystem.FileOperations;
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
 import ch.systemsx.cisd.common.filesystem.IFileOperations;
 import ch.systemsx.cisd.common.filesystem.IImmutableCopier;
 import ch.systemsx.cisd.common.utilities.PropertyUtils;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
 import ch.systemsx.cisd.openbis.dss.generic.shared.IPostRegistrationDatasetHandler;
 import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
+import ch.systemsx.cisd.openbis.generic.client.web.client.exception.UserFailureException;
+import ch.systemsx.cisd.openbis.generic.shared.basic.TechId;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IEntityProperty;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ListSampleCriteria;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Sample;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifier;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifierFactory;
 
 /**
  * Post registration data set handler which makes a hard-link copy of all flow-line files
@@ -44,13 +55,15 @@ class FlowLineFeeder implements IPostRegistrationDatasetHandler
     static final String FLOW_LINE_DROP_BOX_TEMPLATE = "flow-line-drop-box-template";
     static final String FILE_TYPE = ".srf";
     
+    private final IEncapsulatedOpenBISService service;
     private final MessageFormat flowLineDropBoxTemplate;
     private final IImmutableCopier copier;
     private final IFileOperations fileOperations;
     private final List<File> flowLineDataSets = new ArrayList<File>();
     
-    FlowLineFeeder(Properties properties)
+    FlowLineFeeder(Properties properties, IEncapsulatedOpenBISService service)
     {
+        this.service = service;
         flowLineDropBoxTemplate =
                 new MessageFormat(PropertyUtils.getMandatoryProperty(properties,
                         FLOW_LINE_DROP_BOX_TEMPLATE));
@@ -60,6 +73,7 @@ class FlowLineFeeder implements IPostRegistrationDatasetHandler
 
     public void handle(File originalData, DataSetInformation dataSetInformation)
     {
+        Map<String, Sample> flowLineSampleMap = createFlowLineSampleMap(dataSetInformation);
         String flowcellID = originalData.getName();
         List<File> files = new ArrayList<File>();
         findFiles(originalData, files);
@@ -68,6 +82,11 @@ class FlowLineFeeder implements IPostRegistrationDatasetHandler
             String flowLine = extractFlowLine(file);
             File dropBox = createDropBoxFile(flowLine);
             File flowLineDataSet = new File(dropBox, flowcellID + "_" + flowLine);
+            if (flowLineDataSet.exists())
+            {
+                throw new EnvironmentFailureException("There is already a data set for flow line "
+                        + flowLine + ".");
+            }
             flowLineDataSets.add(flowLineDataSet);
             boolean success = flowLineDataSet.mkdir();
             if (success == false)
@@ -75,6 +94,7 @@ class FlowLineFeeder implements IPostRegistrationDatasetHandler
                 throw new EnvironmentFailureException("Couldn't create folder '"
                         + flowLineDataSet.getAbsolutePath() + "'.");
             }
+            createMetaDataFile(flowLineDataSet, flowLineSampleMap, flowLine);
             success = copier.copyImmutably(file, flowLineDataSet, null);
             if (success == false)
             {
@@ -86,6 +106,51 @@ class FlowLineFeeder implements IPostRegistrationDatasetHandler
 
     }
 
+    private Map<String, Sample> createFlowLineSampleMap(DataSetInformation dataSetInformation)
+    {
+        SampleIdentifier sampleIdentifier = dataSetInformation.getSampleIdentifier();
+        Sample flowCell = service.tryGetSampleWithExperiment(sampleIdentifier);
+        if (flowCell == null)
+        {
+            throw new UserFailureException("Unkown flow cell sample: " + sampleIdentifier);
+        }
+        TechId flowCellID = new TechId(flowCell.getId());
+        ListSampleCriteria criteria = ListSampleCriteria.createForContainer(flowCellID);
+        List<Sample> flowLineSamples = service.listSamples(criteria);
+        Map<String, Sample> flowLineSampleMap = new LinkedHashMap<String, Sample>();
+        for (Sample flowLineSample : flowLineSamples)
+        {
+            flowLineSampleMap.put(flowLineSample.getSubCode(), flowLineSample);
+        }
+        return flowLineSampleMap;
+    }
+
+    private void createMetaDataFile(File flowLineDataSet, Map<String, Sample> flowLineSampleMap,
+            String flowLine)
+    {
+        Sample flowLineSample = flowLineSampleMap.get(flowLine);
+        if (flowLineSample == null)
+        {
+            throw new UserFailureException("No flow line sample for flow line " + flowLine + " exists");
+        }
+        StringBuilder builder = new StringBuilder();
+        addLine(builder, "Parent", flowLineSample.getGeneratedFrom().getIdentifier());
+        addLine(builder, "Code", flowLineSample.getCode());
+        addLine(builder, "Contact Person Email", flowLineSample.getRegistrator().getEmail());
+        SampleIdentifier identifier = SampleIdentifierFactory.parse(flowLineSample.getIdentifier());
+        IEntityProperty[] properties = service.getPropertiesOfTopSampleRegisteredFor(identifier);
+        for (IEntityProperty property : properties)
+        {
+            addLine(builder, property.getPropertyType().getLabel(), property.tryGetAsString());
+        }
+        FileUtilities.writeToFile(new File(flowLineDataSet, "meta-data.tsv"), builder.toString());
+    }
+    
+    private void addLine(StringBuilder builder, String key, String value)
+    {
+        builder.append(key).append('\t').append(value).append('\n');
+    }
+
     private File createDropBoxFile(String flowLine)
     {
         File dropBox = new File(flowLineDropBoxTemplate.format(new Object[] {flowLine}));

deep_sequencing_unit/source/java/ch/ethz/bsse/cisd/dsu/dss/StorageProcessor.java

@@ -19,6 +19,7 @@ package ch.ethz.bsse.cisd.dsu.dss;
 import java.util.Properties;
 
 import ch.systemsx.cisd.etlserver.DelegatingStorageProcessorWithDropbox;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
 
 /**
  * Storage processor which feeds flow-line drop boxes. Needs the property
@@ -31,7 +32,7 @@ public class StorageProcessor extends DelegatingStorageProcessorWithDropbox
 {
     public StorageProcessor(Properties properties)
     {
-        super(properties, new FlowLineFeeder(properties));
+        super(properties, new FlowLineFeeder(properties, ServiceProvider.getOpenBISService()));
     }
 
 }

deep_sequencing_unit/sourceTest/java/ch/ethz/bsse/cisd/dsu/dss/FlowLineFeederTest.java

@@ -18,8 +18,15 @@ package ch.ethz.bsse.cisd.dsu.dss;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
 import java.util.Properties;
 
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.Description;
+import org.jmock.Expectations;
+import org.jmock.Mockery;
+import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
@@ -27,6 +34,14 @@ import ch.systemsx.cisd.base.tests.AbstractFileSystemTestCase;
 import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
 import ch.systemsx.cisd.common.filesystem.FileUtilities;
 import ch.systemsx.cisd.common.test.AssertionUtil;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
+import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
+import ch.systemsx.cisd.openbis.generic.client.web.client.exception.UserFailureException;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ListSampleCriteria;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Person;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Sample;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifier;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifierFactory;
 
 /**
  * 
@@ -35,21 +50,52 @@ import ch.systemsx.cisd.common.test.AssertionUtil;
  */
 public class FlowLineFeederTest extends AbstractFileSystemTestCase
 {
+    private static final Sample EXAMPLE_FLOW_CELL_SAMPLE = createFlowCellSample();
+    private static final DataSetInformation EXAMPLE_DATA_SET_INFO = createDataSetInfo();
+    private static final String SAMPLE_CODE = "fc";
     private static final String DROP_BOX_PREFIX = "drop-box-";
+    
+    private static DataSetInformation createDataSetInfo()
+    {
+        DataSetInformation dataSetInfo = new DataSetInformation();
+        dataSetInfo.setSampleCode(SAMPLE_CODE);
+        return dataSetInfo;
+    }
+    
+    private static Sample createFlowCellSample()
+    {
+        Sample sample = new Sample();
+        sample.setId(42L);
+        sample.setCode(SAMPLE_CODE);
+        return sample;
+    }
+    
     private FlowLineFeeder flowLineFeeder;
+    private Mockery context;
+    private IEncapsulatedOpenBISService service;
 
     @Override
     @BeforeMethod
     public void setUp() throws IOException
     {
         super.setUp();
+        context = new Mockery();
+        service = context.mock(IEncapsulatedOpenBISService.class);
         assertEquals(true, new File(workingDirectory, DROP_BOX_PREFIX + "1").mkdirs());
         assertEquals(true, new File(workingDirectory, DROP_BOX_PREFIX + "2").mkdirs());
         Properties properties = new Properties();
         properties.setProperty(FlowLineFeeder.FLOW_LINE_DROP_BOX_TEMPLATE, new File(
                 workingDirectory, DROP_BOX_PREFIX).getAbsolutePath()
                 + "{0}");
-        flowLineFeeder = new FlowLineFeeder(properties);
+        flowLineFeeder = new FlowLineFeeder(properties, service);
+    }
+    
+    @AfterMethod
+    public void tearDown()
+    {
+        // To following line of code should also be called at the end of each test method.
+        // Otherwise one does not known which test failed.
+        context.assertIsSatisfied();
     }
     
     @Test
@@ -57,13 +103,15 @@ public class FlowLineFeederTest extends AbstractFileSystemTestCase
     {
         try
         {
-            new FlowLineFeeder(new Properties());
+            new FlowLineFeeder(new Properties(), service);
             fail("ConfigurationFailureException expected");
         } catch (ConfigurationFailureException ex)
         {
             assertEquals("Given key '" + FlowLineFeeder.FLOW_LINE_DROP_BOX_TEMPLATE
                     + "' not found in properties '[]'", ex.getMessage());
         }
+        
+        context.assertIsSatisfied();
     }
     
     @Test 
@@ -72,15 +120,38 @@ public class FlowLineFeederTest extends AbstractFileSystemTestCase
         File flowCell = new File(workingDirectory, "abc");
         assertEquals(true, flowCell.mkdir());
         FileUtilities.writeToFile(new File(flowCell, "s_3.srf"), "hello flow line 3");
+        prepareLoadFlowCellSample(EXAMPLE_FLOW_CELL_SAMPLE);
+        prepareListFlowLines(EXAMPLE_FLOW_CELL_SAMPLE, Arrays.<Sample>asList());
         
         try
         {
-            flowLineFeeder.handle(flowCell, null);
+            flowLineFeeder.handle(flowCell, EXAMPLE_DATA_SET_INFO);
             fail("ConfigurationFailureException expected");
         } catch (ConfigurationFailureException ex)
         {
             AssertionUtil.assertContains(DROP_BOX_PREFIX + "3", ex.getMessage());
         }
+        
+        context.assertIsSatisfied();
+    }
+    
+    @Test
+    public void testUnkownFlowCell()
+    {
+        File flowCell = new File(workingDirectory, "abc");
+        prepareLoadFlowCellSample(null);
+        
+        try
+        {
+            flowLineFeeder.handle(flowCell, EXAMPLE_DATA_SET_INFO);
+            fail("UserFailureException expected");
+        } catch (UserFailureException ex)
+        {
+            assertEquals("Unkown flow cell sample: " + EXAMPLE_DATA_SET_INFO.getSampleIdentifier(),
+                    ex.getMessage());
+        }
+        
+        context.assertIsSatisfied();
     }
     
     @Test
@@ -97,11 +168,18 @@ public class FlowLineFeederTest extends AbstractFileSystemTestCase
         FileUtilities.writeToFile(originalFlowLine1, "hello flow line 1");
         File originalFlowLine2 = new File(srfFolder, "2.srf");
         FileUtilities.writeToFile(originalFlowLine2, "hello flow line 2");
+        prepareLoadFlowCellSample(EXAMPLE_FLOW_CELL_SAMPLE);
+        
+        Sample fl1 = createFlowLineSample(1);
+        Sample fl2 = createFlowLineSample(2);
+        prepareListFlowLines(EXAMPLE_FLOW_CELL_SAMPLE, Arrays.asList(fl1, fl2));
 
-        flowLineFeeder.handle(flowCell, null);
+        flowLineFeeder.handle(flowCell, EXAMPLE_DATA_SET_INFO);
         
         checkFlowLineDataSet(originalFlowLine1, "1");
         checkFlowLineDataSet(originalFlowLine2, "2");
+        
+        context.assertIsSatisfied();
     }
     
     @Test
@@ -116,6 +194,53 @@ public class FlowLineFeederTest extends AbstractFileSystemTestCase
         assertEquals(0, new File(workingDirectory, DROP_BOX_PREFIX + "1").list().length);
         assertEquals(0, new File(workingDirectory, DROP_BOX_PREFIX + "2").list().length);
     }
+    
+    private void prepareLoadFlowCellSample(final Sample flowCellSample)
+    {
+        context.checking(new Expectations()
+            {
+                {
+                    one(service).tryGetSampleWithExperiment(EXAMPLE_DATA_SET_INFO.getSampleIdentifier());
+                    will(returnValue(flowCellSample));
+                }
+            });
+    }
+    
+    private void prepareListFlowLines(final Sample flowCellSample, final List<Sample> flowLineSamples)
+    {
+        context.checking(new Expectations()
+            {
+                {
+                    one(service).listSamples(with(new BaseMatcher<ListSampleCriteria>()
+                        {
+
+                            public boolean matches(Object item)
+                            {
+                                if (item instanceof ListSampleCriteria)
+                                {
+                                    ListSampleCriteria criteria = (ListSampleCriteria) item;
+                                    return criteria.getContainerSampleId().getId().equals(
+                                            flowCellSample.getId());
+                                }
+                                return false;
+                            }
+
+                            public void describeTo(Description description)
+                            {
+                                description.appendText("Flow cell with ID "
+                                        + flowCellSample.getId());
+                            }
+                        }));
+                    will(returnValue(flowLineSamples));
+                    for (Sample sample : flowLineSamples)
+                    {
+                        SampleIdentifier identifier =
+                                SampleIdentifierFactory.parse(sample.getIdentifier());
+                        one(service).getPropertiesOfTopSampleRegisteredFor(identifier);
+                    }
+                }
+            });
+    }
 
     private void checkFlowLineDataSet(File originalFlowLine, String flowLineNumber)
     {
@@ -132,4 +257,19 @@ public class FlowLineFeederTest extends AbstractFileSystemTestCase
         originalFlowLine.setLastModified(4711000);
         assertEquals(4711000, flowLine.lastModified());
     }
+    
+    private Sample createFlowLineSample(int flowLineNumber)
+    {
+        Sample sample = new Sample();
+        sample.setCode(SAMPLE_CODE);
+        sample.setSubCode(Integer.toString(flowLineNumber));
+        sample.setGeneratedFrom(EXAMPLE_FLOW_CELL_SAMPLE);
+        Person registrator = new Person();
+        registrator.setEmail("ab@c.de");
+        sample.setRegistrator(registrator);
+        sample.setIdentifier(SAMPLE_CODE + SampleIdentifier.CONTAINED_SAMPLE_CODE_SEPARARTOR_STRING
+                + flowLineNumber);
+        return sample;
+    }
+    
 }