LMS-1584 New version of feature storage processor that stores data in the db.

SVN: 16657

screening/etc/service.properties

@@ -328,3 +328,5 @@ genedata-image-analysis-results.type-extractor.is-measured = false
 
 # The storage processor (IStorageProcessor implementation)
 genedata-image-analysis-results.storage-processor = ch.systemsx.cisd.openbis.dss.etl.genedata.FeatureStorageProcessor
+genedata-image-analysis-results.storage-processor.processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor
+genedata-image-analysis-results.storage-processor.data-source = imaging-db

screening/source/java/ch/systemsx/cisd/openbis/dss/etl/genedata/FeatureStorageProcessor.java

@@ -17,27 +17,46 @@
 package ch.systemsx.cisd.openbis.dss.etl.genedata;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Properties;
 import java.util.StringTokenizer;
 
+import javax.sql.DataSource;
+
+import net.lemnik.eodsql.QueryTool;
+
+import ch.systemsx.cisd.base.exceptions.IOExceptionUnchecked;
 import ch.systemsx.cisd.common.exceptions.UserFailureException;
 import ch.systemsx.cisd.common.filesystem.FileUtilities;
+import ch.systemsx.cisd.common.mail.IMailClient;
+import ch.systemsx.cisd.etlserver.AbstractDelegatingStorageProcessor;
 import ch.systemsx.cisd.etlserver.DefaultStorageProcessor;
+import ch.systemsx.cisd.etlserver.ITypeExtractor;
 import ch.systemsx.cisd.etlserver.utils.Column;
 import ch.systemsx.cisd.etlserver.utils.TableBuilder;
+import ch.systemsx.cisd.openbis.dss.etl.ScreeningContainerDatasetInfo;
+import ch.systemsx.cisd.openbis.dss.etl.dataaccess.IImagingUploadDAO;
+import ch.systemsx.cisd.openbis.dss.etl.featurevector.CanonicalFeatureVector;
+import ch.systemsx.cisd.openbis.dss.etl.featurevector.FeatureVectorUploader;
+import ch.systemsx.cisd.openbis.dss.etl.featurevector.GenedataFormatToCanonicalFeatureVector;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
 
 /**
  * @author Franz-Josef Elmer
  */
-public class FeatureStorageProcessor extends DefaultStorageProcessor
+public class FeatureStorageProcessor extends AbstractDelegatingStorageProcessor
 {
     private static final char DELIMITER = ';';
 
     private static final String LAYER_PREFIX = "<Layer=";
 
-    private File originalFile;
+    private final DataSource dataSource;
+
+    // Execution state of this object -- set to null after an execution is finished.
+    private IImagingUploadDAO dataAccessObject = null;
 
     private final class ColumnsBuilder
     {
@@ -131,18 +150,92 @@ public class FeatureStorageProcessor extends DefaultStorageProcessor
     public FeatureStorageProcessor(Properties properties)
     {
         super(properties);
+        this.dataSource = ServiceProvider.getDataSourceProvider().getDataSource(properties);
     }
 
     @Override
-    protected void transform(File originalDataSet, File targetFolderForTransformedDataSet)
+    public File storeData(DataSetInformation dataSetInformation, ITypeExtractor typeExtractor,
+            IMailClient mailClient, File incomingDataSetDirectory, File rootDir)
+    {
+        File storedDataSet =
+                super.storeData(dataSetInformation, typeExtractor, mailClient,
+                        incomingDataSetDirectory, rootDir);
+        File originalDir = DefaultStorageProcessor.getOriginalDirectory(storedDataSet);
+        final File targetFile = new File(originalDir, incomingDataSetDirectory.getName());
+        transform(targetFile, storedDataSet, dataSetInformation);
+
+        return storedDataSet;
+    }
+
+    protected void transform(File originalDataSet, File targetFolderForTransformedDataSet,
+            DataSetInformation dataSetInformation)
     {
-        originalFile = originalDataSet;
         List<String> lines = FileUtilities.loadToStringList(originalDataSet);
         if (lines.isEmpty())
         {
             throw new UserFailureException("Empty file: " + originalDataSet.getName());
         }
         String barCode = extractBarCode(lines.get(0));
+        ColumnsBuilder columnsBuilder = convertLinesIntoColumns(lines);
+        String columnsString = convertColumnsToString(barCode, columnsBuilder);
+        File originalDirectory =
+                DefaultStorageProcessor.getOriginalDirectory(targetFolderForTransformedDataSet);
+        File file = new File(originalDirectory, originalDataSet.getName() + ".txt");
+        FileUtilities.writeToFile(file, columnsString);
+
+        try
+        {
+            loadDataSetIntoDatabase(lines, dataSetInformation);
+        } catch (IOException ex)
+        {
+            throw new IOExceptionUnchecked(ex);
+        }
+    }
+
+    private void loadDataSetIntoDatabase(List<String> lines, DataSetInformation dataSetInformation)
+            throws IOException
+    {
+        GenedataFormatToCanonicalFeatureVector convertor =
+                new GenedataFormatToCanonicalFeatureVector(lines, LAYER_PREFIX);
+        ArrayList<CanonicalFeatureVector> fvecs = convertor.convert();
+
+        dataAccessObject = createDAO();
+        FeatureVectorUploader uploader =
+                new FeatureVectorUploader(dataAccessObject, ScreeningContainerDatasetInfo
+                        .createScreeningDatasetInfo(dataSetInformation));
+        uploader.uploadFeatureVectors(fvecs);
+    }
+
+    protected IImagingUploadDAO createDAO()
+    {
+        return QueryTool.getQuery(dataSource, IImagingUploadDAO.class);
+    }
+
+    private String convertColumnsToString(String barCode, ColumnsBuilder columnsBuilder)
+    {
+        List<Column> columns = columnsBuilder.getColumns();
+        StringBuilder builder = new StringBuilder();
+        builder.append("barcode").append(DELIMITER).append("row").append(DELIMITER).append("col");
+        for (Column column : columns)
+        {
+            builder.append(DELIMITER).append(column.getHeader());
+        }
+        for (int i = 0, n = columnsBuilder.getNumberOfWells(); i < n; i++)
+        {
+            builder.append('\n').append(barCode);
+            builder.append(DELIMITER).append(columnsBuilder.getRowLetter(i));
+            builder.append(DELIMITER).append(columnsBuilder.getColumnNumber(i));
+            for (Column column : columns)
+            {
+                builder.append(DELIMITER).append(column.getValues().get(i));
+            }
+        }
+        String columnsString = builder.toString();
+        return columnsString;
+    }
+
+    private ColumnsBuilder convertLinesIntoColumns(List<String> lines)
+    {
         ColumnsBuilder columnsBuilder = new ColumnsBuilder();
         for (int i = 1; i < lines.size(); i++)
         {
@@ -163,35 +256,44 @@ public class FeatureStorageProcessor extends DefaultStorageProcessor
             }
         }
         columnsBuilder.finish();
-        List<Column> columns = columnsBuilder.getColumns();
-        StringBuilder builder = new StringBuilder();
-        builder.append("barcode").append(DELIMITER).append("row").append(DELIMITER).append("col");
-        for (Column column : columns)
-        {
-            builder.append(DELIMITER).append(column.getHeader());
-        }
-        for (int i = 0, n = columnsBuilder.getNumberOfWells(); i < n; i++)
+        return columnsBuilder;
+    }
+
+    @Override
+    public void commit()
+    {
+        super.commit();
+
+        if (null == dataAccessObject)
         {
-            builder.append('\n').append(barCode);
-            builder.append(DELIMITER).append(columnsBuilder.getRowLetter(i));
-            builder.append(DELIMITER).append(columnsBuilder.getColumnNumber(i));
-            for (Column column : columns)
-            {
-                builder.append(DELIMITER).append(column.getValues().get(i));
-            }
+            return;
         }
-        File originalDirectory = getOriginalDirectory(targetFolderForTransformedDataSet);
-        File file = new File(originalDirectory, originalDataSet.getName() + ".txt");
-        FileUtilities.writeToFile(file, builder.toString());
+
+        dataAccessObject.commit();
+        closeDataAccessObject();
+    }
+
+    /**
+     * Close the DAO and set it to null to make clear that it is not initialized.
+     */
+    private void closeDataAccessObject()
+    {
+        dataAccessObject.close();
+        dataAccessObject = null;
     }
 
     @Override
-    public void commit()
+    public UnstoreDataAction rollback(final File incomingDataSetDirectory,
+            final File storedDataDirectory, Throwable exception)
     {
-        if (originalFile != null && originalFile.exists())
+        // Delete the data from the database
+        if (null != dataAccessObject)
         {
-            originalFile.delete();
+            dataAccessObject.rollback();
+            closeDataAccessObject();
         }
+
+        return super.rollback(incomingDataSetDirectory, storedDataDirectory, exception);
     }
 
     private String extractBarCode(String firstLine)

screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/genedata/FeatureStorageProcessorTest.java

@@ -17,27 +17,92 @@
 package ch.systemsx.cisd.openbis.dss.etl.genedata;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.List;
 import java.util.Properties;
 
+import org.jmock.Expectations;
+import org.jmock.Mockery;
+import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
 import ch.systemsx.cisd.base.tests.AbstractFileSystemTestCase;
 import ch.systemsx.cisd.common.filesystem.FileUtilities;
 import ch.systemsx.cisd.etlserver.IStorageProcessor;
-import ch.systemsx.cisd.openbis.dss.etl.genedata.FeatureStorageProcessor;
+import ch.systemsx.cisd.etlserver.PlateDimensionParser;
+import ch.systemsx.cisd.openbis.dss.etl.dataaccess.IImagingUploadDAO;
+import ch.systemsx.cisd.openbis.dss.etl.dataaccess.ImgDatasetDTO;
+import ch.systemsx.cisd.openbis.dss.etl.dataaccess.ImgFeatureDefDTO;
+import ch.systemsx.cisd.openbis.dss.etl.dataaccess.ImgFeatureValuesDTO;
 import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Experiment;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.GenericValueEntityProperty;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IEntityProperty;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.PropertyType;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Sample;
 
 /**
  * @author Franz-Josef Elmer
  */
 public class FeatureStorageProcessorTest extends AbstractFileSystemTestCase
 {
+    private static final String DATA_SET_PERM_ID = "dataset-1";
+
+    private static final String CONTAINER_PERM_ID = "perm12";
+
+    private static final String EXPERIMENT_PERM_ID = "perm11";
+
     private static final String EXAMPLE1 =
             "barcode = Plate_042" + "\n\n<Layer=alpha>\n" + "\t1\t2\n" + "A\t4.5\t4.6\n"
                     + "B\t3.5\t5.6\n" + "C\t3.3\t5.7\n" + "\n\n<Layer=beta>\n" + "\t1\t2\n"
                     + "A\t14.5\t14.6\n" + "B\t13.5\t15.6\n" + "C\t13.3\t15.7\n";
 
+    private Mockery context;
+
+    private IImagingUploadDAO dao;
+
+    @Override
+    @BeforeMethod
+    public void setUp() throws IOException
+    {
+        super.setUp();
+
+        context = new Mockery();
+        dao = context.mock(IImagingUploadDAO.class);
+
+        context.checking(new Expectations()
+            {
+                {
+                    one(dao).tryGetExperimentIdByPermId(EXPERIMENT_PERM_ID);
+                    will(returnValue((long) 1));
+                    one(dao).tryGetContainerIdPermId(CONTAINER_PERM_ID);
+                    will(returnValue((long) 1));
+
+                    ImgDatasetDTO dataSetDTO = new ImgDatasetDTO(DATA_SET_PERM_ID, 3, 2, 1);
+                    dataSetDTO.setId(1);
+                    one(dao).tryGetDatasetByPermId(DATA_SET_PERM_ID);
+                    will(returnValue(dataSetDTO));
+
+                    ImgFeatureDefDTO featureDTO = new ImgFeatureDefDTO("alpha", "alpha", 1);
+                    one(dao).addFeatureDef(with(equal(featureDTO)));
+                    will(returnValue((long) 1));
+
+                    one(dao).addFeatureValues(with(any(ImgFeatureValuesDTO.class)));
+                    will(returnValue((long) 1));
+
+                    featureDTO = new ImgFeatureDefDTO("beta", "beta", 1);
+                    one(dao).addFeatureDef(with(equal(featureDTO)));
+                    will(returnValue((long) 2));
+
+                    one(dao).addFeatureValues(with(any(ImgFeatureValuesDTO.class)));
+                    will(returnValue((long) 2));
+
+                    one(dao).commit();
+                    one(dao).close();
+                }
+            });
+    }
+
     @Test
     public void test()
     {
@@ -47,9 +112,19 @@ public class FeatureStorageProcessorTest extends AbstractFileSystemTestCase
         FileUtilities.writeToFile(dataSetFile, EXAMPLE1);
         File rootDir = new File(workingDirectory, "rootDir");
         rootDir.mkdirs();
-        IStorageProcessor storageProcessor = new FeatureStorageProcessor(new Properties());
+        Properties storageProcessorProps = createStorageProcessorProperties();
+        IStorageProcessor storageProcessor = new FeatureStorageProcessor(storageProcessorProps)
+            {
+                // For Testing
+                @Override
+                protected IImagingUploadDAO createDAO()
+                {
+                    return dao;
+                }
+            };
 
-        storageProcessor.storeData(new DataSetInformation(), null, null, dataSetFile, rootDir);
+        DataSetInformation dataSetInfo = createDataSetInformation();
+        storageProcessor.storeData(dataSetInfo, null, null, dataSetFile, rootDir);
 
         assertEquals(0, incomingDir.listFiles().length);
         assertEquals(1, rootDir.listFiles().length);
@@ -59,8 +134,8 @@ public class FeatureStorageProcessorTest extends AbstractFileSystemTestCase
 
         storageProcessor.commit();
 
-        assertEquals(1, original.listFiles().length);
-        File transformedDataSetFile = original.listFiles()[0];
+        assertEquals(2, original.listFiles().length);
+        File transformedDataSetFile = original.listFiles()[1];
         assertEquals("Plate042.stat.txt", transformedDataSetFile.getName());
         List<String> lines = FileUtilities.loadToStringList(transformedDataSetFile);
         assertEquals("barcode;row;col;alpha;beta", lines.get(0));
@@ -72,4 +147,45 @@ public class FeatureStorageProcessorTest extends AbstractFileSystemTestCase
         assertEquals("Plate_042;C;2;5.7;15.7", lines.get(6));
         assertEquals(7, lines.size());
     }
+
+    private DataSetInformation createDataSetInformation()
+    {
+        // Set the Experiment
+        DataSetInformation dataSetInfo = new DataSetInformation();
+        Experiment exp = new Experiment();
+        exp.setIdentifier("/Test/Test1/Exp1");
+        exp.setPermId(EXPERIMENT_PERM_ID);
+        dataSetInfo.setExperiment(exp);
+
+        // Set the Sample
+        Sample sample = new Sample();
+        sample.setCode("Samp1");
+        sample.setExperiment(exp);
+        sample.setPermId(CONTAINER_PERM_ID);
+        dataSetInfo.setSample(sample);
+
+        // Set the DataSet
+        dataSetInfo.setDataSetCode(DATA_SET_PERM_ID);
+
+        // Set the properties
+        IEntityProperty properties[] = new IEntityProperty[1];
+        GenericValueEntityProperty entityProperty = new GenericValueEntityProperty();
+        PropertyType propertyType = new PropertyType();
+        propertyType.setCode(PlateDimensionParser.PLATE_GEOMETRY_PROPERTY_NAME);
+        entityProperty.setPropertyType(propertyType);
+        entityProperty.setValue("A_2X2");
+        properties[0] = entityProperty;
+        dataSetInfo.setProperties(properties);
+        return dataSetInfo;
+    }
+
+    private Properties createStorageProcessorProperties()
+    {
+        Properties storageProcessorProps = new Properties();
+        storageProcessorProps.setProperty("processor",
+                "ch.systemsx.cisd.etlserver.DefaultStorageProcessor");
+        // storageProcessorProps.setProperty("data-source", "imaging-db");
+        storageProcessorProps.setProperty("data-source", "imaging-db");
+        return storageProcessorProps;
+    }
 }

screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/genedata/GenedataFormatToCanonicalFeatureVectorTest.java

@@ -31,7 +31,9 @@ import ch.systemsx.cisd.openbis.dss.etl.featurevector.CanonicalFeatureVector;
 import ch.systemsx.cisd.openbis.dss.etl.featurevector.GenedataFormatToCanonicalFeatureVector;
 
 /**
- * @author Franz-Josef Elmer
+ * Check that Genedata feature vectors can be converted to the canonical form.
+ * 
+ * @author Chandrasekhar Ramakrishnan
  */
 public class GenedataFormatToCanonicalFeatureVectorTest extends AbstractFileSystemTestCase
 {