diff --git a/screening/etc/service.properties b/screening/etc/service.properties
index 544e979528d8bd9d45b997dc920abe448335e314..bf6905f8aee63fa15384add29aced43f32b1931d 100644
--- a/screening/etc/service.properties
+++ b/screening/etc/service.properties
@@ -328,3 +328,5 @@ genedata-image-analysis-results.type-extractor.is-measured = false
# The storage processor (IStorageProcessor implementation)
genedata-image-analysis-results.storage-processor = ch.systemsx.cisd.openbis.dss.etl.genedata.FeatureStorageProcessor
+genedata-image-analysis-results.storage-processor.processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor
+genedata-image-analysis-results.storage-processor.data-source = imaging-db
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/genedata/FeatureStorageProcessor.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/genedata/FeatureStorageProcessor.java
index 29369d8834840575caaf1d5c8a6c2f35a50aba95..6527d125e0324bfa6f0c87a4d5a84bb8fc250733 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/genedata/FeatureStorageProcessor.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/genedata/FeatureStorageProcessor.java
@@ -17,27 +17,46 @@
package ch.systemsx.cisd.openbis.dss.etl.genedata;
import java.io.File;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.StringTokenizer;
+import javax.sql.DataSource;
+
+import net.lemnik.eodsql.QueryTool;
+
+import ch.systemsx.cisd.base.exceptions.IOExceptionUnchecked;
import ch.systemsx.cisd.common.exceptions.UserFailureException;
import ch.systemsx.cisd.common.filesystem.FileUtilities;
+import ch.systemsx.cisd.common.mail.IMailClient;
+import ch.systemsx.cisd.etlserver.AbstractDelegatingStorageProcessor;
import ch.systemsx.cisd.etlserver.DefaultStorageProcessor;
+import ch.systemsx.cisd.etlserver.ITypeExtractor;
import ch.systemsx.cisd.etlserver.utils.Column;
import ch.systemsx.cisd.etlserver.utils.TableBuilder;
+import ch.systemsx.cisd.openbis.dss.etl.ScreeningContainerDatasetInfo;
+import ch.systemsx.cisd.openbis.dss.etl.dataaccess.IImagingUploadDAO;
+import ch.systemsx.cisd.openbis.dss.etl.featurevector.CanonicalFeatureVector;
+import ch.systemsx.cisd.openbis.dss.etl.featurevector.FeatureVectorUploader;
+import ch.systemsx.cisd.openbis.dss.etl.featurevector.GenedataFormatToCanonicalFeatureVector;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
/**
* @author Franz-Josef Elmer
*/
-public class FeatureStorageProcessor extends DefaultStorageProcessor
+public class FeatureStorageProcessor extends AbstractDelegatingStorageProcessor
{
private static final char DELIMITER = ';';
private static final String LAYER_PREFIX = "<Layer=";
- private File originalFile;
+ private final DataSource dataSource;
+
+ // Execution state of this object -- set to null after an execution is finished.
+ private IImagingUploadDAO dataAccessObject = null;
private final class ColumnsBuilder
{
@@ -131,18 +150,92 @@ public class FeatureStorageProcessor extends DefaultStorageProcessor
public FeatureStorageProcessor(Properties properties)
{
super(properties);
+ this.dataSource = ServiceProvider.getDataSourceProvider().getDataSource(properties);
}
@Override
- protected void transform(File originalDataSet, File targetFolderForTransformedDataSet)
+ public File storeData(DataSetInformation dataSetInformation, ITypeExtractor typeExtractor,
+ IMailClient mailClient, File incomingDataSetDirectory, File rootDir)
+ {
+ File storedDataSet =
+ super.storeData(dataSetInformation, typeExtractor, mailClient,
+ incomingDataSetDirectory, rootDir);
+ File originalDir = DefaultStorageProcessor.getOriginalDirectory(storedDataSet);
+ final File targetFile = new File(originalDir, incomingDataSetDirectory.getName());
+ transform(targetFile, storedDataSet, dataSetInformation);
+
+ return storedDataSet;
+ }
+
+ protected void transform(File originalDataSet, File targetFolderForTransformedDataSet,
+ DataSetInformation dataSetInformation)
{
- originalFile = originalDataSet;
List<String> lines = FileUtilities.loadToStringList(originalDataSet);
if (lines.isEmpty())
{
throw new UserFailureException("Empty file: " + originalDataSet.getName());
}
String barCode = extractBarCode(lines.get(0));
+ ColumnsBuilder columnsBuilder = convertLinesIntoColumns(lines);
+ String columnsString = convertColumnsToString(barCode, columnsBuilder);
+ File originalDirectory =
+ DefaultStorageProcessor.getOriginalDirectory(targetFolderForTransformedDataSet);
+ File file = new File(originalDirectory, originalDataSet.getName() + ".txt");
+ FileUtilities.writeToFile(file, columnsString);
+
+ try
+ {
+ loadDataSetIntoDatabase(lines, dataSetInformation);
+ } catch (IOException ex)
+ {
+ throw new IOExceptionUnchecked(ex);
+ }
+ }
+
+ private void loadDataSetIntoDatabase(List<String> lines, DataSetInformation dataSetInformation)
+ throws IOException
+ {
+ GenedataFormatToCanonicalFeatureVector convertor =
+ new GenedataFormatToCanonicalFeatureVector(lines, LAYER_PREFIX);
+ ArrayList<CanonicalFeatureVector> fvecs = convertor.convert();
+
+ dataAccessObject = createDAO();
+ FeatureVectorUploader uploader =
+ new FeatureVectorUploader(dataAccessObject, ScreeningContainerDatasetInfo
+ .createScreeningDatasetInfo(dataSetInformation));
+ uploader.uploadFeatureVectors(fvecs);
+ }
+
+ protected IImagingUploadDAO createDAO()
+ {
+ return QueryTool.getQuery(dataSource, IImagingUploadDAO.class);
+ }
+
+ private String convertColumnsToString(String barCode, ColumnsBuilder columnsBuilder)
+ {
+ List<Column> columns = columnsBuilder.getColumns();
+ StringBuilder builder = new StringBuilder();
+ builder.append("barcode").append(DELIMITER).append("row").append(DELIMITER).append("col");
+ for (Column column : columns)
+ {
+ builder.append(DELIMITER).append(column.getHeader());
+ }
+ for (int i = 0, n = columnsBuilder.getNumberOfWells(); i < n; i++)
+ {
+ builder.append('\n').append(barCode);
+ builder.append(DELIMITER).append(columnsBuilder.getRowLetter(i));
+ builder.append(DELIMITER).append(columnsBuilder.getColumnNumber(i));
+ for (Column column : columns)
+ {
+ builder.append(DELIMITER).append(column.getValues().get(i));
+ }
+ }
+ String columnsString = builder.toString();
+ return columnsString;
+ }
+
+ private ColumnsBuilder convertLinesIntoColumns(List<String> lines)
+ {
ColumnsBuilder columnsBuilder = new ColumnsBuilder();
for (int i = 1; i < lines.size(); i++)
{
@@ -163,35 +256,44 @@ public class FeatureStorageProcessor extends DefaultStorageProcessor
}
}
columnsBuilder.finish();
- List<Column> columns = columnsBuilder.getColumns();
- StringBuilder builder = new StringBuilder();
- builder.append("barcode").append(DELIMITER).append("row").append(DELIMITER).append("col");
- for (Column column : columns)
- {
- builder.append(DELIMITER).append(column.getHeader());
- }
- for (int i = 0, n = columnsBuilder.getNumberOfWells(); i < n; i++)
+ return columnsBuilder;
+ }
+
+ @Override
+ public void commit()
+ {
+ super.commit();
+
+ if (null == dataAccessObject)
{
- builder.append('\n').append(barCode);
- builder.append(DELIMITER).append(columnsBuilder.getRowLetter(i));
- builder.append(DELIMITER).append(columnsBuilder.getColumnNumber(i));
- for (Column column : columns)
- {
- builder.append(DELIMITER).append(column.getValues().get(i));
- }
+ return;
}
- File originalDirectory = getOriginalDirectory(targetFolderForTransformedDataSet);
- File file = new File(originalDirectory, originalDataSet.getName() + ".txt");
- FileUtilities.writeToFile(file, builder.toString());
+
+ dataAccessObject.commit();
+ closeDataAccessObject();
+ }
+
+ /**
+ * Close the DAO and set it to null to make clear that it is not initialized.
+ */
+ private void closeDataAccessObject()
+ {
+ dataAccessObject.close();
+ dataAccessObject = null;
}
@Override
- public void commit()
+ public UnstoreDataAction rollback(final File incomingDataSetDirectory,
+ final File storedDataDirectory, Throwable exception)
{
- if (originalFile != null && originalFile.exists())
+ // Delete the data from the database
+ if (null != dataAccessObject)
{
- originalFile.delete();
+ dataAccessObject.rollback();
+ closeDataAccessObject();
}
+
+ return super.rollback(incomingDataSetDirectory, storedDataDirectory, exception);
}
private String extractBarCode(String firstLine)
diff --git a/screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/genedata/FeatureStorageProcessorTest.java b/screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/genedata/FeatureStorageProcessorTest.java
index 008745f53f390ba6a93d4686364b9a7d5ee29ac1..3000019a5b5b126a49edaf2e1bf9c5b55b604ec7 100644
--- a/screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/genedata/FeatureStorageProcessorTest.java
+++ b/screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/genedata/FeatureStorageProcessorTest.java
@@ -17,27 +17,92 @@
package ch.systemsx.cisd.openbis.dss.etl.genedata;
import java.io.File;
+import java.io.IOException;
import java.util.List;
import java.util.Properties;
+import org.jmock.Expectations;
+import org.jmock.Mockery;
+import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import ch.systemsx.cisd.base.tests.AbstractFileSystemTestCase;
import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.etlserver.IStorageProcessor;
-import ch.systemsx.cisd.openbis.dss.etl.genedata.FeatureStorageProcessor;
+import ch.systemsx.cisd.etlserver.PlateDimensionParser;
+import ch.systemsx.cisd.openbis.dss.etl.dataaccess.IImagingUploadDAO;
+import ch.systemsx.cisd.openbis.dss.etl.dataaccess.ImgDatasetDTO;
+import ch.systemsx.cisd.openbis.dss.etl.dataaccess.ImgFeatureDefDTO;
+import ch.systemsx.cisd.openbis.dss.etl.dataaccess.ImgFeatureValuesDTO;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Experiment;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.GenericValueEntityProperty;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IEntityProperty;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.PropertyType;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Sample;
/**
* @author Franz-Josef Elmer
*/
public class FeatureStorageProcessorTest extends AbstractFileSystemTestCase
{
+ private static final String DATA_SET_PERM_ID = "dataset-1";
+
+ private static final String CONTAINER_PERM_ID = "perm12";
+
+ private static final String EXPERIMENT_PERM_ID = "perm11";
+
private static final String EXAMPLE1 =
"barcode = Plate_042" + "\n\n<Layer=alpha>\n" + "\t1\t2\n" + "A\t4.5\t4.6\n"
+ "B\t3.5\t5.6\n" + "C\t3.3\t5.7\n" + "\n\n<Layer=beta>\n" + "\t1\t2\n"
+ "A\t14.5\t14.6\n" + "B\t13.5\t15.6\n" + "C\t13.3\t15.7\n";
+ private Mockery context;
+
+ private IImagingUploadDAO dao;
+
+ @Override
+ @BeforeMethod
+ public void setUp() throws IOException
+ {
+ super.setUp();
+
+ context = new Mockery();
+ dao = context.mock(IImagingUploadDAO.class);
+
+ context.checking(new Expectations()
+ {
+ {
+ one(dao).tryGetExperimentIdByPermId(EXPERIMENT_PERM_ID);
+ will(returnValue((long) 1));
+ one(dao).tryGetContainerIdPermId(CONTAINER_PERM_ID);
+ will(returnValue((long) 1));
+
+ ImgDatasetDTO dataSetDTO = new ImgDatasetDTO(DATA_SET_PERM_ID, 3, 2, 1);
+ dataSetDTO.setId(1);
+ one(dao).tryGetDatasetByPermId(DATA_SET_PERM_ID);
+ will(returnValue(dataSetDTO));
+
+ ImgFeatureDefDTO featureDTO = new ImgFeatureDefDTO("alpha", "alpha", 1);
+ one(dao).addFeatureDef(with(equal(featureDTO)));
+ will(returnValue((long) 1));
+
+ one(dao).addFeatureValues(with(any(ImgFeatureValuesDTO.class)));
+ will(returnValue((long) 1));
+
+ featureDTO = new ImgFeatureDefDTO("beta", "beta", 1);
+ one(dao).addFeatureDef(with(equal(featureDTO)));
+ will(returnValue((long) 2));
+
+ one(dao).addFeatureValues(with(any(ImgFeatureValuesDTO.class)));
+ will(returnValue((long) 2));
+
+ one(dao).commit();
+ one(dao).close();
+ }
+ });
+ }
+
@Test
public void test()
{
@@ -47,9 +112,19 @@ public class FeatureStorageProcessorTest extends AbstractFileSystemTestCase
FileUtilities.writeToFile(dataSetFile, EXAMPLE1);
File rootDir = new File(workingDirectory, "rootDir");
rootDir.mkdirs();
- IStorageProcessor storageProcessor = new FeatureStorageProcessor(new Properties());
+ Properties storageProcessorProps = createStorageProcessorProperties();
+ IStorageProcessor storageProcessor = new FeatureStorageProcessor(storageProcessorProps)
+ {
+ // For Testing
+ @Override
+ protected IImagingUploadDAO createDAO()
+ {
+ return dao;
+ }
+ };
- storageProcessor.storeData(new DataSetInformation(), null, null, dataSetFile, rootDir);
+ DataSetInformation dataSetInfo = createDataSetInformation();
+ storageProcessor.storeData(dataSetInfo, null, null, dataSetFile, rootDir);
assertEquals(0, incomingDir.listFiles().length);
assertEquals(1, rootDir.listFiles().length);
@@ -59,8 +134,8 @@ public class FeatureStorageProcessorTest extends AbstractFileSystemTestCase
storageProcessor.commit();
- assertEquals(1, original.listFiles().length);
- File transformedDataSetFile = original.listFiles()[0];
+ assertEquals(2, original.listFiles().length);
+ File transformedDataSetFile = original.listFiles()[1];
assertEquals("Plate042.stat.txt", transformedDataSetFile.getName());
List<String> lines = FileUtilities.loadToStringList(transformedDataSetFile);
assertEquals("barcode;row;col;alpha;beta", lines.get(0));
@@ -72,4 +147,45 @@ public class FeatureStorageProcessorTest extends AbstractFileSystemTestCase
assertEquals("Plate_042;C;2;5.7;15.7", lines.get(6));
assertEquals(7, lines.size());
}
+
+ private DataSetInformation createDataSetInformation()
+ {
+ // Set the Experiment
+ DataSetInformation dataSetInfo = new DataSetInformation();
+ Experiment exp = new Experiment();
+ exp.setIdentifier("/Test/Test1/Exp1");
+ exp.setPermId(EXPERIMENT_PERM_ID);
+ dataSetInfo.setExperiment(exp);
+
+ // Set the Sample
+ Sample sample = new Sample();
+ sample.setCode("Samp1");
+ sample.setExperiment(exp);
+ sample.setPermId(CONTAINER_PERM_ID);
+ dataSetInfo.setSample(sample);
+
+ // Set the DataSet
+ dataSetInfo.setDataSetCode(DATA_SET_PERM_ID);
+
+ // Set the properties
+ IEntityProperty properties[] = new IEntityProperty[1];
+ GenericValueEntityProperty entityProperty = new GenericValueEntityProperty();
+ PropertyType propertyType = new PropertyType();
+ propertyType.setCode(PlateDimensionParser.PLATE_GEOMETRY_PROPERTY_NAME);
+ entityProperty.setPropertyType(propertyType);
+ entityProperty.setValue("A_2X2");
+ properties[0] = entityProperty;
+ dataSetInfo.setProperties(properties);
+ return dataSetInfo;
+ }
+
+ private Properties createStorageProcessorProperties()
+ {
+ Properties storageProcessorProps = new Properties();
+ storageProcessorProps.setProperty("processor",
+ "ch.systemsx.cisd.etlserver.DefaultStorageProcessor");
+ // storageProcessorProps.setProperty("data-source", "imaging-db");
+ storageProcessorProps.setProperty("data-source", "imaging-db");
+ return storageProcessorProps;
+ }
}
diff --git a/screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/genedata/GenedataFormatToCanonicalFeatureVectorTest.java b/screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/genedata/GenedataFormatToCanonicalFeatureVectorTest.java
index 764ff0a3559726f9c756baaec782d8580f8a873d..00e57286d1c4e0d7af5a5a871ba76036b8921f99 100644
--- a/screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/genedata/GenedataFormatToCanonicalFeatureVectorTest.java
+++ b/screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/genedata/GenedataFormatToCanonicalFeatureVectorTest.java
@@ -31,7 +31,9 @@ import ch.systemsx.cisd.openbis.dss.etl.featurevector.CanonicalFeatureVector;
import ch.systemsx.cisd.openbis.dss.etl.featurevector.GenedataFormatToCanonicalFeatureVector;
/**
- * @author Franz-Josef Elmer
+ * Check that Genedata feature vectors can be converted to the canonical form.
+ *
+ * @author Chandrasekhar Ramakrishnan
*/
public class GenedataFormatToCanonicalFeatureVectorTest extends AbstractFileSystemTestCase
{