diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/PlateGeometryOracle.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/PlateGeometryOracle.java index 6ad0623023aae6b19482c07e45bd0baece76d2c9..42473106e3ecaf6ccea6f3681047f1ff6407dfd1 100644 --- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/PlateGeometryOracle.java +++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/PlateGeometryOracle.java @@ -17,6 +17,7 @@ package ch.systemsx.cisd.openbis.dss.etl; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -26,7 +27,13 @@ import java.util.Map; import ch.systemsx.cisd.bds.hcs.Location; import ch.systemsx.cisd.common.exceptions.UserFailureException; import ch.systemsx.cisd.common.shared.basic.utils.StringUtils; +import ch.systemsx.cisd.etlserver.registrator.DataSetRegistrationDetails; +import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.ImageDataSetInformation; +import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageFileInfo; +import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.VocabularyTerm; import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.Geometry; +import ch.systemsx.cisd.openbis.plugin.screening.shared.basic.dto.ScreeningConstants; import ch.systemsx.cisd.openbis.plugin.screening.shared.basic.dto.WellLocation; /** @@ -37,6 +44,39 @@ import ch.systemsx.cisd.openbis.plugin.screening.shared.basic.dto.WellLocation; */ public class PlateGeometryOracle { + public static String figureGeometry( + DataSetRegistrationDetails<ImageDataSetInformation> registrationDetails, + IEncapsulatedOpenBISService openBisService) + { + List<ImageFileInfo> images = + registrationDetails.getDataSetInformation().getImageDataSetStructure().getImages(); + List<WellLocation> locations = extractLocations(images); + List<String> plateGeometries = + loadPlateGeometries(openBisService); + return PlateGeometryOracle.figureGeometry(locations, plateGeometries); + } + + private static List<String> loadPlateGeometries(IEncapsulatedOpenBISService openbisService) + { + Collection<VocabularyTerm> terms = + openbisService.listVocabularyTerms(ScreeningConstants.PLATE_GEOMETRY); + List<String> plateGeometries = new ArrayList<String>(); + for (VocabularyTerm v : terms) + { + plateGeometries.add(v.getCode()); + } + return plateGeometries; + } + + private static List<WellLocation> extractLocations(List<ImageFileInfo> images) + { + List<WellLocation> locations = new ArrayList<WellLocation>(); + for (ImageFileInfo image : images) + { + locations.add(image.tryGetWellLocation()); + } + return locations; + } public static String figureGeometry(List<WellLocation> plateLocations, List<String> plateGeometries) diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/FeatureVectorDataSet.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/FeatureVectorDataSet.java index c4c8d1040b1e6b298493359516ebd80ada2f4d7a..0597293943e8008b78a56259e3ae8c2938ef3ea7 100644 --- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/FeatureVectorDataSet.java +++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/FeatureVectorDataSet.java @@ -22,6 +22,7 @@ import java.util.List; import ch.systemsx.cisd.etlserver.registrator.DataSetRegistrationDetails; import ch.systemsx.cisd.etlserver.registrator.api.v1.impl.DataSet; import ch.systemsx.cisd.openbis.dss.etl.dto.api.v2.IFeatureVectorDataSet; +import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService; import ch.systemsx.cisd.openbis.dss.generic.shared.api.internal.v1.IDataSetImmutable; import ch.systemsx.cisd.openbis.dss.generic.shared.api.internal.v1.IExperimentImmutable; import ch.systemsx.cisd.openbis.dss.generic.shared.api.internal.v1.IExternalDataManagementSystemImmutable; @@ -33,19 +34,23 @@ import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.DataSetType; * * @author Franz-Josef Elmer */ -public class FeatureVectorDataSet implements IFeatureVectorDataSet +public class FeatureVectorDataSet extends DataSet<FeatureVectorDataSetInformation> implements + IFeatureVectorDataSet { private final DataSet<FeatureVectorDataSetInformation> dataSet; - public FeatureVectorDataSet(DataSet<FeatureVectorDataSetInformation> dataSet) + public FeatureVectorDataSet(DataSet<FeatureVectorDataSetInformation> dataSet, + IEncapsulatedOpenBISService service) { + super(dataSet.getRegistrationDetails(), dataSet.getDataSetStagingFolder(), service); this.dataSet = dataSet; } @Override public void setAnalysisProcedure(String analysisProcedure) { - dataSet.getRegistrationDetails().getDataSetInformation().setAnalysisProcedure(analysisProcedure); + dataSet.getRegistrationDetails().getDataSetInformation() + .setAnalysisProcedure(analysisProcedure); } @Override @@ -54,11 +59,13 @@ public class FeatureVectorDataSet implements IFeatureVectorDataSet return dataSet.equals(obj); } + @Override public DataSetRegistrationDetails<? extends FeatureVectorDataSetInformation> getRegistrationDetails() { return dataSet.getRegistrationDetails(); } + @Override public File getDataSetStagingFolder() { return dataSet.getDataSetStagingFolder(); @@ -263,6 +270,7 @@ public class FeatureVectorDataSet implements IFeatureVectorDataSet return dataSet.toString(); } + @Override public File tryDataSetContents() { return dataSet.tryDataSetContents(); diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/ImageContainerDataSet.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/ImageContainerDataSet.java index 8d47609679f34d8cc5adc4926eb9780a4a64899e..20944b729310f5aca62374767b5494732a0b8772 100644 --- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/ImageContainerDataSet.java +++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/ImageContainerDataSet.java @@ -23,6 +23,7 @@ import java.util.List; import ch.systemsx.cisd.etlserver.registrator.DataSetRegistrationDetails; import ch.systemsx.cisd.etlserver.registrator.api.v1.IDataSet; import ch.systemsx.cisd.etlserver.registrator.api.v1.impl.DataSet; +import ch.systemsx.cisd.openbis.dss.etl.PlateGeometryOracle; import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.IImageDataSet; import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService; import ch.systemsx.cisd.openbis.dss.generic.shared.api.internal.v1.IExperimentImmutable; @@ -124,4 +125,16 @@ public class ImageContainerDataSet extends DataSet<DataSetInformation> implement } } + @Override + public String figureGeometry() + { + return PlateGeometryOracle.figureGeometry(getImageRegistrationDetails(), service); + } + + @SuppressWarnings("unchecked") + private DataSetRegistrationDetails<ImageDataSetInformation> getImageRegistrationDetails() + { + return (DataSetRegistrationDetails<ImageDataSetInformation>) originalDataset + .getRegistrationDetails(); + } } diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/IImageDataSet.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/IImageDataSet.java index 221f500fa40d9662f4d8ac0d96912dd30f12c9a1..b613532c31bdd5a032afec1bfa7e0bb9794e9547 100644 --- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/IImageDataSet.java +++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/IImageDataSet.java @@ -18,6 +18,7 @@ package ch.systemsx.cisd.openbis.dss.etl.dto.api.v1; import java.util.List; +import ch.systemsx.cisd.common.exceptions.UserFailureException; import ch.systemsx.cisd.etlserver.registrator.api.v1.IDataSet; /** @@ -41,4 +42,16 @@ public interface IImageDataSet extends IDataSet * By default there will be no such links. */ void establishSampleLinkForContainedDataSets(); + + /** + * Utility method to find out the plate geometry by looking for which wells images are + * available. + * + * @return a constant which can be used as a vocabulary term value for $PLATE_GEOMETRY property + * of a plate/ + * @throws UserFailureException if all available geometries in openBIS are too small (there is a + * well outside). + */ + String figureGeometry(); + } diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/SimpleImageDataConfig.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/SimpleImageDataConfig.java index 837ab9711bbed2835aec5f8d4e571b72ba46af44..c68f5bd0a3723ee9cf2b148401a13522a44b8f69 100644 --- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/SimpleImageDataConfig.java +++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/SimpleImageDataConfig.java @@ -242,6 +242,10 @@ abstract public class SimpleImageDataConfig private String thumbnailsFileFormat; + private List<Channel> channels; + + private List<ChannelColorComponent> channelColorComponentsOrNull; + // --- getters & setters ---------------------------------------------- public ImageStorageConfiguraton getImageStorageConfiguration() @@ -331,6 +335,16 @@ abstract public class SimpleImageDataConfig { return computeCommonIntensityRangeOfAllImagesIsDefault; } + + public List<Channel> getChannels() + { + return channels; + } + + public List<ChannelColorComponent> getChannelColorComponentsOrNull() + { + return channelColorComponentsOrNull; + } // ----- Setters ------------------------- @@ -355,6 +369,24 @@ abstract public class SimpleImageDataConfig this.recognizedImageExtensions = recognizedImageExtensions; } + /** Sets all channels available in the data set. */ + public void setChannels(List<Channel> channels) + { + this.channels = channels; + } + + /** + * Use this method if channels are encoded in color components of one image (or in other words: + * each image contains merged channels). For each channel you have to specify the corresponding + * color component of the image. + */ + public void setChannels(List<Channel> channels, + List<ChannelColorComponent> channelColorComponents) + { + this.channels = channels; + channelColorComponentsOrNull = channelColorComponents; + } + /** should thumbnails be generated? False by default. */ public void setGenerateThumbnails(boolean generateThumbnails) { diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/JythonPlateDatasetFactory.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/JythonPlateDatasetFactory.java index 5b709cd9c4e4248416edb9f50f3c19d0d163e506..9d7cf8cc350f71b3a7a6ee4080eb4ecd0d3f945f 100644 --- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/JythonPlateDatasetFactory.java +++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/JythonPlateDatasetFactory.java @@ -130,34 +130,8 @@ public class JythonPlateDatasetFactory extends JythonObjectFactory<DataSetInform public String figureGeometry( DataSetRegistrationDetails<ImageDataSetInformation> registrationDetails) { - List<ImageFileInfo> images = - registrationDetails.getDataSetInformation().getImageDataSetStructure().getImages(); - List<WellLocation> locations = extractLocations(images); - List<String> plateGeometries = - loadPlateGeometries(registratorState.getGlobalState().getOpenBisService()); - return PlateGeometryOracle.figureGeometry(locations, plateGeometries); - } - - private static List<String> loadPlateGeometries(IEncapsulatedOpenBISService openbisService) - { - Collection<VocabularyTerm> terms = - openbisService.listVocabularyTerms(ScreeningConstants.PLATE_GEOMETRY); - List<String> plateGeometries = new ArrayList<String>(); - for (VocabularyTerm v : terms) - { - plateGeometries.add(v.getCode()); - } - return plateGeometries; - } - - private static List<WellLocation> extractLocations(List<ImageFileInfo> images) - { - List<WellLocation> locations = new ArrayList<WellLocation>(); - for (ImageFileInfo image : images) - { - locations.add(image.tryGetWellLocation()); - } - return locations; + IEncapsulatedOpenBISService openBisService = registratorState.getGlobalState().getOpenBisService(); + return PlateGeometryOracle.figureGeometry(registrationDetails, openBisService); } // ---- diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/SimpleImageDataSetRegistrator.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/SimpleImageDataSetRegistrator.java index b945d108c8781f96db7d34260b7c911cfae93d0a..f22766dcd323a7cdc9946cb13d07899a22e53f58 100644 --- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/SimpleImageDataSetRegistrator.java +++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/SimpleImageDataSetRegistrator.java @@ -49,6 +49,7 @@ import ch.systemsx.cisd.openbis.dss.etl.dto.ImageLibraryInfo; import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.ImageDataSetInformation; import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.ImageDataSetStructure; import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.Channel; +import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ChannelColorComponent; import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageFileInfo; import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageIdentifier; import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageMetadata; @@ -162,6 +163,15 @@ public class SimpleImageDataSetRegistrator imageDatasetFactory.createDataSetRegistrationDetails(); ImageDataSetInformation imageDataset = registrationDetails.getDataSetInformation(); setImageDataset(incoming, imageDataset); + List<ChannelColorComponent> channelColorComponentsOrNull = + simpleImageConfig.getChannelColorComponentsOrNull(); + if (channelColorComponentsOrNull == null) + { + imageDataset.setChannels(simpleImageConfig.getChannels()); + } else + { + imageDataset.setChannels(simpleImageConfig.getChannels(), channelColorComponentsOrNull); + } setRegistrationDetails(registrationDetails, imageDataset); return registrationDetails; } diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/v2/ImagingDataSetRegistrationTransactionV2Delegate.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/v2/ImagingDataSetRegistrationTransactionV2Delegate.java index 1cf09f1bb3d506b145ec80af4c9418e119b52335..9bdbafaa6ac5c84ec08300d4d28705c2ab1bfe34 100644 --- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/v2/ImagingDataSetRegistrationTransactionV2Delegate.java +++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/v2/ImagingDataSetRegistrationTransactionV2Delegate.java @@ -103,7 +103,7 @@ public class ImagingDataSetRegistrationTransactionV2Delegate implements DataSet<FeatureVectorDataSetInformation> dataSet = (DataSet<FeatureVectorDataSetInformation>) transaction .createNewDataSet(registrationDetails); - return new FeatureVectorDataSet(dataSet); + return new FeatureVectorDataSet(dataSet, transaction.getGlobalState().getOpenBisService()); } @Override diff --git a/screening/sourceTest/core-plugins/AggregatedFeatureVectorsTest/1/dss/drop-boxes/AggregatedFeatureVectorsTest-drop-box/data-set-handler.py b/screening/sourceTest/core-plugins/AggregatedFeatureVectorsTest/1/dss/drop-boxes/AggregatedFeatureVectorsTest-drop-box/data-set-handler.py index dee6dbe3c12e41978bd344d3c38d789f405a59ec..2c76f4d32d326869e9deb5e23fc2125b540b7295 100644 --- a/screening/sourceTest/core-plugins/AggregatedFeatureVectorsTest/1/dss/drop-boxes/AggregatedFeatureVectorsTest-drop-box/data-set-handler.py +++ b/screening/sourceTest/core-plugins/AggregatedFeatureVectorsTest/1/dss/drop-boxes/AggregatedFeatureVectorsTest-drop-box/data-set-handler.py @@ -1,5 +1,3 @@ -from ch.systemsx.cisd.openbis.dss.etl.dto.api.v2 import SimpleFeatureVectorDataConfig - PLATE_GEOMETRY_PROPERTY_CODE = "$PLATE_GEOMETRY" PLATE_GEOMETRY = "384_WELLS_16X24" @@ -34,16 +32,16 @@ def create_plate(tr, experiment, plateCode, gene): return plate -def create_analysis_data_set(tr, plate, config, analysis_procedure, ds_file): - analysis_data_set = tr.createNewFeatureVectorDataSet(config, None) +def create_analysis_data_set(tr, plate, builder, analysis_procedure, ds_file): + analysis_registration_details = factory.createFeatureVectorDatasetDetails(builder) + analysis_data_set = tr.createNewDataSet(analysis_registration_details) analysis_data_set.setSample(plate) analysis_data_set.setPropertyValue("$ANALYSIS_PROCEDURE", analysis_procedure) - analysis_data_set_file = tr.moveFile(tr.incoming.getPath() + "/" + ds_file, analysis_data_set) + analysis_data_set_file = tr.moveFile(incoming.getPath() + "/" + ds_file, analysis_data_set) def create_dataset_with_features1(tr, experiment, gene): plate1 = create_plate(tr, experiment, "PLATE1", gene) - config = SimpleFeatureVectorDataConfig() - builder = config.featuresBuilder + builder = factory.createFeaturesBuilder() featureX = builder.defineFeature("X") featureX.addValue(1, 1, "1") @@ -53,12 +51,11 @@ def create_dataset_with_features1(tr, experiment, gene): featureY.addValue(1, 1, "3") featureY.addValue(1, 2, "2") - create_analysis_data_set(tr, plate1, config, "p1", "data-set-1.csv") + create_analysis_data_set(tr, plate1, builder, "p1", "data-set-1.csv") def create_dataset_with_features2(tr, experiment, gene): plate2 = create_plate(tr, experiment, "PLATE2", gene) - config = SimpleFeatureVectorDataConfig() - builder = config.featuresBuilder + builder = factory.createFeaturesBuilder() featureA = builder.defineFeature("A") featureA.addValue(1, 1, "10") @@ -72,19 +69,18 @@ def create_dataset_with_features2(tr, experiment, gene): featureX.addValue(1, 1, "5") featureX.addValue(1, 2, "6") - create_analysis_data_set(tr, plate2, config, "p2", "data-set-2.file") + create_analysis_data_set(tr, plate2, builder, "p2", "data-set-2.file") -def process(transaction): - incoming = transaction.incoming - experiment = create_experiment(transaction) +tr = service.transaction() +experiment = create_experiment(tr) - geneCode = "G" - geneG = transaction.createNewMaterial(geneCode, "GENE") +geneCode = "G" +geneG = tr.createNewMaterial(geneCode, "GENE") - create_dataset_with_features1(transaction, experiment, geneG) - create_dataset_with_features2(transaction, experiment, geneG) +create_dataset_with_features1(tr, experiment, geneG) +create_dataset_with_features2(tr, experiment, geneG) - # delete the empty incoming, its - # contents have been moved to the data sets - incoming.delete() \ No newline at end of file +# delete the empty incoming, its +# contents have been moved to the data sets +incoming.delete() \ No newline at end of file diff --git a/screening/sourceTest/core-plugins/AggregatedFeatureVectorsTest/1/dss/drop-boxes/AggregatedFeatureVectorsTest-drop-box/plugin.properties b/screening/sourceTest/core-plugins/AggregatedFeatureVectorsTest/1/dss/drop-boxes/AggregatedFeatureVectorsTest-drop-box/plugin.properties index c4cbfdd9875c3e5540664a6b21eb9521133f52b3..2af97cd0ad789825fb8575fd54c6e1a21ac014d7 100644 --- a/screening/sourceTest/core-plugins/AggregatedFeatureVectorsTest/1/dss/drop-boxes/AggregatedFeatureVectorsTest-drop-box/plugin.properties +++ b/screening/sourceTest/core-plugins/AggregatedFeatureVectorsTest/1/dss/drop-boxes/AggregatedFeatureVectorsTest-drop-box/plugin.properties @@ -1,7 +1,7 @@ incoming-dir = ${root-dir}/incoming-AggregatedFeatureVectorsTest incoming-dir-create = true incoming-data-completeness-condition = auto-detection -top-level-data-set-handler = ch.systemsx.cisd.openbis.dss.etl.jython.v2.JythonPlateDataSetHandlerV2 +top-level-data-set-handler = ch.systemsx.cisd.openbis.dss.etl.jython.JythonPlateDataSetHandler script-path = data-set-handler.py storage-processor = ch.systemsx.cisd.openbis.dss.etl.featurevector.FeatureVectorStorageProcessor storage-processor.processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor diff --git a/screening/sourceTest/core-plugins/AggregatedFeatureVectorsV2Test/1/dss/drop-boxes/AggregatedFeatureVectorsV2Test-drop-box/data-set-handler.py b/screening/sourceTest/core-plugins/AggregatedFeatureVectorsV2Test/1/dss/drop-boxes/AggregatedFeatureVectorsV2Test-drop-box/data-set-handler.py new file mode 100644 index 0000000000000000000000000000000000000000..dee6dbe3c12e41978bd344d3c38d789f405a59ec --- /dev/null +++ b/screening/sourceTest/core-plugins/AggregatedFeatureVectorsV2Test/1/dss/drop-boxes/AggregatedFeatureVectorsV2Test-drop-box/data-set-handler.py @@ -0,0 +1,90 @@ +from ch.systemsx.cisd.openbis.dss.etl.dto.api.v2 import SimpleFeatureVectorDataConfig + +PLATE_GEOMETRY_PROPERTY_CODE = "$PLATE_GEOMETRY" +PLATE_GEOMETRY = "384_WELLS_16X24" + +def create_experiment(tr): + space = tr.getSpace("TEST") + if space == None: + space = tr.createNewSpace("TEST", "etlserver") + project = tr.getProject("/TEST/TEST-PROJECT") + if project == None: + project = tr.createNewProject("/TEST/TEST-PROJECT") + expid = "/TEST/TEST-PROJECT/AGGREGATED_FEATURES_EXP" + + exp = tr.createNewExperiment(expid, 'SIRNA_HCS') + exp.setPropertyValue("DESCRIPTION", "Test experiment") + + return exp + +def create_plate(tr, experiment, plateCode, gene): + plateId = "/TEST/" + plateCode + plate = tr.createNewSample(plateId, 'PLATE') + plate.setPropertyValue(PLATE_GEOMETRY_PROPERTY_CODE, PLATE_GEOMETRY) + plate.setExperiment(experiment) + + wellA1 = tr.createNewSample(plate.getSampleIdentifier() + ":A1", "SIRNA_WELL") + wellA1.setPropertyValue("GENE", gene.getMaterialIdentifier()) + wellA1.setContainer(plate) + + wellA2 = tr.createNewSample(plate.getSampleIdentifier() + ":A2", "SIRNA_WELL") + wellA2.setPropertyValue("GENE", gene.getMaterialIdentifier()) + wellA2.setContainer(plate) + + return plate + + +def create_analysis_data_set(tr, plate, config, analysis_procedure, ds_file): + analysis_data_set = tr.createNewFeatureVectorDataSet(config, None) + analysis_data_set.setSample(plate) + analysis_data_set.setPropertyValue("$ANALYSIS_PROCEDURE", analysis_procedure) + analysis_data_set_file = tr.moveFile(tr.incoming.getPath() + "/" + ds_file, analysis_data_set) + +def create_dataset_with_features1(tr, experiment, gene): + plate1 = create_plate(tr, experiment, "PLATE1", gene) + config = SimpleFeatureVectorDataConfig() + builder = config.featuresBuilder + + featureX = builder.defineFeature("X") + featureX.addValue(1, 1, "1") + featureX.addValue(1, 2, "2") + + featureY = builder.defineFeature("Y") + featureY.addValue(1, 1, "3") + featureY.addValue(1, 2, "2") + + create_analysis_data_set(tr, plate1, config, "p1", "data-set-1.csv") + +def create_dataset_with_features2(tr, experiment, gene): + plate2 = create_plate(tr, experiment, "PLATE2", gene) + config = SimpleFeatureVectorDataConfig() + builder = config.featuresBuilder + + featureA = builder.defineFeature("A") + featureA.addValue(1, 1, "10") + featureA.addValue(1, 2, "20") + + featureB = builder.defineFeature("B") + featureB.addValue(1, 1, "2") + featureB.addValue(1, 2, "NaN") + + featureX = builder.defineFeature("X") + featureX.addValue(1, 1, "5") + featureX.addValue(1, 2, "6") + + create_analysis_data_set(tr, plate2, config, "p2", "data-set-2.file") + + +def process(transaction): + incoming = transaction.incoming + experiment = create_experiment(transaction) + + geneCode = "G" + geneG = transaction.createNewMaterial(geneCode, "GENE") + + create_dataset_with_features1(transaction, experiment, geneG) + create_dataset_with_features2(transaction, experiment, geneG) + + # delete the empty incoming, its + # contents have been moved to the data sets + incoming.delete() \ No newline at end of file diff --git a/screening/sourceTest/core-plugins/AggregatedFeatureVectorsV2Test/1/dss/drop-boxes/AggregatedFeatureVectorsV2Test-drop-box/plugin.properties b/screening/sourceTest/core-plugins/AggregatedFeatureVectorsV2Test/1/dss/drop-boxes/AggregatedFeatureVectorsV2Test-drop-box/plugin.properties new file mode 100644 index 0000000000000000000000000000000000000000..aaad6d3f89c2ff141359fd5ffc09e91c7f8b401e --- /dev/null +++ b/screening/sourceTest/core-plugins/AggregatedFeatureVectorsV2Test/1/dss/drop-boxes/AggregatedFeatureVectorsV2Test-drop-box/plugin.properties @@ -0,0 +1,8 @@ +incoming-dir = ${root-dir}/incoming-AggregatedFeatureVectorsV2Test +incoming-dir-create = true +incoming-data-completeness-condition = auto-detection +top-level-data-set-handler = ch.systemsx.cisd.openbis.dss.etl.jython.v2.JythonPlateDataSetHandlerV2 +script-path = data-set-handler.py +storage-processor = ch.systemsx.cisd.openbis.dss.etl.featurevector.FeatureVectorStorageProcessor +storage-processor.processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor +storage-processor.data-source = imaging-db \ No newline at end of file diff --git a/screening/sourceTest/core-plugins/AggregatedFeatureVectorsV2Test/1/dss/reporting-plugins/AggregatedFeatureVectorsV2Test-viewer/plugin.properties b/screening/sourceTest/core-plugins/AggregatedFeatureVectorsV2Test/1/dss/reporting-plugins/AggregatedFeatureVectorsV2Test-viewer/plugin.properties new file mode 100644 index 0000000000000000000000000000000000000000..a9f9c0846b62fcce959264983b9615989b35cc8f --- /dev/null +++ b/screening/sourceTest/core-plugins/AggregatedFeatureVectorsV2Test/1/dss/reporting-plugins/AggregatedFeatureVectorsV2Test-viewer/plugin.properties @@ -0,0 +1,9 @@ +label = Viewer +class = ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.DecoratingTableModelReportingPlugin +dataset-types = HCS_ANALYSIS_WELL_FEATURES +reporting-plugin.class = ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.TSVViewReportingPlugin +reporting-plugin.separator = , +transformation.class = ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.EntityLinksDecorator +transformation.link-columns = GENEID +transformation.GENEID.entity-kind = MATERIAL +transformation.GENEID.material-type = GENE \ No newline at end of file diff --git a/screening/sourceTest/core-plugins/TransformedImageRepresentationsV2Test/1/dss/drop-boxes/TransformedImageRepresentationsV2Test-drop-box/data-set-handler.py b/screening/sourceTest/core-plugins/TransformedImageRepresentationsV2Test/1/dss/drop-boxes/TransformedImageRepresentationsV2Test-drop-box/data-set-handler.py new file mode 100644 index 0000000000000000000000000000000000000000..2a26deec724dabc7793b28de62019679b9bde88a --- /dev/null +++ b/screening/sourceTest/core-plugins/TransformedImageRepresentationsV2Test/1/dss/drop-boxes/TransformedImageRepresentationsV2Test-drop-box/data-set-handler.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python + +PLATE_GEOMETRY_PROPERTY_CODE = "$PLATE_GEOMETRY" +PLATE_GEOMETRY = "384_WELLS_16X24" + +""" +An Jython dropbox for importing HCS image datasets for use by the TransformedImageRepresentationsV2Test + +The folder loaded to the dropbox folder should have the same name as the plate that the data will be attached to. +""" + +import os +from ch.systemsx.cisd.openbis.dss.etl.dto.api.v1 import SimpleImageDataConfig, ImageMetadata, Location, Channel, ChannelColor, ChannelColorComponent +from ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto import Geometry +from ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.transformations import ImageTransformationBuffer +from ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.thumbnails import ResolutionBasedThumbnailsConfiguration + + +class ImageDataSetFlexible(SimpleImageDataConfig): + def extractImageMetadata(self, imagePath): + """ + Extracts tile number, channel code and well code for a given relative path to an image. + Will be called for each file found in the incoming directory which has the allowed image extension. + + Example file name: bDZ01-1A_wD17_s3_z123_t321_cGFP + Returns: + ImageMetadata + """ + image_tokens = ImageMetadata() + + basename = os.path.splitext(imagePath)[0] + # + token_dict = {} + for token in basename.split("_"): + token_dict[token[:1]] = token[1:] + + image_tokens.well = token_dict["w"] + fieldText = token_dict["s"] + try: + image_tokens.tileNumber = int(fieldText) + except ValueError: + raise Exception("Cannot parse field number from '" + fieldText + "' in '" + basename + "' file name.") + + image_tokens.channelCode = token_dict["c"] + return image_tokens + + def getTileGeometry(self, imageTokens, maxTileNumber): + """ + Overrides the default implementation which returns (1, maxTileNumber) geometry. + + Calculates the width and height of the matrix of tiles (a.k.a. fields or sides) in the well. + + Parameter imageMetadataList: a list of metadata for each encountered image + Parameter maxTileNumber: the biggest tile number among all encountered images + Returns: + Geometry + """ + return Geometry.createFromRowColDimensions(1, 1); + + def getTileCoordinates(self, tileNumber, tileGeometry): + """ + Overrides the default implementation which does the same thing (to demonstrate how this can be done). + + For a given tile number and tiles geometry returns (x,y) which describes where the tile is + located on the well. + + Parameter tileNumber: number of the tile + Parameter tileGeometry: the geometry of the well matrix + Returns: + Location + """ + columns = tileGeometry.getWidth() + row = ((tileNumber - 1) / columns) + 1 + col = ((tileNumber - 1) % columns) + 1 + return Location(row, col) + + +def getAvailableChannelTransformations(): + """ + Create a collection of transformations that are applicable to the image + """ + transforms = ImageTransformationBuffer() + transforms.appendImageMagicConvert("-edge 1", "Edge detection") + transforms.appendImageMagicConvert("-radial-blur 30", "Radial Blur") + transforms.appendImageMagicConvert("-blur 3x.7 -solarize 50% -level 50%,0", "Fuzzy") + transforms.appendImageMagicConvert("-shade 0x45", "3D 1") + transforms.appendImageMagicConvert("-shade 90x60", "3D 2") + transforms.appendImageMagicConvert("-blur 0x3 -shade 120x45 -normalize", "3D 3") + transforms.appendImageMagicConvert("-motion-blur 0x12+45", "Motion Blur") + transforms.appendImageMagicConvert("-fft -delete 1 -auto-level -evaluate log 100000", "FFT") + + return transforms.getTransformations() + +def create_experiment(tr): + space = tr.getSpace("TEST") + if space == None: + space = tr.createNewSpace("TEST", "etlserver") + project = tr.getProject("/TEST/TEST-PROJECT") + if project == None: + project = tr.createNewProject("/TEST/TEST-PROJECT") + expid = "/TEST/TEST-PROJECT/TRANSFORMED_THUMBNAILS_EXP" + + exp = tr.createNewExperiment(expid, 'SIRNA_HCS') + exp.setPropertyValue("DESCRIPTION", "Test experiment") + + return exp + +def create_plate(tr, experiment, plateCode): + plateId = "/TEST/" + plateCode + plate = tr.createNewSample(plateId, 'PLATE') + plate.setPropertyValue(PLATE_GEOMETRY_PROPERTY_CODE, PLATE_GEOMETRY) + plate.setExperiment(experiment) + + wellA1 = tr.createNewSample(plate.getSampleIdentifier() + ":A1", "SIRNA_WELL") + wellA1.setContainer(plate) + + wellA2 = tr.createNewSample(plate.getSampleIdentifier() + ":A2", "SIRNA_WELL") + wellA2.setContainer(plate) + + return plate + + +def process(transaction): + incoming = transaction.incoming + if incoming.isDirectory(): + experiment = create_experiment(transaction) + plate = create_plate(transaction, experiment, 'TRANSFORMED-THUMB-PLATE') + + imageDataset = ImageDataSetFlexible() + imageDataset.setRawImageDatasetType() + imageDataset.setPlate("TEST", 'TRANSFORMED-THUMB-PLATE') + transforms = getAvailableChannelTransformations() + # We want thumbnails generarted for the following resolutions, and they should be JPEG and have the + # Radial Blur transform applied + for resolution in ['64x64', '128x128', '256x256']: + representation = imageDataset.addGeneratedImageRepresentationWithResolution(resolution) + for channel in ["DAPI", "GFP", "Cy5"]: + representation.setTransformation(channel, transforms[1].getCode()) + representation.setFileFormat('JPEG') + + channels = [ Channel(code, code) for code in ["DAPI", "GFP", "Cy5"]] + colorComponents = [ ChannelColorComponent.BLUE, ChannelColorComponent.GREEN, ChannelColorComponent.RED] + # Add transforms to the channels + for channel in channels: + channel.setAvailableTransformations(transforms) + imageDataset.setChannels(channels, colorComponents) + + dataSet = transaction.createNewImageDataSet(imageDataset, incoming) + transaction.moveFile(incoming.getPath(), dataSet) diff --git a/screening/sourceTest/core-plugins/TransformedImageRepresentationsV2Test/1/dss/drop-boxes/TransformedImageRepresentationsV2Test-drop-box/plugin.properties b/screening/sourceTest/core-plugins/TransformedImageRepresentationsV2Test/1/dss/drop-boxes/TransformedImageRepresentationsV2Test-drop-box/plugin.properties new file mode 100644 index 0000000000000000000000000000000000000000..c7e21a5b85c4d343560a0c541f585ede0937f0db --- /dev/null +++ b/screening/sourceTest/core-plugins/TransformedImageRepresentationsV2Test/1/dss/drop-boxes/TransformedImageRepresentationsV2Test-drop-box/plugin.properties @@ -0,0 +1,8 @@ +incoming-dir = ${root-dir}/incoming-TransformedImageRepresentationsV2Test +incoming-dir-create = true +incoming-data-completeness-condition = auto-detection +top-level-data-set-handler = ch.systemsx.cisd.openbis.dss.etl.jython.v2.JythonPlateDataSetHandlerV2 +script-path = data-set-handler.py +storage-processor = ch.systemsx.cisd.openbis.dss.etl.PlateStorageProcessor +storage-processor.processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor +storage-processor.data-source = imaging-db \ No newline at end of file diff --git a/screening/sourceTest/java/ch/systemsx/cisd/openbis/screening/systemtests/AggregatedFeatureVectorsV2Test.java b/screening/sourceTest/java/ch/systemsx/cisd/openbis/screening/systemtests/AggregatedFeatureVectorsV2Test.java new file mode 100644 index 0000000000000000000000000000000000000000..f0de124a8f9c6d9a5823b264d45da357805705d9 --- /dev/null +++ b/screening/sourceTest/java/ch/systemsx/cisd/openbis/screening/systemtests/AggregatedFeatureVectorsV2Test.java @@ -0,0 +1,31 @@ +/* + * Copyright 2012 ETH Zuerich, CISD + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.systemsx.cisd.openbis.screening.systemtests; + +import org.testng.annotations.Test; + +/** + * + * + * @author Franz-Josef Elmer + */ +@Test(groups = +{ "slow", "systemtest" }) +public class AggregatedFeatureVectorsV2Test extends AggregatedFeatureVectorsTest +{ + +} diff --git a/screening/sourceTest/java/ch/systemsx/cisd/openbis/screening/systemtests/TransformedImageRepresentationsTest.java b/screening/sourceTest/java/ch/systemsx/cisd/openbis/screening/systemtests/TransformedImageRepresentationsTest.java index 94d49863226207876ce2ef9e62647d11e746b5e9..cdf67b920e48915c66da9512fddd50d5f6f37bb9 100644 --- a/screening/sourceTest/java/ch/systemsx/cisd/openbis/screening/systemtests/TransformedImageRepresentationsTest.java +++ b/screening/sourceTest/java/ch/systemsx/cisd/openbis/screening/systemtests/TransformedImageRepresentationsTest.java @@ -142,7 +142,7 @@ public class TransformedImageRepresentationsTest extends AbstractScreeningSystem private String getTestDataFolder() { - return "../screening/resource/test-data/" + getClass().getSimpleName() + "/"; + return "../screening/resource/test-data/TransformedImageRepresentationsTest/"; } @Override diff --git a/screening/sourceTest/java/ch/systemsx/cisd/openbis/screening/systemtests/TransformedImageRepresentationsV2Test.java b/screening/sourceTest/java/ch/systemsx/cisd/openbis/screening/systemtests/TransformedImageRepresentationsV2Test.java new file mode 100644 index 0000000000000000000000000000000000000000..565cc42639576ec637f9b23e31d2caaa0a499fe9 --- /dev/null +++ b/screening/sourceTest/java/ch/systemsx/cisd/openbis/screening/systemtests/TransformedImageRepresentationsV2Test.java @@ -0,0 +1,31 @@ +/* + * Copyright 2012 ETH Zuerich, CISD + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.systemsx.cisd.openbis.screening.systemtests; + +import org.testng.annotations.Test; + +/** + * + * + * @author Franz-Josef Elmer + */ +@Test(groups = + { "slow", "systemtest" }) +public class TransformedImageRepresentationsV2Test extends TransformedImageRepresentationsTest +{ + +}