SP-179, BIS-21: Adding feature vector support.

SVN: 26044

screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/FeatureVectorDataSet.java

+/*
+ * Copyright 2012 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.etl.dto.api.impl;
+
+import java.io.File;
+import java.util.List;
+
+import ch.systemsx.cisd.etlserver.registrator.DataSetRegistrationDetails;
+import ch.systemsx.cisd.etlserver.registrator.api.v1.impl.DataSet;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v2.IFeatureVectorDataSet;
+import ch.systemsx.cisd.openbis.dss.generic.shared.api.internal.v1.IDataSetImmutable;
+import ch.systemsx.cisd.openbis.dss.generic.shared.api.internal.v1.IExperimentImmutable;
+import ch.systemsx.cisd.openbis.dss.generic.shared.api.internal.v1.IExternalDataManagementSystemImmutable;
+import ch.systemsx.cisd.openbis.dss.generic.shared.api.internal.v1.ISampleImmutable;
+import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.DataSetType;
+
+/**
+ * 
+ *
+ * @author Franz-Josef Elmer
+ */
+public class FeatureVectorDataSet implements IFeatureVectorDataSet
+{
+    private final DataSet<FeatureVectorDataSetInformation> dataSet;
+
+    public FeatureVectorDataSet(DataSet<FeatureVectorDataSetInformation> dataSet)
+    {
+        this.dataSet = dataSet;
+    }
+
+    @Override
+    public void setAnalysisProcedure(String analysisProcedure)
+    {
+        dataSet.getRegistrationDetails().getDataSetInformation().setAnalysisProcedure(analysisProcedure);
+    }
+
+    @Override
+    public boolean equals(Object obj)
+    {
+        return dataSet.equals(obj);
+    }
+
+    public DataSetRegistrationDetails<? extends FeatureVectorDataSetInformation> getRegistrationDetails()
+    {
+        return dataSet.getRegistrationDetails();
+    }
+
+    public File getDataSetStagingFolder()
+    {
+        return dataSet.getDataSetStagingFolder();
+    }
+
+    @Override
+    public String getDataSetCode()
+    {
+        return dataSet.getDataSetCode();
+    }
+
+    @Override
+    public IExperimentImmutable getExperiment()
+    {
+        return dataSet.getExperiment();
+    }
+
+    @Override
+    public ISampleImmutable getSample()
+    {
+        return dataSet.getSample();
+    }
+
+    @Override
+    public String getFileFormatType()
+    {
+        return dataSet.getFileFormatType();
+    }
+
+    @Override
+    public int getSpeedHint()
+    {
+        return dataSet.getSpeedHint();
+    }
+
+    @Override
+    public String getDataSetType()
+    {
+        return dataSet.getDataSetType();
+    }
+
+    @Override
+    public DataSetType getDataSetTypeWithPropertyTypes()
+    {
+        return dataSet.getDataSetTypeWithPropertyTypes();
+    }
+
+    @Override
+    public String getPropertyValue(String propertyCode)
+    {
+        return dataSet.getPropertyValue(propertyCode);
+    }
+
+    @Override
+    public List<String> getAllPropertyCodes()
+    {
+        return dataSet.getAllPropertyCodes();
+    }
+
+    @Override
+    public List<String> getParentDatasets()
+    {
+        return dataSet.getParentDatasets();
+    }
+
+    @Override
+    public List<String> getContainedDataSetCodes()
+    {
+        return dataSet.getContainedDataSetCodes();
+    }
+
+    @Override
+    public List<IDataSetImmutable> getChildrenDataSets()
+    {
+        return dataSet.getChildrenDataSets();
+    }
+
+    @Override
+    public String getContainerDataSet()
+    {
+        return dataSet.getContainerDataSet();
+    }
+
+    @Override
+    public IExternalDataManagementSystemImmutable getExternalDataManagementSystem()
+    {
+        return dataSet.getExternalDataManagementSystem();
+    }
+
+    @Override
+    public String getExternalCode()
+    {
+        return dataSet.getExternalCode();
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return dataSet.hashCode();
+    }
+
+    @Override
+    public void setExperiment(IExperimentImmutable experiment)
+    {
+        dataSet.setExperiment(experiment);
+    }
+
+    @Override
+    public void setSample(ISampleImmutable sampleOrNull)
+    {
+        dataSet.setSample(sampleOrNull);
+    }
+
+    @Override
+    public void setFileFormatType(String fileFormatTypeCode)
+    {
+        dataSet.setFileFormatType(fileFormatTypeCode);
+    }
+
+    @Override
+    public boolean isMeasuredData()
+    {
+        return dataSet.isMeasuredData();
+    }
+
+    @Override
+    public void setMeasuredData(boolean measuredData)
+    {
+        dataSet.setMeasuredData(measuredData);
+    }
+
+    @Override
+    public void setSpeedHint(int speedHint)
+    {
+        dataSet.setSpeedHint(speedHint);
+    }
+
+    @Override
+    public void setDataSetType(String dataSetTypeCode)
+    {
+        dataSet.setDataSetType(dataSetTypeCode);
+    }
+
+    @Override
+    public void setPropertyValue(String propertyCode, String propertyValue)
+    {
+        dataSet.setPropertyValue(propertyCode, propertyValue);
+    }
+
+    @Override
+    public void setParentDatasets(List<String> parentDatasetCodes)
+    {
+        dataSet.setParentDatasets(parentDatasetCodes);
+    }
+
+    @Override
+    public boolean isContainerDataSet()
+    {
+        return dataSet.isContainerDataSet();
+    }
+
+    @Override
+    public void setContainedDataSetCodes(List<String> containedDataSetCodes)
+    {
+        dataSet.setContainedDataSetCodes(containedDataSetCodes);
+    }
+
+    @Override
+    public boolean isContainedDataSet()
+    {
+        return dataSet.isContainedDataSet();
+    }
+
+    @Override
+    public void setExternalDataManagementSystem(
+            IExternalDataManagementSystemImmutable externalDataManagementSystem)
+    {
+        dataSet.setExternalDataManagementSystem(externalDataManagementSystem);
+    }
+
+    @Override
+    public boolean isLinkDataSet()
+    {
+        return dataSet.isLinkDataSet();
+    }
+
+    @Override
+    public void setExternalCode(String externalCode)
+    {
+        dataSet.setExternalCode(externalCode);
+    }
+
+    @Override
+    public boolean isNoFileDataSet()
+    {
+        return dataSet.isNoFileDataSet();
+    }
+
+    @Override
+    public String toString()
+    {
+        return dataSet.toString();
+    }
+
+    public File tryDataSetContents()
+    {
+        return dataSet.tryDataSetContents();
+    }
+
+}

screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v2/IFeatureVectorDataSet.java

+/*
+ * Copyright 2012 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.etl.dto.api.v2;
+
+import ch.systemsx.cisd.etlserver.registrator.api.v1.IDataSet;
+
+/**
+ * Extension of {@link IDataSet} specific for feature vector data sets.
+ *
+ * @author Franz-Josef Elmer
+ */
+public interface IFeatureVectorDataSet extends IDataSet
+{
+    /**
+     * Sets the analysis procedure used to create the feature vectors.
+     */
+    public void setAnalysisProcedure(String analysisProcedure);
+}

screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v2/SimpleFeatureVectorDataConfig.java

+/*
+ * Copyright 2012 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.etl.dto.api.v2;
+
+import java.util.Properties;
+
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.FeaturesBuilder;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.IFeaturesBuilder;
+
+/**
+ * Configuration for creating a feature vector data set.
+ *
+ * @author Franz-Josef Elmer
+ */
+public class SimpleFeatureVectorDataConfig
+{
+    private Properties properties;
+    private IFeaturesBuilder featuresBuilder;
+    
+    /**
+     * Creates an instance with undefined properties assuming that the {@link FeaturesBuilder} is
+     * used to create the feature vectors.
+     */
+    public SimpleFeatureVectorDataConfig()
+    {
+    }
+
+    /**
+     * Creates an instance for the specified properties object which is used to configure parsing of
+     * a feature vector file.
+     */
+    public SimpleFeatureVectorDataConfig(Properties properties)
+    {
+        this.properties = properties;
+    }
+    
+    public IFeaturesBuilder getFeaturesBuilder()
+    {
+        if (featuresBuilder == null)
+        {
+            featuresBuilder = new FeaturesBuilder();
+        }
+        return featuresBuilder;
+    }
+
+    public Properties getProperties()
+    {
+        return properties;
+    }
+
+}

screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/ImagingDataSetRegistrationTransaction.java

@@ -76,6 +76,8 @@ public class ImagingDataSetRegistrationTransaction extends DataSetRegistrationTr
 
     private final String originalDirName;
 
+    private final JythonPlateDatasetFactory factory;
+
     @SuppressWarnings("unchecked")
     public ImagingDataSetRegistrationTransaction(File rollBackStackParentFolder,
             File workingDirectory, File stagingDirectory,
@@ -89,12 +91,17 @@ public class ImagingDataSetRegistrationTransaction extends DataSetRegistrationTr
         assert registrationDetailsFactory instanceof JythonPlateDatasetFactory : "JythonPlateDatasetFactory expected, but got: "
                 + registrationDetailsFactory.getClass().getCanonicalName();
 
-        JythonPlateDatasetFactory factory = (JythonPlateDatasetFactory) registrationDetailsFactory;
+        factory = (JythonPlateDatasetFactory) registrationDetailsFactory;
         this.imageDatasetFactory = factory.imageDatasetFactory;
         this.imageContainerDatasetFactory = factory.imageContainerDatasetFactory;
         this.originalDirName = originalDirName;
     }
 
+    public JythonPlateDatasetFactory getFactory()
+    {
+        return factory;
+    }
+
     @Override
     public IImageDataSet createNewImageDataSet(SimpleImageDataConfig imageDataSet,
             File incomingFolderWithImages)

screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/JythonPlateDatasetFactory.java

@@ -204,7 +204,7 @@ public class JythonPlateDatasetFactory extends JythonObjectFactory<DataSetInform
         return createFeatureVectorRegistrationDetails(featureDefinitions);
     }
 
-    private DataSetRegistrationDetails<FeatureVectorDataSetInformation> createFeatureVectorRegistrationDetails(
+    public DataSetRegistrationDetails<FeatureVectorDataSetInformation> createFeatureVectorRegistrationDetails(
             List<FeatureDefinition> featureDefinitions)
     {
         DataSetRegistrationDetails<FeatureVectorDataSetInformation> registrationDetails =

screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/v2/IImagingDataSetRegistrationTransactionV2.java

@@ -19,8 +19,11 @@ package ch.systemsx.cisd.openbis.dss.etl.jython.v2;
 import java.io.File;
 
 import ch.systemsx.cisd.etlserver.registrator.api.v2.IDataSetRegistrationTransactionV2;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.FeaturesBuilder;
 import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.IImageDataSet;
 import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.SimpleImageDataConfig;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v2.IFeatureVectorDataSet;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v2.SimpleFeatureVectorDataConfig;
 
 /**
  * 
@@ -31,9 +34,16 @@ public interface IImagingDataSetRegistrationTransactionV2 extends
  IDataSetRegistrationTransactionV2
 {
     /**
-     * Creates a new image dataset. See {@link SimpleImageDataConfig} documentation for
+     * Creates a new image data set. See {@link SimpleImageDataConfig} documentation for
      * configuration details.
      */
     IImageDataSet createNewImageDataSet(SimpleImageDataConfig imageDataSet,
             File incomingFolderWithImages);
+    
+    /**
+     * Creates a new feature vector data set based on either the {@link FeaturesBuilder} provided by
+     * the specified {@link SimpleFeatureVectorDataConfig} or the specified file.
+     */
+    IFeatureVectorDataSet createNewFeatureVectorDataSet(
+            SimpleFeatureVectorDataConfig featureDataSetConfig, File featureVectorFileOrNull);
 }

screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/v2/ImagingDataSetRegistrationTransactionV2Delegate.java

@@ -17,11 +17,16 @@
 package ch.systemsx.cisd.openbis.dss.etl.jython.v2;
 
 import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Properties;
 
 import net.lemnik.eodsql.DynamicTransactionQuery;
 
+import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
 import ch.systemsx.cisd.etlserver.TopLevelDataSetRegistratorGlobalState;
 import ch.systemsx.cisd.etlserver.registrator.DataSetRegistrationContext;
+import ch.systemsx.cisd.etlserver.registrator.DataSetRegistrationDetails;
 import ch.systemsx.cisd.etlserver.registrator.api.v1.IDataSet;
 import ch.systemsx.cisd.etlserver.registrator.api.v1.IDataSetUpdatable;
 import ch.systemsx.cisd.etlserver.registrator.api.v1.IExperiment;
@@ -29,8 +34,16 @@ import ch.systemsx.cisd.etlserver.registrator.api.v1.IMaterial;
 import ch.systemsx.cisd.etlserver.registrator.api.v1.IProject;
 import ch.systemsx.cisd.etlserver.registrator.api.v1.ISample;
 import ch.systemsx.cisd.etlserver.registrator.api.v1.ISpace;
+import ch.systemsx.cisd.etlserver.registrator.api.v1.impl.DataSet;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.FeatureDefinition;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.FeatureVectorDataSet;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.FeatureVectorDataSetInformation;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.FeaturesBuilder;
 import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.IImageDataSet;
 import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.SimpleImageDataConfig;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v2.IFeatureVectorDataSet;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v2.SimpleFeatureVectorDataConfig;
+import ch.systemsx.cisd.openbis.dss.etl.featurevector.CsvFeatureVectorParser;
 import ch.systemsx.cisd.openbis.dss.etl.jython.ImagingDataSetRegistrationTransaction;
 import ch.systemsx.cisd.openbis.dss.generic.shared.api.internal.authorization.IAuthorizationService;
 import ch.systemsx.cisd.openbis.dss.generic.shared.api.internal.v1.IDataSetImmutable;
@@ -63,6 +76,36 @@ public class ImagingDataSetRegistrationTransactionV2Delegate implements
         return transaction.createNewImageDataSet(imageDataSet, incomingFolderWithImages);
     }
 
+    @Override
+    public IFeatureVectorDataSet createNewFeatureVectorDataSet(
+            SimpleFeatureVectorDataConfig featureDataSetConfig, File featureVectorFileOrNull)
+    {
+        List<FeatureDefinition> featureDefinitions;
+        Properties properties = featureDataSetConfig.getProperties();
+        if (properties == null)
+        {
+            featureDefinitions =
+                    ((FeaturesBuilder) featureDataSetConfig.getFeaturesBuilder())
+                            .getFeatureDefinitionValuesList();
+        } else
+        {
+            try
+            {
+                featureDefinitions = CsvFeatureVectorParser.parse(featureVectorFileOrNull, properties);
+            } catch (IOException ex)
+            {
+                throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+            }
+        }
+        DataSetRegistrationDetails<FeatureVectorDataSetInformation> registrationDetails =
+                transaction.getFactory().createFeatureVectorRegistrationDetails(featureDefinitions);
+        @SuppressWarnings("unchecked")
+        DataSet<FeatureVectorDataSetInformation> dataSet =
+                (DataSet<FeatureVectorDataSetInformation>) transaction
+                        .createNewDataSet(registrationDetails);
+        return new FeatureVectorDataSet(dataSet);
+    }
+
     @Override
     public IDataSet createNewDataSet()
     {

screening/sourceTest/core-plugins/AggregatedFeatureVectorsTest/1/dss/drop-boxes/AggregatedFeatureVectorsTest-drop-box/data-set-handler.py

+from ch.systemsx.cisd.openbis.dss.etl.dto.api.v2 import SimpleFeatureVectorDataConfig 
+
 PLATE_GEOMETRY_PROPERTY_CODE = "$PLATE_GEOMETRY"
 PLATE_GEOMETRY = "384_WELLS_16X24"
 
@@ -32,16 +34,16 @@ def create_plate(tr, experiment, plateCode, gene):
     return plate
 
 
-def create_analysis_data_set(tr, plate, builder, analysis_procedure, ds_file):    
-    analysis_registration_details = factory.createFeatureVectorDatasetDetails(builder)    
-    analysis_data_set = tr.createNewDataSet(analysis_registration_details)
+def create_analysis_data_set(tr, plate, config, analysis_procedure, ds_file):    
+    analysis_data_set = tr.createNewFeatureVectorDataSet(config, None)
     analysis_data_set.setSample(plate)
     analysis_data_set.setPropertyValue("$ANALYSIS_PROCEDURE", analysis_procedure)
-    analysis_data_set_file = tr.moveFile(incoming.getPath() + "/" + ds_file, analysis_data_set)
+    analysis_data_set_file = tr.moveFile(tr.incoming.getPath() + "/" + ds_file, analysis_data_set)
 
 def create_dataset_with_features1(tr, experiment, gene):
     plate1 = create_plate(tr, experiment, "PLATE1", gene)
-    builder = factory.createFeaturesBuilder()
+    config = SimpleFeatureVectorDataConfig()
+    builder = config.featuresBuilder
     
     featureX = builder.defineFeature("X")
     featureX.addValue(1, 1, "1")
@@ -51,11 +53,12 @@ def create_dataset_with_features1(tr, experiment, gene):
     featureY.addValue(1, 1, "3")
     featureY.addValue(1, 2, "2")
 
-    create_analysis_data_set(tr, plate1, builder, "p1", "data-set-1.csv")    
+    create_analysis_data_set(tr, plate1, config, "p1", "data-set-1.csv")    
 
 def create_dataset_with_features2(tr, experiment, gene):
     plate2 = create_plate(tr, experiment, "PLATE2", gene)
-    builder = factory.createFeaturesBuilder()
+    config = SimpleFeatureVectorDataConfig()
+    builder = config.featuresBuilder
     
     featureA = builder.defineFeature("A")
     featureA.addValue(1, 1, "10")
@@ -69,18 +72,19 @@ def create_dataset_with_features2(tr, experiment, gene):
     featureX.addValue(1, 1, "5")
     featureX.addValue(1, 2, "6")
 
-    create_analysis_data_set(tr, plate2, builder, "p2", "data-set-2.file")    
+    create_analysis_data_set(tr, plate2, config, "p2", "data-set-2.file")    
 
 
-tr = service.transaction()
-experiment = create_experiment(tr)
+def process(transaction): 
+    incoming = transaction.incoming
+    experiment = create_experiment(transaction)
 
-geneCode = "G"
-geneG = tr.createNewMaterial(geneCode, "GENE") 
+    geneCode = "G"
+    geneG = transaction.createNewMaterial(geneCode, "GENE") 
 
-create_dataset_with_features1(tr, experiment, geneG)
-create_dataset_with_features2(tr, experiment, geneG)
+    create_dataset_with_features1(transaction, experiment, geneG)
+    create_dataset_with_features2(transaction, experiment, geneG)
 
-# delete the empty incoming, its 
-# contents have been moved to the data sets 
-incoming.delete()
+    # delete the empty incoming, its 
+    # contents have been moved to the data sets 
+    incoming.delete()
\ No newline at end of file

screening/sourceTest/core-plugins/AggregatedFeatureVectorsTest/1/dss/drop-boxes/AggregatedFeatureVectorsTest-drop-box/plugin.properties

 incoming-dir = ${root-dir}/incoming-AggregatedFeatureVectorsTest
 incoming-dir-create = true
 incoming-data-completeness-condition = auto-detection
-top-level-data-set-handler = ch.systemsx.cisd.openbis.dss.etl.jython.JythonPlateDataSetHandler
+top-level-data-set-handler = ch.systemsx.cisd.openbis.dss.etl.jython.v2.JythonPlateDataSetHandlerV2
 script-path = data-set-handler.py
 storage-processor = ch.systemsx.cisd.openbis.dss.etl.featurevector.FeatureVectorStorageProcessor
 storage-processor.processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor