From 8349862b8532f493f50949c1916f0f83a898323d Mon Sep 17 00:00:00 2001
From: tpylak <tpylak>
Date: Tue, 1 Mar 2011 09:57:58 +0000
Subject: [PATCH] LMS-2081 a framework to register feature vectors from the
jython dropbox
SVN: 20167
---
.../DataSetRegistrationService.java | 2 +-
.../dto/api/impl/FeatureDefinitionValues.java | 246 ++++++++++++++++++
.../impl/FeatureVectorDataSetInformation.java | 56 ++++
.../dss/etl/dto/api/impl/FeaturesBuilder.java | 64 +++++
.../dss/etl/dto/api/v1/FeatureDefinition.java | 120 +++++++++
.../dss/etl/dto/api/v1/IFeatureValues.java | 48 ++++
.../dss/etl/dto/api/v1/IFeaturesBuilder.java | 29 +++
.../CsvToCanonicalFeatureVector.java | 181 ++-----------
.../etl/featurevector/FeatureValuesMap.java | 152 +++++++++++
.../FeatureVectorStorageProcessor.java | 58 ++++-
.../featurevector/FeatureVectorUploader.java | 14 +-
.../etl/jython/JythonPlateDataSetHandler.java | 121 +++++----
.../jython/SimpleImageDataSetRegistrator.java | 12 +-
.../registrator/api/v1/impl/ImageDataSet.java | 3 +-
.../dataaccess/ImgFeatureValuesDTO.java | 2 +-
.../api/impl/FeatureDefinitionValuesTest.java | 112 ++++++++
16 files changed, 997 insertions(+), 223 deletions(-)
create mode 100644 screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/FeatureDefinitionValues.java
create mode 100644 screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/FeatureVectorDataSetInformation.java
create mode 100644 screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/FeaturesBuilder.java
create mode 100644 screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/FeatureDefinition.java
create mode 100644 screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/IFeatureValues.java
create mode 100644 screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/IFeaturesBuilder.java
create mode 100644 screening/source/java/ch/systemsx/cisd/openbis/dss/etl/featurevector/FeatureValuesMap.java
create mode 100644 screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/FeatureDefinitionValuesTest.java
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetRegistrationService.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetRegistrationService.java
index 3655d081169..9d75cd6791b 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetRegistrationService.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetRegistrationService.java
@@ -159,7 +159,7 @@ public class DataSetRegistrationService<T extends DataSetInformation> implements
/**
* Create a new transaction that atomically performs file operations and registers entities.
*/
- public IDataSetRegistrationTransaction transaction(File dataSetFile,
+ public DataSetRegistrationTransaction<T> transaction(File dataSetFile,
IDataSetRegistrationDetailsFactory<T> detailsFactory)
{
File workingDirectory = dataSetFile.getParentFile();
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/FeatureDefinitionValues.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/FeatureDefinitionValues.java
new file mode 100644
index 00000000000..ff39220551b
--- /dev/null
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/FeatureDefinitionValues.java
@@ -0,0 +1,246 @@
+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.etl.dto.api.impl;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.IFeatureValues;
+import ch.systemsx.cisd.openbis.dss.etl.featurevector.CanonicalFeatureVector;
+import ch.systemsx.cisd.openbis.dss.etl.featurevector.FeatureValuesMap;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.Geometry;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.basic.dto.WellLocation;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.dto.PlateFeatureValues;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgFeatureDefDTO;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgFeatureValuesDTO;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgFeatureVocabularyTermDTO;
+
+/**
+ * Values of one feature for all wells + the way to build this structure.
+ *
+ * @author Tomasz Pylak
+ */
+public class FeatureDefinitionValues implements IFeatureValues
+{
+ private final ImgFeatureDefDTO imgFeatureDefDTO;
+
+ private final List<FeatureValuesMap> values;
+
+ private FeatureValuesMap currentFeatureVector;
+
+ public FeatureDefinitionValues(ImgFeatureDefDTO imgFeatureDefDTO)
+ {
+ this(imgFeatureDefDTO, new FeatureValuesMap(null, null));
+ }
+
+ public FeatureDefinitionValues(ImgFeatureDefDTO imgFeatureDefDTO,
+ FeatureValuesMap currentFeatureVector)
+ {
+ assert imgFeatureDefDTO != null : "featureDefinition is null";
+ this.imgFeatureDefDTO = imgFeatureDefDTO;
+ this.values = new ArrayList<FeatureValuesMap>();
+ this.currentFeatureVector = currentFeatureVector;
+ }
+
+ public void setSeries(Double timeOrNull, Double depthOrNull)
+ {
+ flushCurrent();
+ currentFeatureVector = new FeatureValuesMap(timeOrNull, depthOrNull);
+ }
+
+ private void flushCurrent()
+ {
+ if (currentFeatureVector != null && currentFeatureVector.isEmpty() == false)
+ {
+ values.add(currentFeatureVector);
+ currentFeatureVector = null;
+ }
+ }
+
+ /**
+ * @param well code of the well, e.g. A1
+ * @param value value of the feature in the specified well
+ */
+ public void addValue(String well, String value)
+ {
+ WellLocation wellPos = WellLocation.parseLocationStr(well);
+ addValueToCurrent(value, wellPos);
+ }
+
+ /**
+ * @param wellRow row coordinate of the well, top-left well has (1,1) coordinates.
+ * @param wellColumn column coordinate of the well, top-left well has (1,1) coordinates.
+ * @param value value of the feature in the specified well
+ */
+ public void addValue(int wellRow, int wellColumn, String value)
+ {
+ WellLocation wellPos = new WellLocation(wellRow, wellColumn);
+ addValueToCurrent(value, wellPos);
+ }
+
+ private void addValueToCurrent(String value, WellLocation wellPos)
+ {
+ currentFeatureVector.addValue(value, wellPos);
+ }
+
+ private void validate(Geometry plateGeometry)
+ {
+ for (FeatureValuesMap valuesMap : values)
+ {
+ valuesMap.validate(plateGeometry);
+ }
+ }
+
+ // ----- converter
+
+ /** @return feature vector in a canonical form with all the values added so far. */
+ public CanonicalFeatureVector getCanonicalFeatureVector(Geometry plateGeometry)
+ {
+ flushCurrent();
+ validate(plateGeometry);
+
+ CanonicalFeatureVector canonicalFeatureVector = new CanonicalFeatureVector();
+ canonicalFeatureVector.setFeatureDef(imgFeatureDefDTO);
+
+ Set<String> uniqueValues = getUniqueAvailableValues();
+ Map<String, Integer/* value sequence number */> termToSequanceMap =
+ fixVocabularyTermSequences(uniqueValues);
+
+ List<ImgFeatureVocabularyTermDTO> vocabularyTerms = null;
+ List<Map<WellLocation, Float>> floatValuesList = tryCreateFloatValueList();
+ if (floatValuesList == null)
+ {
+ floatValuesList = calculateWellTermsMappingList(termToSequanceMap);
+ vocabularyTerms = tryCreateVocabularyTerms(termToSequanceMap);
+ }
+ List<ImgFeatureValuesDTO> featureDTOs = createValueDTOs(plateGeometry, floatValuesList);
+ canonicalFeatureVector.setValues(featureDTOs);
+ canonicalFeatureVector.setVocabularyTerms(vocabularyTerms);
+ return canonicalFeatureVector;
+ }
+
+ private List<ImgFeatureValuesDTO> createValueDTOs(Geometry plateGeometry,
+ List<Map<WellLocation, Float>> floatValuesList)
+ {
+ List<ImgFeatureValuesDTO> featureDTOs = new ArrayList<ImgFeatureValuesDTO>();
+ for (int i = 0; i < values.size(); i++)
+ {
+ FeatureValuesMap featureValuesMap = values.get(i);
+ Map<WellLocation, Float> floatValues = floatValuesList.get(i);
+ ImgFeatureValuesDTO featureValuesDTO =
+ createFeatureValuesDTO(plateGeometry, featureValuesMap, floatValues);
+ featureDTOs.add(featureValuesDTO);
+ }
+ return featureDTOs;
+ }
+
+ private static ImgFeatureValuesDTO createFeatureValuesDTO(Geometry plateGeometry,
+ FeatureValuesMap featureValuesMap, Map<WellLocation, Float> floatValues)
+ {
+ final PlateFeatureValues valuesValues =
+ convertColumnToByteArray(plateGeometry, floatValues);
+ ImgFeatureValuesDTO featureValuesDTO =
+ new ImgFeatureValuesDTO(featureValuesMap.tryGetTime(),
+ featureValuesMap.tryGetDepth(), valuesValues, 0);
+ return featureValuesDTO;
+ }
+
+ private static PlateFeatureValues convertColumnToByteArray(Geometry geometry,
+ Map<WellLocation, Float> values)
+ {
+ final PlateFeatureValues featureValues = new PlateFeatureValues(geometry);
+ for (WellLocation loc : values.keySet())
+ {
+ final Float value = values.get(loc);
+ featureValues.setForWellLocation(value, loc);
+ }
+ return featureValues;
+ }
+
+ private List<Map<WellLocation, Float>> calculateWellTermsMappingList(
+ Map<String, Integer> termToSequanceMap)
+ {
+ List<Map<WellLocation, Float>> list = new ArrayList<Map<WellLocation, Float>>();
+ for (FeatureValuesMap featureValuesMap : values)
+ {
+ Map<WellLocation, Float> floatValues =
+ featureValuesMap.calculateWellTermsMapping(termToSequanceMap);
+ list.add(floatValues);
+ }
+ return list;
+ }
+
+ private List<Map<WellLocation, Float>> tryCreateFloatValueList()
+ {
+ List<Map<WellLocation, Float>> list = new ArrayList<Map<WellLocation, Float>>();
+ for (FeatureValuesMap featureValuesMap : values)
+ {
+ Map<WellLocation, Float> floatValues = featureValuesMap.tryExtractFloatValues();
+ if (floatValues == null)
+ {
+ return null;
+ }
+ list.add(floatValues);
+ }
+ return list;
+ }
+
+ private static List<ImgFeatureVocabularyTermDTO> tryCreateVocabularyTerms(
+ Map<String, Integer> valueToSequanceMap)
+ {
+ if (valueToSequanceMap.isEmpty())
+ {
+ return null;
+ }
+ List<ImgFeatureVocabularyTermDTO> vocabularyTerms =
+ new ArrayList<ImgFeatureVocabularyTermDTO>();
+ for (Entry<String, Integer> entry : valueToSequanceMap.entrySet())
+ {
+ vocabularyTerms.add(new ImgFeatureVocabularyTermDTO(entry.getKey(), entry.getValue()));
+ }
+ return vocabularyTerms;
+ }
+
+ private static Map<String, Integer/* value sequence number */> fixVocabularyTermSequences(
+ Set<String> uniqueValues)
+ {
+ Map<String, Integer> valueToSequanceMap = new HashMap<String, Integer>();
+ int sequenceNumber = 0;
+ for (String value : uniqueValues)
+ {
+ valueToSequanceMap.put(value, sequenceNumber++);
+ }
+ return valueToSequanceMap;
+ }
+
+ private Set<String> getUniqueAvailableValues()
+ {
+ List<FeatureValuesMap> valuesMaps = values;
+ Set<String> uniqueValues = new HashSet<String>();
+ for (FeatureValuesMap valuesMap : valuesMaps)
+ {
+ uniqueValues.addAll(valuesMap.getUniqueAvailableValues());
+ }
+ return uniqueValues;
+ }
+
+}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/FeatureVectorDataSetInformation.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/FeatureVectorDataSetInformation.java
new file mode 100644
index 00000000000..01a79ed6ce9
--- /dev/null
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/FeatureVectorDataSetInformation.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.etl.dto.api.impl;
+
+import java.util.List;
+
+import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
+import ch.systemsx.cisd.openbis.generic.shared.IServer;
+
+/**
+ * Extends {@link DataSetInformation} with information about images analysis on the well level
+ * (relevant for HCS).
+ *
+ * @author Tomasz Pylak
+ */
+public class FeatureVectorDataSetInformation extends DataSetInformation
+{
+ private static final long serialVersionUID = IServer.VERSION;
+
+ private List<FeatureDefinitionValues> features;
+
+ public FeatureVectorDataSetInformation()
+ {
+ }
+
+ public List<FeatureDefinitionValues> getFeatures()
+ {
+ return features;
+ }
+
+ public void setFeatures(List<FeatureDefinitionValues> features)
+ {
+ this.features = features;
+ }
+
+ /** are all necessary fields filled? */
+ public boolean isValid()
+ {
+ return features != null && features.size() > 0;
+ }
+
+}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/FeaturesBuilder.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/FeaturesBuilder.java
new file mode 100644
index 00000000000..7aecfaa1401
--- /dev/null
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/FeaturesBuilder.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.etl.dto.api.impl;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.FeatureDefinition;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.IFeatureValues;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.IFeaturesBuilder;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgFeatureDefDTO;
+
+/**
+ * Allows to define feature vectors of one image analysis dataset.
+ *
+ * @author Tomasz Pylak
+ */
+public class FeaturesBuilder implements IFeaturesBuilder
+{
+ private final List<FeatureDefinitionValues> featureDefinitionValuesList;
+
+ public FeaturesBuilder()
+ {
+ this.featureDefinitionValuesList = new ArrayList<FeatureDefinitionValues>();
+ }
+
+ /** Defines a container to which values of the feature for each well can be added. */
+ public IFeatureValues defineFeature(FeatureDefinition featureDefinition)
+ {
+ featureDefinition.ensureValid();
+ FeatureDefinitionValues featureDefinitionValues =
+ new FeatureDefinitionValues(convert(featureDefinition));
+ featureDefinitionValuesList.add(featureDefinitionValues);
+ return featureDefinitionValues;
+ }
+
+ private static ImgFeatureDefDTO convert(FeatureDefinition featureDefinition)
+ {
+ ImgFeatureDefDTO dto = new ImgFeatureDefDTO();
+ dto.setCode(featureDefinition.getCode());
+ dto.setLabel(featureDefinition.getLabel());
+ dto.setDescription(featureDefinition.getDescription());
+ return dto;
+ }
+
+ public List<FeatureDefinitionValues> getFeatureDefinitionValuesList()
+ {
+ return featureDefinitionValuesList;
+ }
+}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/FeatureDefinition.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/FeatureDefinition.java
new file mode 100644
index 00000000000..9ca81d7402e
--- /dev/null
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/FeatureDefinition.java
@@ -0,0 +1,120 @@
+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.etl.dto.api.v1;
+
+import org.apache.commons.lang.StringUtils;
+
+/**
+ * Definition of the feature.
+ *
+ * @author Tomasz Pylak
+ */
+public class FeatureDefinition
+{
+ private String code;
+
+ private String label;
+
+ private String description;
+
+ public FeatureDefinition()
+ {
+ }
+
+ /** Creates a feature definition, the label will be equal to the code. */
+ public FeatureDefinition(String code)
+ {
+ this.code = code;
+ this.label = code;
+ }
+
+ public String getCode()
+ {
+ return code;
+ }
+
+ /** Sets the code of a feature. */
+ public void setCode(String code)
+ {
+ this.code = code;
+ }
+
+ public String getLabel()
+ {
+ return label;
+ }
+
+ /** Sets the label of a feature. */
+ public void setLabel(String label)
+ {
+ this.label = label;
+ }
+
+ public String getDescription()
+ {
+ return description;
+ }
+
+ /** Sets description of a feature. */
+ public void setDescription(String description)
+ {
+ this.description = description;
+ }
+
+ /**
+ * Validates that tile number, well and channel have been specified.
+ *
+ * @throws IllegalStateException if the object is not valid.
+ */
+ public void ensureValid()
+ {
+ if (StringUtils.isBlank(code))
+ {
+ throw new IllegalStateException("Code is not specified");
+ }
+ if (StringUtils.isBlank(label))
+ {
+ throw new IllegalStateException("Label is not specified");
+ }
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return code.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj)
+ {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ FeatureDefinition other = (FeatureDefinition) obj;
+ if (code == null)
+ {
+ return other.code == null;
+ } else
+ {
+ return code.equals(other.code);
+ }
+ }
+
+}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/IFeatureValues.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/IFeatureValues.java
new file mode 100644
index 00000000000..113829fd518
--- /dev/null
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/IFeatureValues.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.etl.dto.api.v1;
+
+/**
+ * An interface which allows to define values of one feature.
+ *
+ * @author Tomasz Pylak
+ */
+public interface IFeatureValues
+{
+ /**
+ * Relevant only in cases where feature values for different timepoints and/or depth-scans are
+ * available. In other cases one does not have to call this method at all.
+ * <p>
+ * Sets the timepoint and/or the depth-scan values which will be used in all subsequent calls to
+ * {@link #addValue} until this method will be called again.
+ * </p>
+ */
+ void setSeries(Double timeOrNull, Double depthOrNull);
+
+ /**
+ * @param well code of the well, e.g. A1
+ * @param value value of the feature in the specified well
+ */
+ void addValue(String well, String value);
+
+ /**
+ * @param wellRow row coordinate of the well, top-left well has (1,1) coordinates.
+ * @param wellColumn column coordinate of the well, top-left well has (1,1) coordinates.
+ * @param value value of the feature in the specified well
+ */
+ void addValue(int wellRow, int wellColumn, String value);
+}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/IFeaturesBuilder.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/IFeaturesBuilder.java
new file mode 100644
index 00000000000..c5df9d5a4ce
--- /dev/null
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/IFeaturesBuilder.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.etl.dto.api.v1;
+
+/**
+ * Allows to define feature vectors of one image analysis dataset.
+ *
+ * @author Tomasz Pylak
+ */
+public interface IFeaturesBuilder
+{
+ /** Defines a container to which values of the feature for each well can be added. */
+ public IFeatureValues defineFeature(FeatureDefinition featureDefinition);
+
+}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/featurevector/CsvToCanonicalFeatureVector.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/featurevector/CsvToCanonicalFeatureVector.java
index 884288ba7b2..94fdc31e415 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/featurevector/CsvToCanonicalFeatureVector.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/featurevector/CsvToCanonicalFeatureVector.java
@@ -18,24 +18,18 @@ package ch.systemsx.cisd.openbis.dss.etl.featurevector;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
import java.util.Set;
import ch.systemsx.cisd.common.shared.basic.utils.StringUtils;
import ch.systemsx.cisd.common.utilities.Counters;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.FeatureDefinitionValues;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.DatasetFileLines;
import ch.systemsx.cisd.openbis.dss.generic.shared.utils.CodeAndLabelUtil;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.CodeAndLabel;
import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.Geometry;
import ch.systemsx.cisd.openbis.plugin.screening.shared.basic.dto.WellLocation;
-import ch.systemsx.cisd.openbis.plugin.screening.shared.dto.PlateFeatureValues;
import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgFeatureDefDTO;
-import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgFeatureValuesDTO;
-import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgFeatureVocabularyTermDTO;
/**
* Converts feature vectors from CSV files to CanonicaFeatureVector objects.
@@ -53,8 +47,9 @@ public class CsvToCanonicalFeatureVector
private final boolean isSplit;
private final Set<String> columnsToBeIgnored;
-
- public CsvToCanonicalFeatureVectorConfiguration(FeatureVectorStorageProcessorConfiguration config)
+
+ public CsvToCanonicalFeatureVectorConfiguration(
+ FeatureVectorStorageProcessorConfiguration config)
{
this(config.getWellRow(), config.getWellColumn(), config.getColumnsToBeIgnored());
}
@@ -63,8 +58,9 @@ public class CsvToCanonicalFeatureVector
{
this(wellRow, wellColumn, Collections.<String> emptySet());
}
-
- public CsvToCanonicalFeatureVectorConfiguration(String wellRow, String wellColumn, Set<String> columnsToBeIgnored)
+
+ public CsvToCanonicalFeatureVectorConfiguration(String wellRow, String wellColumn,
+ Set<String> columnsToBeIgnored)
{
this.wellRowColumn = wellRow;
this.wellColumnColumn = wellColumn;
@@ -87,7 +83,7 @@ public class CsvToCanonicalFeatureVector
{
return isSplit;
}
-
+
public boolean shouldColumnBeIgnored(String column)
{
return columnsToBeIgnored.contains(column);
@@ -115,6 +111,12 @@ public class CsvToCanonicalFeatureVector
private final int maxPlateGeometryCol;
+ public CsvToCanonicalFeatureVector(DatasetFileLines fileLines,
+ CsvToCanonicalFeatureVectorConfiguration config, Geometry plateGeometry)
+ {
+ this(fileLines, config, plateGeometry.getNumberOfRows(), plateGeometry.getNumberOfColumns());
+ }
+
public CsvToCanonicalFeatureVector(DatasetFileLines fileLines,
CsvToCanonicalFeatureVectorConfiguration config, int maxRow, int maxCol)
{
@@ -157,7 +159,6 @@ public class CsvToCanonicalFeatureVector
private CanonicalFeatureVector convertColumnToFeatureVector(Geometry geometry,
FeatureColumn column, Counters<String> counters)
{
- CanonicalFeatureVector featureVector = new CanonicalFeatureVector();
CodeAndLabel codeAndTitle = CodeAndLabelUtil.create(column.name);
ImgFeatureDefDTO featureDef = new ImgFeatureDefDTO();
featureDef.setLabel(codeAndTitle.getLabel());
@@ -165,54 +166,8 @@ public class CsvToCanonicalFeatureVector
String code = codeAndTitle.getCode();
int count = counters.count(code);
featureDef.setCode(count == 1 ? code : code + count);
- featureVector.setFeatureDef(featureDef);
-
- return setFeatureValues(featureVector, column, geometry);
- }
-
- private CanonicalFeatureVector setFeatureValues(CanonicalFeatureVector featureVector,
- FeatureColumn column, Geometry geometry)
- {
- Map<WellLocation, Float> floatValues = column.tryExtractFloatValues();
- List<ImgFeatureVocabularyTermDTO> vocabularyTerms =
- new ArrayList<ImgFeatureVocabularyTermDTO>();
- if (floatValues == null)
- {
- VocabularyFeatureColumnValues vocabularyValues = column.extractVocabularyValues();
-
- floatValues = vocabularyValues.getWellTermsMapping();
- vocabularyTerms = createVocabularyTerms(vocabularyValues.getValueToSequanceMap());
- }
- final PlateFeatureValues valuesValues = convertColumnToByteArray(geometry, floatValues);
- ImgFeatureValuesDTO values = new ImgFeatureValuesDTO(0., 0., valuesValues, 0);
-
- featureVector.setValues(Collections.singletonList(values));
- featureVector.setVocabularyTerms(vocabularyTerms);
- return featureVector;
- }
- private List<ImgFeatureVocabularyTermDTO> createVocabularyTerms(
- Map<String, Integer> valueToSequanceMap)
- {
- List<ImgFeatureVocabularyTermDTO> vocabularyTerms =
- new ArrayList<ImgFeatureVocabularyTermDTO>();
- for (Entry<String, Integer> entry : valueToSequanceMap.entrySet())
- {
- vocabularyTerms.add(new ImgFeatureVocabularyTermDTO(entry.getKey(), entry.getValue()));
- }
- return vocabularyTerms;
- }
-
- private PlateFeatureValues convertColumnToByteArray(Geometry geometry,
- Map<WellLocation, Float> values)
- {
- final PlateFeatureValues featureValues = new PlateFeatureValues(geometry);
- for (WellLocation loc : values.keySet())
- {
- final Float value = values.get(loc);
- featureValues.setForWellLocation(value, loc);
- }
- return featureValues;
+ return column.createCanonicalFeatureVector(featureDef, geometry);
}
private void readLines()
@@ -242,7 +197,7 @@ public class CsvToCanonicalFeatureVector
String columnValue = line[column.index];
if (StringUtils.isBlank(columnValue) == false)
{
- column.values.put(well, columnValue);
+ column.addValue(well, columnValue);
}
}
@@ -309,115 +264,33 @@ public class CsvToCanonicalFeatureVector
private final boolean isWellName;
- private final HashMap<WellLocation, String> values;
+ private final FeatureValuesMap values;
private FeatureColumn(int index, String name, boolean isWellName)
{
this.index = index;
this.name = name;
this.isWellName = isWellName;
- values = new HashMap<WellLocation, String>();
+ values = new FeatureValuesMap(0., 0.);
}
- public boolean isEmpty()
+ public void addValue(WellLocation well, String columnValue)
{
- return values.isEmpty();
+ values.addValue(columnValue, well);
}
- /**
- * Tries to parse all values as float numbers.
- *
- * @return null if any column value cannot be parsed as float number.
- */
- public Map<WellLocation, Float> tryExtractFloatValues()
+ public CanonicalFeatureVector createCanonicalFeatureVector(ImgFeatureDefDTO featureDef,
+ Geometry geometry)
{
- Map<WellLocation, Float> map = new HashMap<WellLocation, Float>();
- for (Entry<WellLocation, String> entry : values.entrySet())
- {
- try
- {
- WellLocation wellLocation = entry.getKey();
- String value = entry.getValue();
- float floatValue = Float.parseFloat(value);
- map.put(wellLocation, floatValue);
- } catch (NumberFormatException ex)
- {
- return null;
- }
- }
- return map;
- }
+ FeatureDefinitionValues featureDefinitionValues =
+ new FeatureDefinitionValues(featureDef, values);
- /**
- * Assuming that all values come from the set fixed of vocabulary terms calculates the
- * mapping from vocabulary term into a unique term sequence number.<br>
- * Should be called when {@link #tryExtractFloatValues} returns null.
- */
- public VocabularyFeatureColumnValues extractVocabularyValues()
- {
- return VocabularyFeatureColumnValues.create(values);
+ return featureDefinitionValues.getCanonicalFeatureVector(geometry);
}
- }
-
- private static final class VocabularyFeatureColumnValues
- {
- private final Map<String, Integer/* value sequence number */> valueToSequanceMap;
-
- private final Map<WellLocation, Float> wellTermsMapping;
-
- public static VocabularyFeatureColumnValues create(Map<WellLocation, String> values)
- {
- Map<String, Integer> valueToSequanceMap = fixVocabularyTermSequences(values);
-
- Map<WellLocation, Float> wellTermsMapping = new HashMap<WellLocation, Float>();
- for (Entry<WellLocation, String> entry : values.entrySet())
- {
- WellLocation wellLocation = entry.getKey();
- String value = entry.getValue();
- int sequenceNumber = valueToSequanceMap.get(value);
- wellTermsMapping.put(wellLocation, (float) sequenceNumber);
- }
-
- return new VocabularyFeatureColumnValues(valueToSequanceMap, wellTermsMapping);
- }
-
- private static Map<String, Integer/* value sequence number */> fixVocabularyTermSequences(
- Map<WellLocation, String> values)
- {
- Set<String> uniqueValues = new HashSet<String>();
- for (String value : values.values())
- {
- uniqueValues.add(value);
- }
-
- Map<String, Integer> valueToSequanceMap = new HashMap<String, Integer>();
- int sequenceNumber = 0;
- for (String value : uniqueValues)
- {
- valueToSequanceMap.put(value, sequenceNumber++);
- }
- return valueToSequanceMap;
- }
-
- private VocabularyFeatureColumnValues(Map<String, Integer> valueToSequanceMap,
- Map<WellLocation, Float> wellTermsMapping)
- {
- this.valueToSequanceMap = valueToSequanceMap;
- this.wellTermsMapping = wellTermsMapping;
- }
-
- /** mapping from term code to sequence number */
- public Map<String, Integer> getValueToSequanceMap()
- {
- return valueToSequanceMap;
- }
-
- /** mapping between wells and integer sequence numbers of terms casted to floats */
- public Map<WellLocation, Float> getWellTermsMapping()
+ public boolean isEmpty()
{
- return wellTermsMapping;
+ return values.isEmpty();
}
-
}
}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/featurevector/FeatureValuesMap.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/featurevector/FeatureValuesMap.java
new file mode 100644
index 00000000000..f0ecb2d9031
--- /dev/null
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/featurevector/FeatureValuesMap.java
@@ -0,0 +1,152 @@
+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.etl.featurevector;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.Geometry;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.basic.dto.WellLocation;
+
+/**
+ * Stores values of one feature for all wells (and optionally one chosen timepoint and/or
+ * depth-scan).
+ *
+ * @author Tomasz Pylak
+ */
+public class FeatureValuesMap
+{
+ private static final long serialVersionUID = 1L;
+
+ private final Map<WellLocation, String> valuesMap;
+
+ private final Double depthOrNull;
+
+ private final Double timeOrNull;
+
+ public FeatureValuesMap(Double timeOrNull, Double depthOrNull)
+ {
+ this.depthOrNull = depthOrNull;
+ this.timeOrNull = timeOrNull;
+ this.valuesMap = new HashMap<WellLocation, String>();
+ }
+
+ public Double tryGetDepth()
+ {
+ return depthOrNull;
+ }
+
+ public Double tryGetTime()
+ {
+ return timeOrNull;
+ }
+
+ /** Add a value for one well. */
+ public void addValue(String value, WellLocation wellPos)
+ {
+ if (valuesMap.get(wellPos) != null)
+ {
+ throw new IllegalStateException("Value for the well " + wellPos
+ + " has been already defined!");
+ }
+ valuesMap.put(wellPos, value);
+ }
+
+ /** @return set of unique values of this feature (makes sense only for vocabulary terms). */
+ public Set<String> getUniqueAvailableValues()
+ {
+ Set<String> uniqueValues = new HashSet<String>();
+ for (String value : valuesMap.values())
+ {
+ uniqueValues.add(value);
+ }
+ return uniqueValues;
+ }
+
+ /**
+ * Tries to parse all values as float numbers.
+ *
+ * @return null if any column value cannot be parsed as float number.
+ */
+ public Map<WellLocation, Float> tryExtractFloatValues()
+ {
+ Map<WellLocation, Float> map = new HashMap<WellLocation, Float>();
+ for (Entry<WellLocation, String> entry : valuesMap.entrySet())
+ {
+ try
+ {
+ WellLocation wellLocation = entry.getKey();
+ String value = entry.getValue();
+ float floatValue = Float.parseFloat(value);
+ map.put(wellLocation, floatValue);
+ } catch (NumberFormatException ex)
+ {
+ return null;
+ }
+ }
+ return map;
+ }
+
+ /**
+ * Assuming that all values come from the set fixed of vocabulary terms calculates the mapping
+ * from vocabulary term into a unique term sequence number.<br>
+ * Should be called when {@link #tryExtractFloatValues} returns null.
+ *
+ * @return mapping between wells and integer sequence numbers of terms casted to floats
+ */
+ public Map<WellLocation, Float> calculateWellTermsMapping(
+ Map<String, Integer> valueToSequanceMap)
+ {
+ Map<WellLocation, Float> wellTermsMapping = new HashMap<WellLocation, Float>();
+ for (Entry<WellLocation, String> entry : valuesMap.entrySet())
+ {
+ WellLocation wellLocation = entry.getKey();
+ String value = entry.getValue();
+ int sequenceNumber = valueToSequanceMap.get(value);
+ wellTermsMapping.put(wellLocation, (float) sequenceNumber);
+ }
+ return wellTermsMapping;
+ }
+
+ public boolean isEmpty()
+ {
+ return valuesMap.isEmpty();
+ }
+
+ public void validate(Geometry plateGeometry)
+ {
+ for (WellLocation well : valuesMap.keySet())
+ {
+ validate(well, plateGeometry);
+ }
+ }
+
+ private static void validate(WellLocation wellPos, Geometry plateGeometry)
+ {
+ if (wellPos.getRow() > plateGeometry.getNumberOfRows()
+ || wellPos.getColumn() > plateGeometry.getNumberOfColumns())
+ {
+ throw new IllegalStateException(
+ String.format(
+ "Feature value if defined for the well '%s' which is outside of the plate matrix '%s'",
+ wellPos, plateGeometry));
+ }
+ }
+}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/featurevector/FeatureVectorStorageProcessor.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/featurevector/FeatureVectorStorageProcessor.java
index a3d109a0bf6..ad920d80ebe 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/featurevector/FeatureVectorStorageProcessor.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/featurevector/FeatureVectorStorageProcessor.java
@@ -18,6 +18,7 @@ package ch.systemsx.cisd.openbis.dss.etl.featurevector;
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
@@ -33,6 +34,8 @@ import ch.systemsx.cisd.etlserver.DispatcherStorageProcessor.IDispatchableStorag
import ch.systemsx.cisd.etlserver.ITypeExtractor;
import ch.systemsx.cisd.openbis.dss.etl.HCSContainerDatasetInfo;
import ch.systemsx.cisd.openbis.dss.etl.dataaccess.IImagingQueryDAO;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.FeatureDefinitionValues;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.FeatureVectorDataSetInformation;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageDataSetInformation;
import ch.systemsx.cisd.openbis.dss.etl.featurevector.CsvToCanonicalFeatureVector.CsvToCanonicalFeatureVectorConfiguration;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.DatasetFileLines;
@@ -41,6 +44,7 @@ import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ExternalData;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Sample;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.Geometry;
import ch.systemsx.cisd.openbis.plugin.screening.shared.basic.dto.ScreeningConstants;
import ch.systemsx.cisd.utils.CsvFileReaderHelper;
@@ -101,17 +105,56 @@ public class FeatureVectorStorageProcessor extends AbstractDelegatingStorageProc
throws IOException
{
HCSContainerDatasetInfo datasetInfo = createScreeningDatasetInfo(dataSetInformation);
- DatasetFileLines fileLines = getDatasetFileLines(dataSet);
- CsvToCanonicalFeatureVector convertor =
- new CsvToCanonicalFeatureVector(fileLines, convertorConfig,
- datasetInfo.getContainerRows(), datasetInfo.getContainerColumns());
- List<CanonicalFeatureVector> fvecs = convertor.convert();
+
+ List<CanonicalFeatureVector> fvecs =
+ extractCanonicalFeatureVectors(dataSet, dataSetInformation,
+ datasetInfo.getContainerGeometry());
dataAccessObject = createDAO();
FeatureVectorUploader uploader = new FeatureVectorUploader(dataAccessObject, datasetInfo);
uploader.uploadFeatureVectors(fvecs);
}
+ private List<CanonicalFeatureVector> extractCanonicalFeatureVectors(File dataSet,
+ DataSetInformation dataSetInformation, Geometry plateGeometry) throws IOException
+ {
+ if (dataSetInformation instanceof FeatureVectorDataSetInformation)
+ {
+ return extractCanonicalFeatureVectors(
+ (FeatureVectorDataSetInformation) dataSetInformation, plateGeometry);
+ } else
+ {
+ return extractCanonicalFeatureVectorsFromFile(dataSet, plateGeometry);
+ }
+ }
+
+ private static List<CanonicalFeatureVector> extractCanonicalFeatureVectors(
+ FeatureVectorDataSetInformation dataSetInformation, Geometry plateGeometry)
+ {
+ List<FeatureDefinitionValues> featuresDefinitionValuesList =
+ dataSetInformation.getFeatures();
+
+ List<CanonicalFeatureVector> canonicalFeatureVectors =
+ new ArrayList<CanonicalFeatureVector>();
+ for (FeatureDefinitionValues featureDefinitionValues : featuresDefinitionValuesList)
+ {
+ CanonicalFeatureVector canonicalFeatureVector =
+ featureDefinitionValues.getCanonicalFeatureVector(plateGeometry);
+ canonicalFeatureVectors.add(canonicalFeatureVector);
+ }
+ return canonicalFeatureVectors;
+ }
+
+ private List<CanonicalFeatureVector> extractCanonicalFeatureVectorsFromFile(File dataSet,
+ Geometry plateGeometry) throws IOException
+ {
+ DatasetFileLines fileLines = getDatasetFileLines(dataSet);
+ CsvToCanonicalFeatureVector convertor =
+ new CsvToCanonicalFeatureVector(fileLines, convertorConfig, plateGeometry);
+ List<CanonicalFeatureVector> fvecs = convertor.convert();
+ return fvecs;
+ }
+
private HCSContainerDatasetInfo createScreeningDatasetInfo(DataSetInformation dataSetInformation)
{
Sample sampleOrNull = tryFindSampleForDataSet(dataSetInformation);
@@ -194,7 +237,10 @@ public class FeatureVectorStorageProcessor extends AbstractDelegatingStorageProc
return CsvFileReaderHelper.getDatasetFileLines(file, configuration);
}
- /** Accepts all non-image datasets (and assumes they are single CSV files). */
+ /**
+ * Accepts all non-image datasets (and assumes they are single CSV files or
+ * FeatureVectorDataSetInformation).
+ */
public boolean accepts(DataSetInformation dataSetInformation, File incomingDataSet)
{
return dataSetInformation instanceof ImageDataSetInformation == false;
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/featurevector/FeatureVectorUploader.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/featurevector/FeatureVectorUploader.java
index 5d77e1cbf6e..b0caf06cff7 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/featurevector/FeatureVectorUploader.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/featurevector/FeatureVectorUploader.java
@@ -118,14 +118,14 @@ public class FeatureVectorUploader
void createFeatureVocabularyTerms()
{
List<ImgFeatureVocabularyTermDTO> terms = fvec.getVocabularyTerms();
- long featureDefId = fvec.getFeatureDef().getId();
- // The FK of the feature def are not yet valid. Patch them up.
- for (ImgFeatureVocabularyTermDTO term : terms)
- {
- term.setFeatureDefId(featureDefId);
- }
- if (terms.size() > 0)
+ if (terms != null && terms.size() > 0)
{
+ long featureDefId = fvec.getFeatureDef().getId();
+ // The FK of the feature def are not yet valid. Patch them up.
+ for (ImgFeatureVocabularyTermDTO term : terms)
+ {
+ term.setFeatureDefId(featureDefId);
+ }
dao.addFeatureVocabularyTerms(terms);
}
}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/JythonPlateDataSetHandler.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/JythonPlateDataSetHandler.java
index 765d3450d12..3f70572ad39 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/JythonPlateDataSetHandler.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/JythonPlateDataSetHandler.java
@@ -10,19 +10,22 @@ import ch.systemsx.cisd.etlserver.registrator.DataSetRegistrationService;
import ch.systemsx.cisd.etlserver.registrator.IDataSetRegistrationDetailsFactory;
import ch.systemsx.cisd.etlserver.registrator.JythonTopLevelDataSetHandler;
import ch.systemsx.cisd.etlserver.registrator.api.v1.IDataSet;
-import ch.systemsx.cisd.etlserver.registrator.api.v1.impl.DataSet;
import ch.systemsx.cisd.etlserver.registrator.api.v1.impl.DataSetRegistrationTransaction;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.FeatureVectorDataSetInformation;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.FeaturesBuilder;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.BasicDataSetInformation;
-import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.SimpleImageDataConfig;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.IFeaturesBuilder;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageDataSetInformation;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.SimpleImageDataConfig;
+import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.basic.dto.ScreeningConstants;
/**
* Jython dropbox for HCS and Microscopy image datasets.
*
* @author Tomasz Pylak
*/
-public class JythonPlateDataSetHandler extends
- JythonTopLevelDataSetHandler<ImageDataSetInformation>
+public class JythonPlateDataSetHandler extends JythonTopLevelDataSetHandler<DataSetInformation>
{
public JythonPlateDataSetHandler(TopLevelDataSetRegistratorGlobalState globalState)
{
@@ -33,50 +36,97 @@ public class JythonPlateDataSetHandler extends
* Create a screening specific factory available to the python script.
*/
@Override
- protected IDataSetRegistrationDetailsFactory<ImageDataSetInformation> createObjectFactory(
+ protected IDataSetRegistrationDetailsFactory<DataSetInformation> createObjectFactory(
PythonInterpreter interpreter)
{
return new JythonPlateDatasetFactory(getRegistratorState());
}
- public static class JythonPlateDatasetFactory extends
- JythonObjectFactory<ImageDataSetInformation>
+ public static class JythonPlateDatasetFactory extends JythonObjectFactory<DataSetInformation>
{
+ private final IDataSetRegistrationDetailsFactory<ImageDataSetInformation> imageDatasetFactory;
+
+ private final IDataSetRegistrationDetailsFactory<FeatureVectorDataSetInformation> featureVectorDatasetFactory;
+
public JythonPlateDatasetFactory(OmniscientTopLevelDataSetRegistratorState registratorState)
{
super(registratorState);
+ this.imageDatasetFactory =
+ new JythonObjectFactory<ImageDataSetInformation>(this.registratorState)
+ {
+ @Override
+ protected ImageDataSetInformation createDataSetInformation()
+ {
+ return new ImageDataSetInformation();
+ }
+ };
+ this.featureVectorDatasetFactory =
+ new JythonObjectFactory<FeatureVectorDataSetInformation>(this.registratorState)
+ {
+ @Override
+ protected FeatureVectorDataSetInformation createDataSetInformation()
+ {
+ return new FeatureVectorDataSetInformation();
+ }
+ };
}
+ /** By default a starndard dataset is created. */
@Override
- public DataSet<ImageDataSetInformation> createDataSet(
- DataSetRegistrationDetails<ImageDataSetInformation> registrationDetails,
- File stagingFile)
+ protected DataSetInformation createDataSetInformation()
{
- return new DataSet<ImageDataSetInformation>(registrationDetails, stagingFile);
+ return new DataSetInformation();
}
- @Override
- protected ImageDataSetInformation createDataSetInformation()
+ public DataSetRegistrationDetails<ImageDataSetInformation> createImageRegistrationDetails(
+ SimpleImageDataConfig imageDataSet, File incomingDatasetFolder)
{
- return new ImageDataSetInformation();
+ return SimpleImageDataSetRegistrator.createImageDatasetDetails(imageDataSet,
+ incomingDatasetFolder, imageDatasetFactory);
}
- /**
- * Factory method that creates a new registration details object for image datasets.
- */
- public DataSetRegistrationDetails<ImageDataSetInformation> createImageRegistrationDetails()
+ /** a simple method to register the described image dataset in a separate transaction */
+ public void registerImageDataset(SimpleImageDataConfig imageDataSet,
+ File incomingDatasetFolder,
+ DataSetRegistrationService<ImageDataSetInformation> service)
{
- DataSetRegistrationDetails<ImageDataSetInformation> registrationDetails =
- new DataSetRegistrationDetails<ImageDataSetInformation>();
- ImageDataSetInformation dataSetInfo = new ImageDataSetInformation();
- setDatabaseInstance(dataSetInfo);
- registrationDetails.setDataSetInformation(dataSetInfo);
+ DataSetRegistrationDetails<ImageDataSetInformation> imageDatasetDetails =
+ createImageRegistrationDetails(imageDataSet, incomingDatasetFolder);
+ DataSetRegistrationTransaction<ImageDataSetInformation> transaction =
+ service.transaction(incomingDatasetFolder, imageDatasetFactory);
+ IDataSet newDataset = transaction.createNewDataSet(imageDatasetDetails);
+ transaction.moveFile(incomingDatasetFolder.getPath(), newDataset);
+ transaction.commit();
+ }
+
+ // ----
+
+ public IFeaturesBuilder createFeaturesBuilder()
+ {
+ return new FeaturesBuilder();
+ }
+
+ public DataSetRegistrationDetails<FeatureVectorDataSetInformation> createFeatureVectorRegistrationDetails(
+ IFeaturesBuilder featureBuilder, File incomingDatasetFolder)
+ {
+ FeaturesBuilder myFeatureBuilder = (FeaturesBuilder) featureBuilder;
+ DataSetRegistrationDetails<FeatureVectorDataSetInformation> registrationDetails =
+ featureVectorDatasetFactory.createDataSetRegistrationDetails();
+ FeatureVectorDataSetInformation featureVectorDataSet =
+ registrationDetails.getDataSetInformation();
+ featureVectorDataSet.setFeatures(myFeatureBuilder.getFeatureDefinitionValuesList());
+ registrationDetails
+ .setDataSetType(ScreeningConstants.DEFAULT_ANALYSIS_WELL_DATASET_TYPE);
+ registrationDetails.setMeasuredData(false);
return registrationDetails;
}
/**
* Factory method that creates a new registration details object for non-image datasets.
+ *
+ * @deprecated used only in Matt's dropbox to register analysis datasets. Will be removed.
*/
+ @Deprecated
public DataSetRegistrationDetails<BasicDataSetInformation> createBasicRegistrationDetails()
{
DataSetRegistrationDetails<BasicDataSetInformation> registrationDetails =
@@ -86,30 +136,5 @@ public class JythonPlateDataSetHandler extends
registrationDetails.setDataSetInformation(dataSetInfo);
return registrationDetails;
}
-
- public DataSetRegistrationDetails<ImageDataSetInformation> createImageRegistrationDetails(
- SimpleImageDataConfig imageDataSet, File incomingDatasetFolder)
- {
- return SimpleImageDataSetRegistrator.createImageDatasetDetails(imageDataSet,
- incomingDatasetFolder, this);
- }
-
- /** a simple method to register the described image dataset */
- public void registerImageDataset(SimpleImageDataConfig imageDataSet,
- File incomingDatasetFolder,
- DataSetRegistrationService<ImageDataSetInformation> service)
- {
- DataSetRegistrationDetails<ImageDataSetInformation> imageDatasetDetails =
- createImageRegistrationDetails(imageDataSet, incomingDatasetFolder);
- IDataSetRegistrationDetailsFactory<ImageDataSetInformation> myself = this;
- // TODO 2011-02-15, Tomasz Pylak: remove this casting
- @SuppressWarnings("unchecked")
- DataSetRegistrationTransaction<ImageDataSetInformation> transaction =
- (DataSetRegistrationTransaction<ImageDataSetInformation>) service.transaction(
- incomingDatasetFolder, myself);
- IDataSet newDataset = transaction.createNewDataSet(imageDatasetDetails);
- transaction.moveFile(incomingDatasetFolder.getPath(), newDataset);
- transaction.commit();
- }
}
}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/SimpleImageDataSetRegistrator.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/SimpleImageDataSetRegistrator.java
index 365d4d1a5eb..14b7e70d7cd 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/SimpleImageDataSetRegistrator.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/SimpleImageDataSetRegistrator.java
@@ -25,13 +25,13 @@ import java.util.Set;
import ch.systemsx.cisd.common.filesystem.FileOperations;
import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.etlserver.registrator.DataSetRegistrationDetails;
+import ch.systemsx.cisd.etlserver.registrator.IDataSetRegistrationDetailsFactory;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.Channel;
-import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.SimpleImageDataConfig;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageDataSetInformation;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageFileInfo;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageMetadata;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.Location;
-import ch.systemsx.cisd.openbis.dss.etl.jython.JythonPlateDataSetHandler.JythonPlateDatasetFactory;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.SimpleImageDataConfig;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetType;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.FileFormatType;
@@ -65,7 +65,8 @@ public class SimpleImageDataSetRegistrator
}
public static DataSetRegistrationDetails<ImageDataSetInformation> createImageDatasetDetails(
- SimpleImageDataConfig imageDataSet, File incoming, JythonPlateDatasetFactory factory)
+ SimpleImageDataConfig imageDataSet, File incoming,
+ IDataSetRegistrationDetailsFactory<ImageDataSetInformation> factory)
{
return new SimpleImageDataSetRegistrator(imageDataSet).createImageDatasetDetails(incoming,
factory);
@@ -79,10 +80,11 @@ public class SimpleImageDataSetRegistrator
}
private DataSetRegistrationDetails<ImageDataSetInformation> createImageDatasetDetails(
- File incoming, JythonPlateDatasetFactory factory)
+ File incoming,
+ IDataSetRegistrationDetailsFactory<ImageDataSetInformation> imageDatasetFactory)
{
DataSetRegistrationDetails<ImageDataSetInformation> registrationDetails =
- factory.createImageRegistrationDetails();
+ imageDatasetFactory.createDataSetRegistrationDetails();
ImageDataSetInformation imageDataset = registrationDetails.getDataSetInformation();
setImageDataset(incoming, imageDataset);
setRegistrationDetails(registrationDetails, imageDataset);
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/registrator/api/v1/impl/ImageDataSet.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/registrator/api/v1/impl/ImageDataSet.java
index 6742975c87b..b83fb6d8393 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/registrator/api/v1/impl/ImageDataSet.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/registrator/api/v1/impl/ImageDataSet.java
@@ -28,7 +28,8 @@ import ch.systemsx.cisd.openbis.dss.etl.registrator.api.v1.IImageDataSet;
*
* @author Franz-Josef Elmer
*/
-// TODO 2011-02-02, Tomasz Pylak: We do not use it, is it really needed? For sure it's not complete.
+// TODO 2011-02-02, Tomasz Pylak: make it complete and accessible through transactional framework.
+// This is a stub now.
public class ImageDataSet extends DataSet<ImageDataSetInformation> implements IImageDataSet
{
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/imaging/dataaccess/ImgFeatureValuesDTO.java b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/imaging/dataaccess/ImgFeatureValuesDTO.java
index a3c0800ab50..42c446be139 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/imaging/dataaccess/ImgFeatureValuesDTO.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/imaging/dataaccess/ImgFeatureValuesDTO.java
@@ -51,7 +51,7 @@ public class ImgFeatureValuesDTO extends AbstractHashable
// All Data-Object classes must have a default constructor.
}
- public ImgFeatureValuesDTO(Double zInM, Double tInSec, PlateFeatureValues values,
+ public ImgFeatureValuesDTO(Double tInSec, Double zInM, PlateFeatureValues values,
long featureDefId)
{
this.z = zInM;
diff --git a/screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/FeatureDefinitionValuesTest.java b/screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/FeatureDefinitionValuesTest.java
new file mode 100644
index 00000000000..06a68ba8f80
--- /dev/null
+++ b/screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/FeatureDefinitionValuesTest.java
@@ -0,0 +1,112 @@
+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.etl.dto.api.impl;
+
+import java.util.List;
+
+import org.testng.AssertJUnit;
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.openbis.dss.etl.featurevector.CanonicalFeatureVector;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.Geometry;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.dto.PlateFeatureValues;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgFeatureDefDTO;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgFeatureValuesDTO;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgFeatureVocabularyTermDTO;
+
+/**
+ * Test of {@link FeatureDefinitionValues}
+ *
+ * @author Tomasz Pylak
+ */
+public class FeatureDefinitionValuesTest extends AssertJUnit
+{
+ @Test
+ public void test()
+ {
+ FeatureDefinitionValues def = new FeatureDefinitionValues(new ImgFeatureDefDTO());
+ def.addValue("A1", "1.2");
+ def.addValue("B2", "2.1");
+ CanonicalFeatureVector vector =
+ def.getCanonicalFeatureVector(Geometry.createFromRowColDimensions(2, 2));
+ assertNull(vector.getVocabularyTerms());
+ assertEquals(1, vector.getValues().size());
+ ImgFeatureValuesDTO featureValuesDTO = vector.getValues().get(0);
+ assertNull(featureValuesDTO.getT());
+ assertNull(featureValuesDTO.getZ());
+ PlateFeatureValues values = featureValuesDTO.getValues();
+ assertEquals(1.2f, values.getForWellLocation(1, 1));
+ assertEquals(2.1f, values.getForWellLocation(2, 2));
+ assertTrue(Float.isNaN(values.getForWellLocation(2, 1)));
+ assertTrue(Float.isNaN(values.getForWellLocation(1, 2)));
+ }
+
+ @Test
+ public void testTimepointsAndDepthScans()
+ {
+ FeatureDefinitionValues def = new FeatureDefinitionValues(new ImgFeatureDefDTO());
+ def.setSeries(1.1, null);
+ def.addValue("A1", "1");
+ def.setSeries(null, 2.2);
+ def.addValue("A1", "10");
+
+ CanonicalFeatureVector vector =
+ def.getCanonicalFeatureVector(Geometry.createFromRowColDimensions(1, 2));
+ assertNull(vector.getVocabularyTerms());
+ assertEquals(2, vector.getValues().size());
+
+ ImgFeatureValuesDTO featureValuesDTO = vector.getValues().get(0);
+ assertEquals(1.1d, featureValuesDTO.getT());
+ assertNull(featureValuesDTO.getZ());
+ PlateFeatureValues values = featureValuesDTO.getValues();
+ assertEquals(1f, values.getForWellLocation(1, 1));
+ assertTrue(Float.isNaN(values.getForWellLocation(1, 2)));
+
+ featureValuesDTO = vector.getValues().get(1);
+ assertEquals(2.2d, featureValuesDTO.getZ());
+ assertNull(featureValuesDTO.getT());
+ values = featureValuesDTO.getValues();
+ assertEquals(10f, values.getForWellLocation(1, 1));
+ assertTrue(Float.isNaN(values.getForWellLocation(1, 2)));
+ }
+
+ @Test
+ public void testVocabularyFeatures()
+ {
+ FeatureDefinitionValues def = new FeatureDefinitionValues(new ImgFeatureDefDTO());
+ def.addValue("A1", "a");
+ def.addValue("A2", "b");
+ def.addValue("A3", "a");
+ CanonicalFeatureVector vector =
+ def.getCanonicalFeatureVector(Geometry.createFromRowColDimensions(1, 3));
+ List<ImgFeatureVocabularyTermDTO> terms = vector.getVocabularyTerms();
+ assertNotNull(terms);
+ assertEquals(2, terms.size());
+ assertEquals("a", terms.get(0).getCode());
+ assertEquals("b", terms.get(1).getCode());
+
+ assertEquals(1, vector.getValues().size());
+ ImgFeatureValuesDTO featureValuesDTO = vector.getValues().get(0);
+ assertNull(featureValuesDTO.getT());
+ assertNull(featureValuesDTO.getZ());
+ PlateFeatureValues values = featureValuesDTO.getValues();
+ assertEquals(terms.get(0).getSequenceNumber(), (int) values.getForWellLocation(1, 1));
+ assertEquals(terms.get(1).getSequenceNumber(), (int) values.getForWellLocation(1, 2));
+ assertEquals(terms.get(0).getSequenceNumber(), (int) values.getForWellLocation(1, 3));
+ }
+
+}
\ No newline at end of file
--
GitLab