diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/fvec/CSVToCanonicalFeatureVector.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/fvec/CSVToCanonicalFeatureVector.java
deleted file mode 100644
index 351d0c08db96510b9f62fd5a33cc0cac7cd1aef8..0000000000000000000000000000000000000000
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/fvec/CSVToCanonicalFeatureVector.java
+++ /dev/null
@@ -1,274 +0,0 @@
-/*
- * Copyright 2010 ETH Zuerich, CISD
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package ch.systemsx.cisd.openbis.dss.etl.dataaccess.fvec;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-
-import ch.systemsx.cisd.base.convert.NativeTaggedArray;
-import ch.systemsx.cisd.base.mdarray.MDDoubleArray;
-import ch.systemsx.cisd.bds.hcs.Location;
-import ch.systemsx.cisd.common.geometry.Point;
-import ch.systemsx.cisd.openbis.dss.etl.dataaccess.ImgFeatureDefDTO;
-import ch.systemsx.cisd.openbis.dss.etl.dataaccess.ImgFeatureValuesDTO;
-import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.DatasetFileLines;
-
-/**
- * @author Chandrasekhar Ramakrishnan
- */
-public class CSVToCanonicalFeatureVector
-{
- public static class CSVToCanonicalFeatureVectorConfiguration
- {
- private final String wellRowColumn;
-
- private final String wellColumnColumn;
-
- private final boolean isWellColumnAlphanumeric;
-
- private final boolean isSplit;
-
- public CSVToCanonicalFeatureVectorConfiguration(String wellRow, String wellColumn,
- boolean wellColumnIsAlphanumeric)
- {
- this.wellRowColumn = wellRow;
- this.wellColumnColumn = wellColumn;
- this.isWellColumnAlphanumeric = wellColumnIsAlphanumeric;
-
- isSplit = (false == wellRow.equals(wellColumn));
- }
-
- public String getWellRowColumn()
- {
- return wellRowColumn;
- }
-
- public String getWellColumnColumn()
- {
- return wellColumnColumn;
- }
-
- public boolean isWellColumnAlphanumeric()
- {
- return isWellColumnAlphanumeric;
- }
-
- public boolean isSplit()
- {
- return isSplit;
- }
- }
-
- private final CSVToCanonicalFeatureVectorConfiguration configuration;
-
- private final String[] header;
-
- private final List<String[]> lines;
-
- private final ArrayList<FeatureColumn> columns = new ArrayList<FeatureColumn>();
-
- // Will be initialized during conversion
- private int xColumn = -1;
-
- private int yColumn = -1;
-
- private int maxX = 0;
-
- private int maxY = 0;
-
- public CSVToCanonicalFeatureVector(DatasetFileLines fileLines,
- CSVToCanonicalFeatureVectorConfiguration config)
- {
- this.configuration = config;
- this.header = fileLines.getHeaderTokens();
- this.lines = fileLines.getDataLines();
- }
-
- public ArrayList<CanonicalFeatureVector> convert()
- {
- initializeColumns();
- readLines();
-
- return convertColumnsToFeatureFectors();
- }
-
- private ArrayList<CanonicalFeatureVector> convertColumnsToFeatureFectors()
- {
- int[] dims =
- { maxX + 1, maxY + 1 };
-
- ArrayList<CanonicalFeatureVector> result = new ArrayList<CanonicalFeatureVector>();
- for (FeatureColumn column : columns)
- {
- if ((true == column.isWellName) || (false == column.isNumeric))
- {
- continue;
- }
- CanonicalFeatureVector featureVector = convertColumnToFeatureVector(dims, column);
- result.add(featureVector);
- }
-
- return result;
- }
-
- private CanonicalFeatureVector convertColumnToFeatureVector(int[] dims, FeatureColumn column)
- {
- CanonicalFeatureVector featureVector = new CanonicalFeatureVector();
- featureVector.setFeatureDef(new ImgFeatureDefDTO(column.name, column.name, 0));
- byte[] valuesValues = convertColumnToByteArray(dims, column);
- ImgFeatureValuesDTO values = new ImgFeatureValuesDTO(0., 0., valuesValues, 0);
- featureVector.setValues(Collections.singletonList(values));
- return featureVector;
- }
-
- private byte[] convertColumnToByteArray(int[] dims, FeatureColumn column)
- {
- MDDoubleArray doubleArray = new MDDoubleArray(dims);
- for (Point loc : column.values.keySet())
- {
- Double value = column.values.get(loc);
- doubleArray.set(value, loc.getX(), loc.getY());
- }
-
- return NativeTaggedArray.toByteArray(doubleArray);
- }
-
- private void readLines()
- {
- for (String[] line : lines)
- {
- readLine(line);
- }
- }
-
- private void readLine(String[] line)
- {
- Point point = readPointFromLine(line);
- for (FeatureColumn column : columns)
- {
- if ((true == column.isWellName) || (false == column.isNumeric))
- {
- continue;
- }
- try
- {
- column.values.put(point, Double.parseDouble(line[column.index]));
- } catch (NumberFormatException ex)
- {
- // skip this column in the future
- column.isNumeric = false;
- }
- }
-
- if (point.getX() > maxX)
- {
- maxX = point.getX();
- }
-
- if (point.getY() > maxY)
- {
- maxY = point.getY();
- }
- }
-
- private Point readPointFromLine(String[] line)
- {
- int x, y;
- if (configuration.isSplit())
- {
- String xString = line[xColumn];
- String yString = line[yColumn];
- if (configuration.isWellColumnAlphanumeric())
- {
- Location loc =
- Location.tryCreateLocationFromSplitMatrixCoordinate(xString, yString);
- x = loc.getX();
- y = loc.getY();
- } else
- {
- x = Integer.parseInt(xString);
- y = Integer.parseInt(yString);
- }
- } else
- {
- Location loc = Location.tryCreateLocationFromMatrixCoordinate(line[xColumn]);
- x = loc.getX();
- y = loc.getY();
- }
-
- // Well-locations are 1-offset; we need 0-offset to put into an matrix
- Point point = new Point(x - 1, y - 1);
- return point;
- }
-
- private void initializeColumns()
- {
- for (int i = 0; i < header.length; ++i)
- {
- String headerName = header[i];
- boolean isWellName;
- if (configuration.getWellRowColumn().equals(headerName))
- {
- xColumn = i;
- isWellName = true;
- } else if (configuration.getWellColumnColumn().equals(headerName))
- {
- yColumn = i;
- isWellName = true;
- } else
- {
- isWellName = false;
- }
- FeatureColumn featureColumn = new FeatureColumn(i, headerName, isWellName);
- columns.add(featureColumn);
- }
-
- if (false == configuration.isSplit())
- {
- yColumn = xColumn;
- }
-
- if (xColumn < 0 || yColumn < 0)
- {
- throw new IllegalArgumentException("Could not parse data set");
- }
- }
-
- private static class FeatureColumn
- {
- private final int index;
-
- private final String name;
-
- private final boolean isWellName;
-
- private final HashMap<Point, Double> values;
-
- // this may change during the course of reading the file
- private boolean isNumeric = true;
-
- private FeatureColumn(int index, String name, boolean isWellName)
- {
- this.index = index;
- this.name = name;
- this.isWellName = isWellName;
- values = new HashMap<Point, Double>();
- }
- }
-}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/fvec/CanonicalFeatureVector.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/fvec/CanonicalFeatureVector.java
deleted file mode 100644
index c79fe4212213e2d95ed55ae94e109d9097dc0057..0000000000000000000000000000000000000000
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/fvec/CanonicalFeatureVector.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright 2010 ETH Zuerich, CISD
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package ch.systemsx.cisd.openbis.dss.etl.dataaccess.fvec;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import ch.systemsx.cisd.openbis.dss.etl.dataaccess.ImgFeatureDefDTO;
-import ch.systemsx.cisd.openbis.dss.etl.dataaccess.ImgFeatureValuesDTO;
-
-/**
- * Image feature vectors stored in a standardized form.
- *
- * @author Chandrasekhar Ramakrishnan
- */
-public class CanonicalFeatureVector
-{
- private ImgFeatureDefDTO featureDef;
-
- private List<ImgFeatureValuesDTO> values = new ArrayList<ImgFeatureValuesDTO>();
-
- public CanonicalFeatureVector()
- {
-
- }
-
- /**
- * The feature def for this feature vector. If the feature def has not yet been commited to the
- * DB, then it has no ID.
- */
- public ImgFeatureDefDTO getFeatureDef()
- {
- return featureDef;
- }
-
- public void setFeatureDef(ImgFeatureDefDTO featureDef)
- {
- this.featureDef = featureDef;
- }
-
- /**
- * The feature values for this feature vector. If the feature def has not yet been commited to
- * the DB, then values FK to the feature def will not be valid. If the feature values have not
- * yet been committed, then their ids will not be valid.
- */
- public List<ImgFeatureValuesDTO> getValues()
- {
- return values;
- }
-
- public void setValues(List<ImgFeatureValuesDTO> values)
- {
- this.values = values;
- }
-}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/fvec/FeatureVectorStorageProcessor.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/fvec/FeatureVectorStorageProcessor.java
deleted file mode 100644
index fa170c9fec12ec54a1cb44049db12ede146b93d4..0000000000000000000000000000000000000000
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/fvec/FeatureVectorStorageProcessor.java
+++ /dev/null
@@ -1,246 +0,0 @@
-/*
- * Copyright 2010 ETH Zuerich, CISD
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package ch.systemsx.cisd.openbis.dss.etl.dataaccess.fvec;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Properties;
-
-import javax.sql.DataSource;
-
-import net.lemnik.eodsql.QueryTool;
-
-import ch.systemsx.cisd.base.exceptions.IOExceptionUnchecked;
-import ch.systemsx.cisd.common.mail.IMailClient;
-import ch.systemsx.cisd.common.utilities.PropertyUtils;
-import ch.systemsx.cisd.etlserver.AbstractDelegatingStorageProcessor;
-import ch.systemsx.cisd.etlserver.ITypeExtractor;
-import ch.systemsx.cisd.openbis.dss.etl.ScreeningContainerDatasetInfo;
-import ch.systemsx.cisd.openbis.dss.etl.dataaccess.IImagingUploadDAO;
-import ch.systemsx.cisd.openbis.dss.etl.dataaccess.fvec.CSVToCanonicalFeatureVector.CSVToCanonicalFeatureVectorConfiguration;
-import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.DatasetFileLines;
-import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
-import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
-import ch.systemsx.cisd.utils.CsvFileReaderHelper;
-import ch.systemsx.cisd.utils.CsvFileReaderHelper.ICsvFileReaderConfiguration;
-
-/**
- * Extract features from the file and store them in the database.
- *
- * @author Chandrasekhar Ramakrishnan
- */
-public class FeatureVectorStorageProcessor extends AbstractDelegatingStorageProcessor
-{
- private static class FeatureVectorStorageProcessorConfiguration implements
- ICsvFileReaderConfiguration
- {
- private static final String SEPARATOR_PROPERTY_KEY = "separator";
-
- private static final String IGNORE_COMMENTS_PROPERTY_KEY = "ignore-comments";
-
- private static final String WELL_NAME_ROW_PROPERTY_KEY = "well-name-row";
-
- private static final String WELL_NAME_COL_PROPERTY_KEY = "well-name-col";
-
- private static final String WELL_NAME_COL_ALPHA_NUM_PROPERTY_KEY =
- "well-name-col-is-alphanum";
-
- private static final char DEFAULT_DELIMITER = ';';
-
- private static final String DEFAULT_WELL_ROW = "WellName";
-
- private static final String DEFAULT_WELL_COL = "WellName";
-
- private static final boolean DEFAULT_WELL_ROW_ALPHANUM = true;
-
- private final char columnDelimiter;
-
- private final boolean ignoreComments;
-
- private final char comment;
-
- private final String wellRow;
-
- private final String wellColumn;
-
- private final boolean isWellColAlphanumeric;
-
- private FeatureVectorStorageProcessorConfiguration(Properties properties)
- {
- comment = '#';
-
- this.columnDelimiter =
- PropertyUtils.getChar(properties, SEPARATOR_PROPERTY_KEY, DEFAULT_DELIMITER);
- this.ignoreComments =
- PropertyUtils.getBoolean(properties, IGNORE_COMMENTS_PROPERTY_KEY, true);
-
- this.wellRow = properties.getProperty(WELL_NAME_ROW_PROPERTY_KEY, DEFAULT_WELL_ROW);
-
- this.wellColumn = properties.getProperty(WELL_NAME_COL_PROPERTY_KEY, DEFAULT_WELL_COL);
-
- this.isWellColAlphanumeric =
- PropertyUtils.getBoolean(properties, WELL_NAME_COL_ALPHA_NUM_PROPERTY_KEY,
- DEFAULT_WELL_ROW_ALPHANUM);
- }
-
- public char getColumnDelimiter()
- {
- return columnDelimiter;
- }
-
- public char getCommentDelimiter()
- {
- return comment;
- }
-
- public boolean isIgnoreComments()
- {
- return ignoreComments;
- }
-
- public boolean isSkipEmptyRecords()
- {
- return true;
- }
-
- public String getWellRow()
- {
- return wellRow;
- }
-
- public String getWellColumn()
- {
- return wellColumn;
- }
-
- public boolean isWellColAlphanumeric()
- {
- return isWellColAlphanumeric;
- }
- }
-
- private static final String ORIGINAL_DIR = "original";
-
- private final FeatureVectorStorageProcessorConfiguration configuration;
-
- private final CSVToCanonicalFeatureVectorConfiguration convertorConfig;
-
- private final DataSource dataSource;
-
- // Execution state of this object -- set to null after an execution is finished.
- private IImagingUploadDAO dataAccessObject = null;
-
- public FeatureVectorStorageProcessor(Properties properties)
- {
- super(properties);
- this.configuration = new FeatureVectorStorageProcessorConfiguration(properties);
- convertorConfig =
- new CSVToCanonicalFeatureVectorConfiguration(configuration.getWellRow(),
- configuration.getWellColumn(), configuration.isWellColAlphanumeric());
- this.dataSource = ServiceProvider.getDataSourceProvider().getDataSource(properties);
- }
-
- @Override
- public File storeData(final DataSetInformation dataSetInformation,
- final ITypeExtractor typeExtractor, final IMailClient mailClient,
- final File incomingDataSetDirectory, final File rootDir)
- {
- File answer =
- super.storeData(dataSetInformation, typeExtractor, mailClient,
- incomingDataSetDirectory, rootDir);
- // Import into the data base
- File parent = new File(answer, ORIGINAL_DIR);
- File dataSet = new File(parent, incomingDataSetDirectory.getName());
-
- try
- {
- loadDataSetIntoDatabase(dataSet, dataSetInformation);
- } catch (IOException ex)
- {
- throw new IOExceptionUnchecked(ex);
- }
-
- return answer;
- }
-
- private void loadDataSetIntoDatabase(File dataSet, DataSetInformation dataSetInformation)
- throws IOException
- {
- DatasetFileLines fileLines = getDatasetFileLines(dataSet);
- CSVToCanonicalFeatureVector convertor =
- new CSVToCanonicalFeatureVector(fileLines, convertorConfig);
- ArrayList<CanonicalFeatureVector> fvecs = convertor.convert();
-
- dataAccessObject = createDAO();
- FeatureVectorUploader uploader =
- new FeatureVectorUploader(dataAccessObject, ScreeningContainerDatasetInfo
- .createScreeningDatasetInfo(dataSetInformation));
- uploader.uploadFeatureVectors(fvecs);
- }
-
- private IImagingUploadDAO createDAO()
- {
- return QueryTool.getQuery(dataSource, IImagingUploadDAO.class);
- }
-
- @Override
- public void commit()
- {
- super.commit();
-
- if (null == dataAccessObject)
- {
- return;
- }
-
- dataAccessObject.commit();
- closeDataAccessObject();
- }
-
- /**
- * Close the DAO and set it to null to make clear that it is not initialized.
- */
- private void closeDataAccessObject()
- {
- dataAccessObject.close();
- dataAccessObject = null;
- }
-
- @Override
- public UnstoreDataAction rollback(final File incomingDataSetDirectory,
- final File storedDataDirectory, Throwable exception)
- {
- // Delete the data from the database
- if (null != dataAccessObject)
- {
- dataAccessObject.rollback();
- closeDataAccessObject();
- }
-
- return super.rollback(incomingDataSetDirectory, storedDataDirectory, exception);
- }
-
- /**
- * Return the tabular data as a DatasetFileLines.
- */
- private DatasetFileLines getDatasetFileLines(File file) throws IOException
- {
- return CsvFileReaderHelper.getDatasetFileLines(file, configuration);
- }
-
-}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/fvec/FeatureVectorUploader.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/fvec/FeatureVectorUploader.java
deleted file mode 100644
index ae25ddaf9f344f622aa983b169b82728d4884b44..0000000000000000000000000000000000000000
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/fvec/FeatureVectorUploader.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Copyright 2010 ETH Zuerich, CISD
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package ch.systemsx.cisd.openbis.dss.etl.dataaccess.fvec;
-
-import java.util.List;
-
-import ch.systemsx.cisd.openbis.dss.etl.ScreeningContainerDatasetInfo;
-import ch.systemsx.cisd.openbis.dss.etl.ScreeningContainerDatasetInfoHelper;
-import ch.systemsx.cisd.openbis.dss.etl.dataaccess.IImagingUploadDAO;
-import ch.systemsx.cisd.openbis.dss.etl.dataaccess.ImgFeatureDefDTO;
-import ch.systemsx.cisd.openbis.dss.etl.dataaccess.ImgFeatureValuesDTO;
-
-/**
- * Helper class for uploading feature vectors from the file system into the data base.
- *
- * @author Chandrasekhar Ramakrishnan
- */
-public class FeatureVectorUploader
-{
- private final IImagingUploadDAO dao;
-
- private final ScreeningContainerDatasetInfo info;
-
- public FeatureVectorUploader(IImagingUploadDAO imagingDao, ScreeningContainerDatasetInfo info)
- {
- this.dao = imagingDao;
- this.info = info;
- }
-
- public void uploadFeatureVectors(List<CanonicalFeatureVector> fvecs)
- {
- ScreeningContainerDatasetInfoHelper helper =
- new ScreeningContainerDatasetInfoHelper(dao, info);
- long expId = helper.getOrCreateExperiment();
- long contId = helper.getOrCreateContainer(expId);
- long dataSetId = helper.getOrCreateDataset(contId);
-
- for (CanonicalFeatureVector fvec : fvecs)
- {
- FeatureVectorUploaderHelper fvecUploader =
- new FeatureVectorUploaderHelper(dao, info, dataSetId, fvec);
- fvecUploader.createFeatureDef();
- fvecUploader.createFeatureValues();
- }
- }
-
- private static class FeatureVectorUploaderHelper
- {
- private final IImagingUploadDAO dao;
-
- private final long dataSetId;
-
- private final CanonicalFeatureVector fvec;
-
- FeatureVectorUploaderHelper(IImagingUploadDAO dao, ScreeningContainerDatasetInfo info,
- long dataSetId, CanonicalFeatureVector fvec)
- {
- this.dao = dao;
- this.dataSetId = dataSetId;
- this.fvec = fvec;
- }
-
- void createFeatureDef()
- {
- // The PK and FK of the feature def are not yet valid. Patch them up.
- ImgFeatureDefDTO featureDef = fvec.getFeatureDef();
- featureDef.setDataSetId(dataSetId);
- long defId = dao.addFeatureDef(featureDef);
- featureDef.setId(defId);
- }
-
- void createFeatureValues()
- {
- ImgFeatureDefDTO featureDef = fvec.getFeatureDef();
-
- // The PK and FK of the feature def are not yet valid. Patch them up.
- for (ImgFeatureValuesDTO featureValues : fvec.getValues())
- {
- featureValues.setFeatureDefId(featureDef.getId());
- long valuesId = dao.addFeatureValues(featureValues);
- featureValues.setId(valuesId);
- }
- }
- }
-}