From 4fa3f6e7c587c963feb41f8e4c13c064855c4424 Mon Sep 17 00:00:00 2001
From: tpylak <tpylak>
Date: Wed, 16 Feb 2011 10:21:53 +0000
Subject: [PATCH] LMS-1992 jython dropbox: Write simple template jython script
for HCS data set import
SVN: 20004
---
...ractDataSetRegistrationDetailsFactory.java | 2 +-
.../dss/etl/dto/api/v1/ImageFileInfo.java | 8 +-
.../dss/etl/dto/api/v1/ImageMetadata.java | 115 ++++++
.../openbis/dss/etl/dto/api/v1/Location.java | 28 ++
.../etl/dto/api/v1/SimpleImageDataConfig.java | 338 ++++++++++++++++++
.../etl/jython/JythonPlateDataSetHandler.java | 45 ++-
.../jython/SimpleImageDataSetRegistrator.java | 217 +++++++++++
7 files changed, 740 insertions(+), 13 deletions(-)
create mode 100644 screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/ImageMetadata.java
create mode 100644 screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/Location.java
create mode 100644 screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/SimpleImageDataConfig.java
create mode 100644 screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/SimpleImageDataSetRegistrator.java
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/AbstractDataSetRegistrationDetailsFactory.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/AbstractDataSetRegistrationDetailsFactory.java
index 0c2118e51e0..fb32778bb22 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/AbstractDataSetRegistrationDetailsFactory.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/AbstractDataSetRegistrationDetailsFactory.java
@@ -48,7 +48,7 @@ public abstract class AbstractDataSetRegistrationDetailsFactory<T extends DataSe
return registrationDetails;
}
- protected final void setDatabaseInstance(T dataSetInfo)
+ protected final void setDatabaseInstance(DataSetInformation dataSetInfo)
{
dataSetInfo.setInstanceCode(registratorState.getHomeDatabaseInstance().getCode());
dataSetInfo.setInstanceUUID(registratorState.getHomeDatabaseInstance().getUuid());
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/ImageFileInfo.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/ImageFileInfo.java
index 968d74151d8..356ca5a0fc5 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/ImageFileInfo.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/ImageFileInfo.java
@@ -21,7 +21,7 @@ public final class ImageFileInfo
private Float depthOrNull;
- private Integer seriesNumber;
+ private Integer seriesNumberOrNull;
public ImageFileInfo(String channelCode, int tileRow, int tileColumn, String imageRelativePath)
{
@@ -80,7 +80,7 @@ public final class ImageFileInfo
public Integer tryGetSeriesNumber()
{
- return seriesNumber;
+ return seriesNumberOrNull;
}
// --- setters
@@ -115,7 +115,7 @@ public final class ImageFileInfo
public void setSeriesNumber(Integer value)
{
- this.seriesNumber = value;
+ this.seriesNumberOrNull = value;
}
@Override
@@ -123,7 +123,7 @@ public final class ImageFileInfo
{
return "ImageFileInfo [well=" + wellLocationOrNull + ", tile=" + tileLocation
+ ", channel=" + channelCode + ", path=" + imageRelativePath + ", timepoint="
- + timepointOrNull + ", depth=" + depthOrNull + ", seriesNumber=" + seriesNumber
+ + timepointOrNull + ", depth=" + depthOrNull + ", seriesNumber=" + seriesNumberOrNull
+ "]";
}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/ImageMetadata.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/ImageMetadata.java
new file mode 100644
index 00000000000..f8ee61cee00
--- /dev/null
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/ImageMetadata.java
@@ -0,0 +1,115 @@
+package ch.systemsx.cisd.openbis.dss.etl.dto.api.v1;
+
+import org.apache.commons.lang.StringUtils;
+
+/**
+ * Store well, channel and tile number to which an image belongs.
+ * Optionally stores timepoint/depth-scan/image series number.
+ *
+ * @author Tomasz Pylak
+ */
+public class ImageMetadata
+{
+ private String channelCode;
+
+ private int tileNumber;
+
+ private String well;
+
+ private Float timepointOrNull;
+
+ private Float depthOrNull;
+
+ private Integer seriesNumberOrNull;
+
+ public String getChannelCode()
+ {
+ return channelCode;
+ }
+
+ /** Sets channel code. */
+ public void setChannelCode(String channelCode)
+ {
+ this.channelCode = channelCode;
+ }
+
+ public int getTileNumber()
+ {
+ return tileNumber;
+ }
+
+ /** Sets tile number. It should start from 1. */
+ public void setTileNumber(int tileNumber)
+ {
+ this.tileNumber = tileNumber;
+ }
+
+ public String getWell()
+ {
+ return well;
+ }
+
+ /** Sets well code, e.g. A1 */
+ public void setWell(String well)
+ {
+ this.well = well;
+ }
+
+ /** Optional. Sets the timepoint of the image. */
+ public void setTimepoint(Float value)
+ {
+ this.timepointOrNull = value;
+ }
+
+ /** Optional. Sets the depth at which the image has been scanned. */
+ public void setDepth(Float value)
+ {
+ this.depthOrNull = value;
+ }
+
+ /**
+ * Optional. Sets the integer series number of the image. Used to order images when there are no
+ * time or depth dimentions but there is a series of images for one well, channel and tile. Can
+ * be also used together with time and depth dimention.
+ */
+ public void setSeriesNumber(Integer value)
+ {
+ this.seriesNumberOrNull = value;
+ }
+
+ public Float tryGetTimepoint()
+ {
+ return timepointOrNull;
+ }
+
+ public Float tryGetDepth()
+ {
+ return depthOrNull;
+ }
+
+ public Integer tryGetSeriesNumber()
+ {
+ return seriesNumberOrNull;
+ }
+
+ /**
+ * Validates that tile number, well and channel have been specified.
+ *
+ * @throws IllegalStateException if the object is not valid.
+ */
+ public void ensureValid()
+ {
+ if (tileNumber <= 0)
+ {
+ throw new IllegalStateException("Tile number has to be > 0, but is " + tileNumber);
+ }
+ if (StringUtils.isBlank(channelCode))
+ {
+ throw new IllegalStateException("Channel code is not specified");
+ }
+ if (StringUtils.isBlank(well))
+ {
+ throw new IllegalStateException("Well is not specified");
+ }
+ }
+}
\ No newline at end of file
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/Location.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/Location.java
new file mode 100644
index 00000000000..6902583a5d5
--- /dev/null
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/Location.java
@@ -0,0 +1,28 @@
+package ch.systemsx.cisd.openbis.dss.etl.dto.api.v1;
+
+/**
+ * Auxiliary structure to store tile location on the well. The top left tile has coordinates (1,1).
+ *
+ * @author Tomasz Pylak
+ */
+public class Location
+{
+ private final int row, column;
+
+ /** Note: The top left tile has coordinates (1,1). */
+ public Location(int row, int column)
+ {
+ this.row = row;
+ this.column = column;
+ }
+
+ public int getRow()
+ {
+ return row;
+ }
+
+ public int getColumn()
+ {
+ return column;
+ }
+}
\ No newline at end of file
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/SimpleImageDataConfig.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/SimpleImageDataConfig.java
new file mode 100644
index 00000000000..bc121c90780
--- /dev/null
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/SimpleImageDataConfig.java
@@ -0,0 +1,338 @@
+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.etl.dto.api.v1;
+
+import java.util.List;
+
+import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.Geometry;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.basic.dto.ScreeningConstants;
+
+/**
+ * Allows to configure extraction of images for a plate or microscopy sample.
+ * <p>
+ * It can be used to configure the jython dropbox for image datasets easily:
+ *
+ * <pre>
+ * ch.systemsx.cisd.openbis.dss.etl.dto.api.v1 import *
+ *
+ * class MyImageDataSetConfig(SimpleImageDataConfig):
+ * """
+ * Extracts tile number, channel code and well code for a given relative path to an image.
+ * Will be called for each file found in the incoming directory which has the allowed image extension.
+ *
+ * Returns:
+ * ImageMetadata
+ * """
+ * def extractImageMetadata(self, imagePath):
+ * image = ImageMetadata()
+ * image.well = ... extract from imagePath ...
+ * image.channelCode = ... extract from imagePath ...
+ * image.tileNumber = ... extract from imagePath ...
+ * return image
+ *
+ * imageDataset = ImageDataSetFlexible()
+ * imageDataset.setRawImageDatasetType()
+ * space = "TEST"
+ * plate = incoming.getName()
+ * imageDataset.setPlate(space, plate)
+ * factory.registerImageDataset(imageDataset, incoming, service)
+ * </pre>
+ *
+ * Additionally one can define the tiles layout in the well by defining 'getTileGeometry' and
+ * 'getTileCoordinates' methods, e.g.:
+ *
+ * <pre>
+ * class MyImageDataSetConfig(SimpleImageDataConfig):
+ * ...
+ *
+ * def getTileGeometry(self, imageTokens, maxTileNumber):
+ * return Geometry.createFromRowColDimensions(maxTileNumber / 3, 3);
+ *
+ * def getTileCoordinates(self, tileNumber, tileGeometry):
+ * columns = tileGeometry.getWidth()
+ * row = ((tileNumber - 1) / columns) + 1
+ * col = ((tileNumber - 1) % columns) + 1
+ * return Location(row, col)
+ * </pre>
+ *
+ * @author Tomasz Pylak
+ */
+abstract public class SimpleImageDataConfig
+{
+ // --- methods which have to be overridden -----------------
+
+ /**
+ * Extracts tile number, channel code and well code for a given relative path to an image.
+ * <p>
+ * Will be called for each file found in the incoming directory which has the extension returned
+ * by {@link #getRecognizedImageExtensions()}.
+ * </p>
+ */
+ abstract public ImageMetadata extractImageMetadata(String imagePath);
+
+ // --- methods which can be overridden -----------------
+
+ /**
+ * By default layouts all images in one row by returning (1, maxTileNumber) geometry.Can be
+ * overridden in subclasses.
+ *
+ * @return the width and height of the matrix of tiles (a.k.a. fields or sides) in the well.
+ */
+ public Geometry getTileGeometry(List<? extends ImageMetadata> imageTokens, int maxTileNumber)
+ {
+ return Geometry.createFromRowColDimensions(1, maxTileNumber);
+ }
+
+ /**
+ * For a given tile number and tiles geometry returns (x,y) which describes where the tile is
+ * located on the well. Can be overridden in subclasses.
+ */
+ public Location getTileCoordinates(int tileNumber, Geometry tileGeometry)
+ {
+ int columns = tileGeometry.getWidth();
+ int row = ((tileNumber - 1) / columns) + 1;
+ int col = ((tileNumber - 1) % columns) + 1;
+ return new Location(row, col);
+ }
+
+ /**
+ * Creates channel description for a given code. The channel label will be equal to the code.
+ * Can be overridden in subclasses.
+ */
+ public Channel createChannel(String channelCode)
+ {
+ return new Channel(channelCode, channelCode);
+ }
+
+ // --- auxiliary structures ----------------------------------------------
+
+ private String datasetTypeCode;
+
+ private String fileFormatCode = ScreeningConstants.UNKNOWN_FILE_FORMAT;
+
+ private String plateCode;
+
+ private String spaceCode;
+
+ private boolean isMeasured = false;
+
+ private String[] recognizedImageExtensions = new String[]
+ { "tiff", "tif", "png", "gif", "jpg", "jpeg" };
+
+ private boolean generateThumbnails = false;
+
+ private int maxThumbnailWidthAndHeight = 256;
+
+ private double allowedMachineLoadDuringThumbnailsGeneration = 1.0;
+
+ private boolean storeChannelsOnExperimentLevel = false;
+
+ private OriginalDataStorageFormat originalDataStorageFormat =
+ OriginalDataStorageFormat.UNCHANGED;
+
+ // --- getters & setters ----------------------------------------------
+
+ public ImageStorageConfiguraton getImageStorageConfiguration()
+ {
+ ImageStorageConfiguraton imageStorageConfiguraton =
+ ImageStorageConfiguraton.createDefault();
+ imageStorageConfiguraton
+ .setStoreChannelsOnExperimentLevel(isStoreChannelsOnExperimentLevel());
+ imageStorageConfiguraton.setOriginalDataStorageFormat(getOriginalDataStorageFormat());
+ if (isGenerateThumbnails())
+ {
+ ThumbnailsStorageFormat thumbnailsStorageFormat = new ThumbnailsStorageFormat();
+ thumbnailsStorageFormat
+ .setAllowedMachineLoadDuringGeneration(getAllowedMachineLoadDuringThumbnailsGeneration());
+ thumbnailsStorageFormat.setMaxWidth(getMaxThumbnailWidthAndHeight());
+ thumbnailsStorageFormat.setMaxHeight(getMaxThumbnailWidthAndHeight());
+ imageStorageConfiguraton.setThumbnailsStorageFormat(thumbnailsStorageFormat);
+ }
+ return imageStorageConfiguraton;
+ }
+
+ public String getPlateSpace()
+ {
+ return spaceCode;
+ }
+
+ public String getPlateCode()
+ {
+ return plateCode;
+ }
+
+ public String[] getRecognizedImageExtensions()
+ {
+ return recognizedImageExtensions;
+ }
+
+ public boolean isGenerateThumbnails()
+ {
+ return generateThumbnails;
+ }
+
+ public int getMaxThumbnailWidthAndHeight()
+ {
+ return maxThumbnailWidthAndHeight;
+ }
+
+ public double getAllowedMachineLoadDuringThumbnailsGeneration()
+ {
+ return allowedMachineLoadDuringThumbnailsGeneration;
+ }
+
+ public boolean isStoreChannelsOnExperimentLevel()
+ {
+ return storeChannelsOnExperimentLevel;
+ }
+
+ public OriginalDataStorageFormat getOriginalDataStorageFormat()
+ {
+ return originalDataStorageFormat;
+ }
+
+ /**
+ * Sets the existing plate to which the dataset should belong.
+ *
+ * @param spaceCode space where the plate for which the dataset has been acquired exist
+ * @param plateCode code of the plate to which the dataset will belong
+ */
+ public void setPlate(String spaceCode, String plateCode)
+ {
+ this.spaceCode = spaceCode;
+ this.plateCode = plateCode;
+ }
+
+ /**
+ * Only files with these extensions will be recognized as images (e.g. ["jpg", "png"]).<br>
+ * By default it is set to [ "tiff", "tif", "png", "gif", "jpg", "jpeg" ].
+ */
+ public void setRecognizedImageExtensions(String[] recognizedImageExtensions)
+ {
+ this.recognizedImageExtensions = recognizedImageExtensions;
+ }
+
+ /** should thumbnails be generated? False by default. */
+ public void setGenerateThumbnails(boolean generateThumbnails)
+ {
+ this.generateThumbnails = generateThumbnails;
+ }
+
+ /** the maximal width and height of the generated thumbnails */
+ public void setMaxThumbnailWidthAndHeight(int maxThumbnailWidthAndHeight)
+ {
+ this.maxThumbnailWidthAndHeight = maxThumbnailWidthAndHeight;
+ }
+
+ /**
+ * Valid only if thumbnails generation is switched on. Set it to a value lower than 1 if you
+ * want only some of your processor cores to be used for thumbnails generation. Number of
+ * threads that are used for thumbnail generation will be equal to: this constant * number of
+ * processor cores.
+ */
+ public void setAllowedMachineLoadDuringThumbnailsGeneration(
+ double allowedMachineLoadDuringThumbnailsGeneration)
+ {
+ this.allowedMachineLoadDuringThumbnailsGeneration =
+ allowedMachineLoadDuringThumbnailsGeneration;
+ }
+
+ /** Should all dataset in one experiment use the same channels? By default set to false. */
+ public void setStoreChannelsOnExperimentLevel(boolean storeChannelsOnExperimentLevel)
+ {
+ this.storeChannelsOnExperimentLevel = storeChannelsOnExperimentLevel;
+ }
+
+ /**
+ * Should the original data be stored in the original form or should we pack them into one
+ * container? Available values are {@link OriginalDataStorageFormat#UNCHANGED},
+ * {@link OriginalDataStorageFormat#HDF5}, {@link OriginalDataStorageFormat#HDF5_COMPRESSED}.
+ * The default is {@link OriginalDataStorageFormat#UNCHANGED}.
+ */
+ public void setOriginalDataStorageFormat(OriginalDataStorageFormat originalDataStorageFormat)
+ {
+ this.originalDataStorageFormat = originalDataStorageFormat;
+ }
+
+ /**
+ * Sets dataset type to the one which should be used for storing raw images. Marks the dataset
+ * as a "measured" one.
+ */
+ public void setRawImageDatasetType()
+ {
+ setDataSetType(ScreeningConstants.DEFAULT_RAW_IMAGE_DATASET_TYPE);
+ setMeasuredData(true);
+ }
+
+ /**
+ * Sets dataset type to the one which should be used for storing overview images generated from
+ * raw images. Marks the dataset as a "derived" one.
+ */
+ public void setOverviewImageDatasetType()
+ {
+ setDataSetType(ScreeningConstants.DEFAULT_OVERVIEW_IMAGE_DATASET_TYPE);
+ setMeasuredData(false);
+ }
+
+ /**
+ * Sets dataset type to the one which should be used for storing overlay images. Marks the
+ * dataset as a "derived" one.
+ */
+ public void setSegmentationImageDatasetType()
+ {
+ setDataSetType(ScreeningConstants.DEFAULT_SEGMENTATION_IMAGE_DATASET_TYPE);
+ setMeasuredData(false);
+ }
+
+ // --- standard
+
+ /** Sets the type of the dataset. */
+ public void setDataSetType(String datasetTypeCode)
+ {
+ this.datasetTypeCode = datasetTypeCode;
+ }
+
+ /** Sets the file type of the dataset. */
+ public void setFileFormatType(String fileFormatCode)
+ {
+ this.fileFormatCode = fileFormatCode;
+ }
+
+ /**
+ * Set whether the data is measured or not. By default false.
+ */
+ public void setMeasuredData(boolean isMeasured)
+ {
+ this.isMeasured = isMeasured;
+ }
+
+ public String getDataSetType()
+ {
+ return datasetTypeCode;
+ }
+
+ public String getFileFormatType()
+ {
+ return fileFormatCode;
+ }
+
+ public boolean isMeasuredData()
+ {
+ return isMeasured;
+ }
+
+}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/JythonPlateDataSetHandler.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/JythonPlateDataSetHandler.java
index 49e20922bb2..765d3450d12 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/JythonPlateDataSetHandler.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/JythonPlateDataSetHandler.java
@@ -6,10 +6,14 @@ import org.python.util.PythonInterpreter;
import ch.systemsx.cisd.etlserver.TopLevelDataSetRegistratorGlobalState;
import ch.systemsx.cisd.etlserver.registrator.DataSetRegistrationDetails;
+import ch.systemsx.cisd.etlserver.registrator.DataSetRegistrationService;
import ch.systemsx.cisd.etlserver.registrator.IDataSetRegistrationDetailsFactory;
import ch.systemsx.cisd.etlserver.registrator.JythonTopLevelDataSetHandler;
+import ch.systemsx.cisd.etlserver.registrator.api.v1.IDataSet;
import ch.systemsx.cisd.etlserver.registrator.api.v1.impl.DataSet;
+import ch.systemsx.cisd.etlserver.registrator.api.v1.impl.DataSetRegistrationTransaction;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.BasicDataSetInformation;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.SimpleImageDataConfig;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageDataSetInformation;
/**
@@ -18,7 +22,7 @@ import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageDataSetInformation;
* @author Tomasz Pylak
*/
public class JythonPlateDataSetHandler extends
- JythonTopLevelDataSetHandler<BasicDataSetInformation>
+ JythonTopLevelDataSetHandler<ImageDataSetInformation>
{
public JythonPlateDataSetHandler(TopLevelDataSetRegistratorGlobalState globalState)
{
@@ -29,14 +33,14 @@ public class JythonPlateDataSetHandler extends
* Create a screening specific factory available to the python script.
*/
@Override
- protected IDataSetRegistrationDetailsFactory<BasicDataSetInformation> createObjectFactory(
+ protected IDataSetRegistrationDetailsFactory<ImageDataSetInformation> createObjectFactory(
PythonInterpreter interpreter)
{
return new JythonPlateDatasetFactory(getRegistratorState());
}
public static class JythonPlateDatasetFactory extends
- JythonObjectFactory<BasicDataSetInformation>
+ JythonObjectFactory<ImageDataSetInformation>
{
public JythonPlateDatasetFactory(OmniscientTopLevelDataSetRegistratorState registratorState)
{
@@ -44,17 +48,17 @@ public class JythonPlateDataSetHandler extends
}
@Override
- public DataSet<BasicDataSetInformation> createDataSet(
- DataSetRegistrationDetails<BasicDataSetInformation> registrationDetails,
+ public DataSet<ImageDataSetInformation> createDataSet(
+ DataSetRegistrationDetails<ImageDataSetInformation> registrationDetails,
File stagingFile)
{
- return new DataSet<BasicDataSetInformation>(registrationDetails, stagingFile);
+ return new DataSet<ImageDataSetInformation>(registrationDetails, stagingFile);
}
@Override
- protected BasicDataSetInformation createDataSetInformation()
+ protected ImageDataSetInformation createDataSetInformation()
{
- return new BasicDataSetInformation();
+ return new ImageDataSetInformation();
}
/**
@@ -82,5 +86,30 @@ public class JythonPlateDataSetHandler extends
registrationDetails.setDataSetInformation(dataSetInfo);
return registrationDetails;
}
+
+ public DataSetRegistrationDetails<ImageDataSetInformation> createImageRegistrationDetails(
+ SimpleImageDataConfig imageDataSet, File incomingDatasetFolder)
+ {
+ return SimpleImageDataSetRegistrator.createImageDatasetDetails(imageDataSet,
+ incomingDatasetFolder, this);
+ }
+
+ /** a simple method to register the described image dataset */
+ public void registerImageDataset(SimpleImageDataConfig imageDataSet,
+ File incomingDatasetFolder,
+ DataSetRegistrationService<ImageDataSetInformation> service)
+ {
+ DataSetRegistrationDetails<ImageDataSetInformation> imageDatasetDetails =
+ createImageRegistrationDetails(imageDataSet, incomingDatasetFolder);
+ IDataSetRegistrationDetailsFactory<ImageDataSetInformation> myself = this;
+ // TODO 2011-02-15, Tomasz Pylak: remove this casting
+ @SuppressWarnings("unchecked")
+ DataSetRegistrationTransaction<ImageDataSetInformation> transaction =
+ (DataSetRegistrationTransaction<ImageDataSetInformation>) service.transaction(
+ incomingDatasetFolder, myself);
+ IDataSet newDataset = transaction.createNewDataSet(imageDatasetDetails);
+ transaction.moveFile(incomingDatasetFolder.getPath(), newDataset);
+ transaction.commit();
+ }
}
}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/SimpleImageDataSetRegistrator.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/SimpleImageDataSetRegistrator.java
new file mode 100644
index 00000000000..365d4d1a5eb
--- /dev/null
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/SimpleImageDataSetRegistrator.java
@@ -0,0 +1,217 @@
+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.etl.jython;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import ch.systemsx.cisd.common.filesystem.FileOperations;
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
+import ch.systemsx.cisd.etlserver.registrator.DataSetRegistrationDetails;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.Channel;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.SimpleImageDataConfig;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageDataSetInformation;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageFileInfo;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageMetadata;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.Location;
+import ch.systemsx.cisd.openbis.dss.etl.jython.JythonPlateDataSetHandler.JythonPlateDatasetFactory;
+import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetType;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.FileFormatType;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.Geometry;
+
+/**
+ * Allows to prepare the image dataset which should be registered easily using the specified
+ * {@link SimpleImageDataConfig}.
+ *
+ * @author Tomasz Pylak
+ */
+public class SimpleImageDataSetRegistrator
+{
+ private static class ImageTokensWithPath extends ImageMetadata
+ {
+ /** path relative to the incoming dataset directory */
+ private String imageRelativePath;
+
+ public ImageTokensWithPath(ImageMetadata imageTokens, String imageRelativePath)
+ {
+ setWell(imageTokens.getWell());
+ setChannelCode(imageTokens.getChannelCode());
+ setTileNumber(imageTokens.getTileNumber());
+ this.imageRelativePath = imageRelativePath;
+ }
+
+ public String getImagePath()
+ {
+ return imageRelativePath;
+ }
+ }
+
+ public static DataSetRegistrationDetails<ImageDataSetInformation> createImageDatasetDetails(
+ SimpleImageDataConfig imageDataSet, File incoming, JythonPlateDatasetFactory factory)
+ {
+ return new SimpleImageDataSetRegistrator(imageDataSet).createImageDatasetDetails(incoming,
+ factory);
+ }
+
+ private final SimpleImageDataConfig imageDataSet;
+
+ private SimpleImageDataSetRegistrator(SimpleImageDataConfig imageDataSet)
+ {
+ this.imageDataSet = imageDataSet;
+ }
+
+ private DataSetRegistrationDetails<ImageDataSetInformation> createImageDatasetDetails(
+ File incoming, JythonPlateDatasetFactory factory)
+ {
+ DataSetRegistrationDetails<ImageDataSetInformation> registrationDetails =
+ factory.createImageRegistrationDetails();
+ ImageDataSetInformation imageDataset = registrationDetails.getDataSetInformation();
+ setImageDataset(incoming, imageDataset);
+ setRegistrationDetails(registrationDetails, imageDataset);
+ return registrationDetails;
+ }
+
+ /**
+ * Finds all images in the directory.
+ */
+ private List<File> listImageFiles(final File incomingDirectory)
+ {
+ return FileOperations.getInstance().listFiles(incomingDirectory,
+ imageDataSet.getRecognizedImageExtensions(), true);
+ }
+
+ /**
+ * Tokenizes file names of all images in the directory.
+ */
+ protected List<ImageTokensWithPath> parseImageTokens(File incomingDirectory)
+ {
+ List<ImageTokensWithPath> imageTokensList = new ArrayList<ImageTokensWithPath>();
+ List<File> imageFiles = listImageFiles(incomingDirectory);
+ for (File imageFile : imageFiles)
+ {
+ String imageRelativePath =
+ FileUtilities.getRelativeFile(incomingDirectory, new File(imageFile.getPath()));
+ ImageMetadata imageTokens = imageDataSet.extractImageMetadata(imageRelativePath);
+ imageTokens.ensureValid();
+ imageTokensList.add(new ImageTokensWithPath(imageTokens, imageRelativePath));
+ }
+ return imageTokensList;
+ }
+
+ /**
+ * Creates ImageFileInfo for a given path to an image.
+ */
+ protected ImageFileInfo createImageInfo(ImageTokensWithPath imageTokens, Geometry tileGeometry)
+ {
+ Location tileCoords =
+ imageDataSet.getTileCoordinates(imageTokens.getTileNumber(), tileGeometry);
+ ImageFileInfo img =
+ new ImageFileInfo(imageTokens.getChannelCode(), tileCoords.getRow(),
+ tileCoords.getColumn(), imageTokens.getImagePath());
+ img.setTimepoint(imageTokens.tryGetTimepoint());
+ img.setDepth(imageTokens.tryGetDepth());
+ img.setSeriesNumber(imageTokens.tryGetSeriesNumber());
+ img.setWell(imageTokens.getWell());
+ return img;
+ }
+
+ /**
+ * @param imageTokensList list of ImageTokens for each image
+ * @param tileGeometry describes the matrix of tiles (aka fields or sides) in the well
+ */
+ protected List<ImageFileInfo> createImageInfos(List<ImageTokensWithPath> imageTokensList,
+ Geometry tileGeometry)
+ {
+ List<ImageFileInfo> images = new ArrayList<ImageFileInfo>();
+ for (ImageTokensWithPath imageTokens : imageTokensList)
+ {
+ ImageFileInfo image = createImageInfo(imageTokens, tileGeometry);
+ images.add(image);
+ }
+ return images;
+ }
+
+ private List<Channel> getAvailableChannels(List<ImageFileInfo> images)
+ {
+ Set<String> channelCodes = new HashSet<String>();
+ for (ImageFileInfo image : images)
+ {
+ channelCodes.add(image.getChannelCode());
+ }
+ List<Channel> channels = new ArrayList<Channel>();
+ for (String channelCode : channelCodes)
+ {
+ channels.add(imageDataSet.createChannel(channelCode));
+ }
+ return channels;
+ }
+
+ private static int getMaxTileNumber(List<ImageTokensWithPath> imageTokensList)
+ {
+ int max = 0;
+ for (ImageMetadata imageTokens : imageTokensList)
+ {
+ max = Math.max(max, imageTokens.getTileNumber());
+ }
+ return max;
+ }
+
+ /**
+ * Extracts all images from the incoming directory.
+ *
+ * @param incoming - folder with images
+ * @param dataset - here the result will be stored
+ */
+ protected void setImageDataset(File incoming, ImageDataSetInformation dataset)
+ {
+ dataset.setDatasetTypeCode(imageDataSet.getDataSetType());
+ dataset.setFileFormatCode(imageDataSet.getFileFormatType());
+ dataset.setMeasured(imageDataSet.isMeasuredData());
+
+ String sampleCode = imageDataSet.getPlateCode();
+ String spaceCode = imageDataSet.getPlateSpace();
+ dataset.setSample(spaceCode, sampleCode);
+ dataset.setMeasured(true);
+
+ List<ImageTokensWithPath> imageTokensList = parseImageTokens(incoming);
+ int maxTileNumber = getMaxTileNumber(imageTokensList);
+ Geometry tileGeometry = imageDataSet.getTileGeometry(imageTokensList, maxTileNumber);
+ List<ImageFileInfo> images = createImageInfos(imageTokensList, tileGeometry);
+ List<Channel> channels = getAvailableChannels(images);
+
+ dataset.setImages(images);
+ dataset.setChannels(channels);
+ dataset.setTileGeometry(tileGeometry.getNumberOfRows(), tileGeometry.getNumberOfColumns());
+
+ dataset.setImageStorageConfiguraton(imageDataSet.getImageStorageConfiguration());
+ }
+
+ private <T extends DataSetInformation> void setRegistrationDetails(
+ DataSetRegistrationDetails<T> registrationDetails, T dataset)
+ {
+ registrationDetails.setDataSetInformation(dataset);
+ registrationDetails.setFileFormatType(new FileFormatType(imageDataSet.getFileFormatType()));
+ registrationDetails.setDataSetType(new DataSetType(imageDataSet.getDataSetType()));
+ registrationDetails.setMeasuredData(imageDataSet.isMeasuredData());
+
+ }
+
+}
--
GitLab