diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/ComputeIntensityLevelTransformationsMaintenanceTask.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/ComputeIntensityLevelTransformationsMaintenanceTask.java
new file mode 100644
index 0000000000000000000000000000000000000000..2ac1cb877d046b7267b09a38cef5aa30af64ebcf
--- /dev/null
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/ComputeIntensityLevelTransformationsMaintenanceTask.java
@@ -0,0 +1,338 @@
+/*
+ * Copyright 2012 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.etl;
+
+import java.io.File;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Properties;
+
+import javax.sql.DataSource;
+
+import net.lemnik.eodsql.QueryTool;
+
+import org.apache.log4j.Logger;
+
+import ch.systemsx.cisd.base.image.IImageTransformerFactory;
+import ch.systemsx.cisd.common.collections.ExtendedBlockingQueueFactory;
+import ch.systemsx.cisd.common.collections.PersistentExtendedBlockingQueueDecorator;
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.image.IntensityRescaling.Levels;
+import ch.systemsx.cisd.common.io.hierarchical_content.api.IHierarchicalContent;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.maintenance.IMaintenanceTask;
+import ch.systemsx.cisd.common.utilities.PropertyUtils;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.transformations.ImageTransformation;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.transformations.ImageTransformationBuffer;
+import ch.systemsx.cisd.openbis.dss.etl.jython.SimpleImageDataSetRegistrator;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ExternalData;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.IImagingTransformerDAO;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgChannelDTO;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgImageDatasetDTO;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgImageEnrichedDTO;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgImageTransformationDTO;
+
+/**
+ * @author Jakub Straszewski
+ */
+public class ComputeIntensityLevelTransformationsMaintenanceTask implements IMaintenanceTask
+{
+
+ protected static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
+ ComputeIntensityLevelTransformationsMaintenanceTask.class);
+
+ /*
+ * Settings
+ */
+
+ private static final String COMPUTE_MIN_MAX_LEVELS_KEY = "compute-min-max-levels";
+
+ private static final String DEFAULT_COMPUTE_MIN_MAX_LEVELS = "true";
+
+ private static final String MIN_LEVEL_KEY = "min-level";
+
+ private static final String MAX_LEVEL_KEY = "max-level";
+
+ private static final String INTENSITY_THRESHOLD_KEY = "intensity-threshold";
+
+ private static final String TRASNFORMATION_CODE_KEY = "transformation-code";
+
+ private static final String TRASNFORMATION_LABEL_KEY = "transformation-label";
+
+ private static final String TRANSFORMATION_DESC_KEY = "transformation-description";
+
+ private static final String IS_DEFAULT_TRANSFORMATION_KEY = "is-default-transformation";
+
+ private static final String STATUS_FILENAME_KEY = "status-filename";
+
+ private boolean computeMinMaxLevels;
+
+ private int minLevel;
+
+ private int maxLevel;
+
+ private float intensityThreshold;
+
+ private String transformationCode;
+
+ private String transformationLabel;
+
+ private String transformationDescription;
+
+ private boolean isDefaultTransformation;
+
+ private File queueFile;
+
+ /*
+ * Resources
+ */
+ private DataSource dataSource;
+
+ private IImagingTransformerDAO dao;
+
+ private IEncapsulatedOpenBISService service;
+
+ private IHierarchicalContentProvider contentProvider;
+
+ @Override
+ public void setUp(String pluginName, Properties properties)
+ {
+ this.dataSource = ServiceProvider.getDataSourceProvider().getDataSource(properties);
+
+ dao = QueryTool.getQuery(dataSource, IImagingTransformerDAO.class);
+ dao.listSpots(0L); // testing correct database set up
+ service = ServiceProvider.getOpenBISService();
+ contentProvider = ServiceProvider.getHierarchicalContentProvider();
+
+ computeMinMaxLevels =
+ Boolean.valueOf(properties.getProperty(COMPUTE_MIN_MAX_LEVELS_KEY,
+ DEFAULT_COMPUTE_MIN_MAX_LEVELS));
+
+ minLevel = Integer.valueOf(properties.getProperty(MIN_LEVEL_KEY, "-1"));
+ maxLevel = Integer.valueOf(properties.getProperty(MAX_LEVEL_KEY, "-1"));
+
+ if (false == computeMinMaxLevels)
+ {
+ if (minLevel == -1 || maxLevel == -1)
+ {
+ throw new ConfigurationFailureException(
+ "The intensity ranges levels must be correctly defined if calculating them is disabled.");
+ }
+ }
+
+ intensityThreshold = Float.valueOf(properties.getProperty(INTENSITY_THRESHOLD_KEY, "-1"));
+ if (computeMinMaxLevels && (intensityThreshold < 0 || intensityThreshold > 1))
+ {
+ throw new ConfigurationFailureException(
+ "The "
+ + INTENSITY_THRESHOLD_KEY
+ + " property must be provided as a real number between 0 and 1 if the levels are calculated.");
+ }
+
+ transformationCode =
+ PropertyUtils.getMandatoryProperty(properties, TRASNFORMATION_CODE_KEY);
+
+ transformationLabel = properties.getProperty(TRASNFORMATION_LABEL_KEY, "Fixed rescaling");
+
+ transformationDescription =
+ properties
+ .getProperty(
+ TRANSFORMATION_DESC_KEY,
+ SimpleImageDataSetRegistrator.OPTIMAL_DATASET_INTENSITY_RESCALING_DESCRIPTION);
+
+ String isDefaultTransformationString =
+ PropertyUtils.getMandatoryProperty(properties, IS_DEFAULT_TRANSFORMATION_KEY);
+
+ isDefaultTransformation = Boolean.valueOf(isDefaultTransformationString);
+
+ String queueFilePath = PropertyUtils.getMandatoryProperty(properties, STATUS_FILENAME_KEY);
+ queueFile = new File(queueFilePath);
+ }
+
+ /**
+ * method overriden to create an underlying database connection.
+ */
+ @Override
+ public void execute()
+ {
+ PersistentExtendedBlockingQueueDecorator<Long> queue =
+ ExtendedBlockingQueueFactory.<Long> createSmartPersist(queueFile);
+
+ try
+ {
+ executeTransformations(queue);
+ } catch (Exception e)
+ {
+ operationLog.error(e);
+ }
+ queue.close();
+ }
+
+ private void executeTransformations(PersistentExtendedBlockingQueueDecorator<Long> queue)
+ {
+ long lastSeenId = getLastSeenIDFromQueue(queue);
+
+ Long nextId = dao.tryGetNextDatasetId(lastSeenId);
+
+ // if this value is null, it means there are no newer datasets
+ if (nextId != null)
+ {
+ if (false == dao.hasDatasetDefinedTransformation(nextId, transformationCode))
+ {
+ executeTransformations(nextId);
+ }
+ storeLastSeenInTheQueue(queue, nextId);
+ }
+ }
+
+ private void executeTransformations(long datasetId)
+ {
+ ImgImageDatasetDTO imageDataset = dao.tryGetImageDatasetById(datasetId);
+ List<ImgChannelDTO> channels = dao.getChannelsByDatasetId(datasetId);
+
+ operationLog.info("Will create transformation " + transformationCode + " for the dataset "
+ + imageDataset.getPermId());
+
+ final Levels intensityRange;
+ if (computeMinMaxLevels)
+ {
+ intensityRange = computeIntensityRange(imageDataset, channels);
+ if (intensityRange != null)
+ {
+ operationLog.info("Calculated intensity ranges for dataset "
+ + imageDataset.getPermId() + ". " + intensityRange);
+ }
+ } else
+ {
+ intensityRange = new Levels(minLevel, maxLevel);
+ operationLog.info("Using default intensity range values " + intensityRange);
+ }
+
+ // if intensity ranges are null, it means that this transformation cannot be applied to this
+ // dataset. Thus we do nothing
+ if (intensityRange != null)
+ {
+ IImageTransformerFactory factory = createImageTransformerFactory(intensityRange);
+
+ for (ImgChannelDTO ch : channels)
+ {
+ // execute inserts on dao, but without a commit
+ addCommonIntensityRangeTransformation(ch, factory);
+ }
+
+ // commits the dao transaction
+ dao.commit();
+ operationLog.info("Succesfully applied intensity ranges transformation to "
+ + imageDataset.getPermId() + " on " + channels.size() + " channels.");
+ } else
+ {
+ operationLog.warn("Could not calculate intensity ranges for dataset "
+ + imageDataset.getPermId() + ". Will be skipped.");
+ }
+ }
+
+ private IImageTransformerFactory createImageTransformerFactory(final Levels intensityRange)
+ {
+ ImageTransformationBuffer buffer = new ImageTransformationBuffer();
+
+ ImageTransformation imageTransformation =
+ buffer.appendRescaleGrayscaleIntensity(intensityRange.getMinLevel(),
+ intensityRange.getMaxLevel(), "N/A");
+
+ IImageTransformerFactory factory = imageTransformation.getImageTransformerFactory();
+ return factory;
+ }
+
+ /**
+ * Computes intensity ranges for all images in the dataset.
+ *
+ * @return calculated levels, or null if calculation cannot be done because of a permanent
+ * problem
+ */
+ private Levels computeIntensityRange(ImgImageDatasetDTO imageDataset,
+ List<ImgChannelDTO> channels)
+ {
+ ExternalData dataset = service.tryGetDataSet(imageDataset.getPermId());
+
+ if (dataset == null)
+ {
+ operationLog
+ .warn("The dataset specified in the image_data_set doesn't exist in openbis.");
+ return null;
+ }
+
+ IHierarchicalContent content = contentProvider.asContent(dataset);
+
+ final List<File> imagePaths = new LinkedList<File>();
+
+ for (ImgChannelDTO ch : channels)
+ {
+ List<ImgImageEnrichedDTO> images =
+ dao.listHCSImages(imageDataset.getPermId(), ch.getCode());
+
+ for (ImgImageEnrichedDTO img : images)
+ {
+ File imageFile = content.getNode(img.getFilePath()).getFile();
+ imagePaths.add(imageFile);
+ if (false == imageFile.exists())
+ {
+ throw new IllegalStateException("The image file for dataset "
+ + imageDataset.getPermId() + " doesn't exist.");
+ }
+ }
+ }
+
+ return SimpleImageDataSetRegistrator.tryComputeCommonIntensityRange(null, imagePaths,
+ intensityThreshold);
+ }
+
+ private void addCommonIntensityRangeTransformation(ImgChannelDTO channel,
+ IImageTransformerFactory factory)
+ {
+ ImgImageTransformationDTO dto =
+ new ImgImageTransformationDTO(transformationCode, transformationLabel,
+ transformationDescription, isDefaultTransformation, channel.getId(),
+ factory, false);
+
+ dao.addImageTransformation(dto);
+ }
+
+ private void storeLastSeenInTheQueue(PersistentExtendedBlockingQueueDecorator<Long> queue,
+ long lastSeenId)
+ {
+ queue.offer(lastSeenId);
+ while (queue.size() > 1)
+ {
+ queue.remove();
+ }
+ }
+
+ private long getLastSeenIDFromQueue(PersistentExtendedBlockingQueueDecorator<Long> queue)
+ {
+ long lastSeenId = -1;
+ if (queue.size() > 0)
+ {
+ lastSeenId = queue.peek();
+ }
+ return lastSeenId;
+ }
+
+}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/transformations/ImageTransformationBuffer.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/transformations/ImageTransformationBuffer.java
index d23e534f0288f61efd2390560dc19f20297dedf5..8e1db0f4b78bd68fb966c4cd3f61750a9267b0c0 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/transformations/ImageTransformationBuffer.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/transformations/ImageTransformationBuffer.java
@@ -240,7 +240,7 @@ public class ImageTransformationBuffer
* @param userFriendlyLabelOrNull label of the transformation. If null a default label is
* assigned.
*/
- private static ImageTransformation createRescaleGrayscaleIntensity(int blackPointIntensity,
+ public static ImageTransformation createRescaleGrayscaleIntensity(int blackPointIntensity,
int whitePointIntensity, String userFriendlyLabelOrNull)
{
if (blackPointIntensity > whitePointIntensity || blackPointIntensity < 0
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/SimpleImageDataSetRegistrator.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/SimpleImageDataSetRegistrator.java
index 856940a71e2b745f1b38d2799cd4792fb0eaf85c..ae8afdae4bb103431990328e5d0bf838da64176b 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/SimpleImageDataSetRegistrator.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/SimpleImageDataSetRegistrator.java
@@ -75,7 +75,7 @@ import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.Geometry;
*/
public class SimpleImageDataSetRegistrator
{
- private static final String OPTIMAL_DATASET_INTENSITY_RESCALING_DESCRIPTION =
+ public static final String OPTIMAL_DATASET_INTENSITY_RESCALING_DESCRIPTION =
"Optimal intensity rescaling for a series of images. "
+ "It allows to compare images of one plate's dataset to each other."
+ "At the same time it causes that the conversion to 8 bit color depth looses less information, "
@@ -556,7 +556,13 @@ public class SimpleImageDataSetRegistrator
return normalizedCodes;
}
- private static Levels tryComputeCommonIntensityRange(IImageReader readerOrNull,
+ /**
+ * Computes common intensity range for a list of files.
+ *
+ * @return calculated levels or null if calculation couldn't succeed because some images where
+ * not in gray scale
+ */
+ public static Levels tryComputeCommonIntensityRange(IImageReader readerOrNull,
List<File> imageFiles, float threshold)
{
String libraryName = (readerOrNull == null) ? null : readerOrNull.getLibraryName();
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/imaging/dataaccess/IImagingReadonlyQueryDAO.java b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/imaging/dataaccess/IImagingReadonlyQueryDAO.java
index f6eec6d11ca3783ecb1aa3fe045e66c4d62f89a6..3b2b6f18ed7b958ad9e6dc62a75d2d963483d211 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/imaging/dataaccess/IImagingReadonlyQueryDAO.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/imaging/dataaccess/IImagingReadonlyQueryDAO.java
@@ -238,6 +238,15 @@ public interface IImagingReadonlyQueryDAO extends BaseQuery
@Select("select * from IMAGE_DATA_SETS where PERM_ID = ?{1}")
public ImgImageDatasetDTO tryGetImageDatasetByPermId(String datasetPermId);
+ @Select("select * from IMAGE_DATA_SETS where ID = ?{1}")
+ public ImgImageDatasetDTO tryGetImageDatasetById(long datasetId);
+
+ @Select("select id from IMAGE_DATA_SETS where ID > ?{1} order by id limit 1")
+ public Long tryGetNextDatasetId(long datasetId);
+
+ @Select("select count(*) > 0 from channels ch, image_transformations it where ch.ds_id = ?{1} and it.channel_id = ch.id and it.code = upper(?{2})")
+ public boolean hasDatasetDefinedTransformation(long datasetId, String transformationCode);
+
@Select("select * from ANALYSIS_DATA_SETS where PERM_ID = ?{1}")
public ImgAnalysisDatasetDTO tryGetAnalysisDatasetByPermId(String datasetPermId);