From d1ec132f159a1c18a58888c25213b4abfa998c99 Mon Sep 17 00:00:00 2001
From: tpylak <tpylak>
Date: Tue, 11 Jan 2011 13:42:38 +0000
Subject: [PATCH] LMS-1965 channels optionally on dataset level in HCS,
handling a case when some overlays are missing LMS-1983 allow image file to
be placed in the dropbox as a zip archive
SVN: 19362
---
.../etlserver/AbstractStorageProcessor.java | 30 +----
.../etlserver/DefaultStorageProcessor.java | 26 +++++
.../SmartParentDataSetInfoExtractor.java | 4 +-
screening/dist/etc/service.properties | 19 +++-
.../dss/etl/AbstractImageFileExtractor.java | 83 +++++++++++++-
.../etl/AbstractImageStorageProcessor.java | 103 +++++-------------
.../openbis/dss/etl/HCSImageDatasetInfo.java | 11 +-
.../dss/etl/HCSImageDatasetUploader.java | 23 +++-
.../dss/etl/HCSImageFileExtractor.java | 8 +-
...roscopyBlackboxSeriesStorageProcessor.java | 2 +-
.../dss/etl/PlateStorageProcessor.java | 27 ++++-
.../etl/dataaccess/ImagingDatasetLoader.java | 15 ++-
.../server/images/ImageChannelsUtils.java | 52 +++++----
.../shared/imaging/HCSDatasetLoader.java | 9 +-
14 files changed, 252 insertions(+), 160 deletions(-)
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/AbstractStorageProcessor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/AbstractStorageProcessor.java
index 6ecca0f1ca3..d50bae5825c 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/AbstractStorageProcessor.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/AbstractStorageProcessor.java
@@ -21,9 +21,7 @@ import java.util.Properties;
import org.apache.commons.io.FilenameUtils;
-import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.utilities.PropertyUtils;
-import ch.systemsx.cisd.etlserver.utils.Unzipper;
/**
* An <code>abtract</code> implementation of <code>IStorageProcessor</code>.
@@ -32,26 +30,16 @@ import ch.systemsx.cisd.etlserver.utils.Unzipper;
*/
public abstract class AbstractStorageProcessor implements IStorageProcessor
{
- protected final Properties properties;
-
- private File storeRootDir;
-
- static final String UNZIP_CRITERIA_KEY = "unzip";
-
- static final String DELETE_UNZIPPED_KEY = "delete_unzipped";
-
private static final String[] ZIP_FILE_EXTENSIONS =
{ "zip" };
- private final boolean unzip;
+ protected final Properties properties;
- private final boolean deleteUnzipped;
+ private File storeRootDir;
protected AbstractStorageProcessor(final Properties properties)
{
this.properties = properties;
- unzip = PropertyUtils.getBoolean(properties, UNZIP_CRITERIA_KEY, false);
- deleteUnzipped = PropertyUtils.getBoolean(properties, DELETE_UNZIPPED_KEY, true);
}
protected final String getMandatoryProperty(final String propertyKey)
@@ -85,19 +73,7 @@ public abstract class AbstractStorageProcessor implements IStorageProcessor
// do nothing
}
- /**
- * Unzips given archive file to selected output directory.
- */
- protected final Status unzipIfMatching(File archiveFile, File outputDirectory)
- {
- if (unzip && isZipFile(archiveFile))
- {
- return Unzipper.unzip(archiveFile, outputDirectory, deleteUnzipped);
- }
- return Status.OK;
- }
-
- public static boolean isZipFile(File file)
+ protected static boolean isZipFile(File file)
{
if (file.isDirectory())
{
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/DefaultStorageProcessor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/DefaultStorageProcessor.java
index 672425e8f0f..01a0c656702 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/DefaultStorageProcessor.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/DefaultStorageProcessor.java
@@ -21,8 +21,11 @@ import java.util.List;
import java.util.Properties;
import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.common.mail.IMailClient;
+import ch.systemsx.cisd.common.utilities.PropertyUtils;
+import ch.systemsx.cisd.etlserver.utils.Unzipper;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
import ch.systemsx.cisd.openbis.generic.shared.dto.StorageFormat;
@@ -38,9 +41,20 @@ public class DefaultStorageProcessor extends AbstractStorageProcessor
static final String NO_RENAME = "Couldn't rename '%s' to '%s'.";
+ static final String UNZIP_CRITERIA_KEY = "unzip";
+
+ static final String DELETE_UNZIPPED_KEY = "delete_unzipped";
+
+ private final boolean unzip;
+
+ private final boolean deleteUnzipped;
+
public DefaultStorageProcessor(final Properties properties)
{
super(properties);
+
+ unzip = PropertyUtils.getBoolean(properties, UNZIP_CRITERIA_KEY, false);
+ deleteUnzipped = PropertyUtils.getBoolean(properties, DELETE_UNZIPPED_KEY, true);
}
//
@@ -106,4 +120,16 @@ public class DefaultStorageProcessor extends AbstractStorageProcessor
}
return files.get(0);
}
+
+ /**
+ * Unzips given archive file to selected output directory.
+ */
+ protected final Status unzipIfMatching(File archiveFile, File outputDirectory)
+ {
+ if (unzip && isZipFile(archiveFile))
+ {
+ return Unzipper.unzip(archiveFile, outputDirectory, deleteUnzipped);
+ }
+ return Status.OK;
+ }
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/SmartParentDataSetInfoExtractor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/SmartParentDataSetInfoExtractor.java
index 0af00d97530..6fa972b2e94 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/SmartParentDataSetInfoExtractor.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/SmartParentDataSetInfoExtractor.java
@@ -177,7 +177,9 @@ public class SmartParentDataSetInfoExtractor extends DefaultDataSetInfoExtractor
if (failWhenParentMissing)
{
throw UserFailureException
- .fromTemplate("No parent datasets of the type '%s' connected to the same sample/experiment could be found.");
+ .fromTemplate(
+ "No parent datasets of the type '%s' connected to the same sample/experiment could be found.",
+ datasetTypePatternOrNull);
} else
{
return new ArrayList<String>(); // no parents will be set
diff --git a/screening/dist/etc/service.properties b/screening/dist/etc/service.properties
index 0fa1001749a..38db9f1686d 100644
--- a/screening/dist/etc/service.properties
+++ b/screening/dist/etc/service.properties
@@ -213,6 +213,7 @@ merged-channels-images.type-extractor.locator-type = RELATIVE_LOCATION
merged-channels-images.type-extractor.data-set-type = HCS_IMAGE
merged-channels-images.type-extractor.is-measured = true
+# Note: this storage processor is able to process folders compressed with zip as well
merged-channels-images.storage-processor = ch.systemsx.cisd.openbis.dss.etl.PlateStorageProcessor
# How should the original data be stored? Possible values:
# unchanged - nothing is changed, the default
@@ -226,12 +227,22 @@ merged-channels-images.storage-processor.generate-thumbnails = false
# Thumbnails size in pixels
# merged-channels-images.storage-processor.thumbnail-max-width = 300
# merged-channels-images.storage-processor.thumbnail-max-height = 200
-# DEPRECATED: use 'channel-codes' and 'channel-labels' instead
-#merged-channels-images.storage-processor.channel-names = gfp, dapi
-# Codes of the channels in which images have been acquired. Allowed characters: [A-Z0-9_]. Number and order of entries must be consistent with 'channel-labels'.
+# Codes of the channels in which images have been acquired. Allowed characters: [A-Z0-9_-].
+# Number and order of entries must be consistent with 'channel-labels'.
merged-channels-images.storage-processor.channel-codes = GFP, DAPI
-# Labels of the channels in which images have been acquired. Number and order of entries must be consistent with 'channel-codes'.
+# Labels of the channels in which images have been acquired.
+# Number and order of entries must be consistent with 'channel-codes'.
merged-channels-images.storage-processor.channel-labels = Gfp, Dapi
+
+# Optional, true by default.
+# Set to false to allow datasets in one experiment to use different channels.
+# In this case 'channel-codes' and 'channel-labels' become optional and are used only to determine the label for each channel code.
+# It should be set to 'false' for overlay image datasets.
+#merged-channels-images.storage-processor.define-channels-per-experiment = false
+
+# This is an optional boolean property which defines if all image datasets in one experiment have the same
+# channels or if each imported dataset can have different channels. By default true if not specified.
+#merged-channels-images.storage-processor.define-channels-per-experiment = false
# Format: [width]>x[height], e.g. 3x4. Specifies the grid into which a microscope divided the well to acquire images.
merged-channels-images.storage-processor.well_geometry = 3x3
# implementation of the IHCSImageFileExtractor interface which maps images to the location on the plate and particular channel
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/AbstractImageFileExtractor.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/AbstractImageFileExtractor.java
index 28e69d57882..1774b2ed918 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/AbstractImageFileExtractor.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/AbstractImageFileExtractor.java
@@ -18,6 +18,7 @@ package ch.systemsx.cisd.openbis.dss.etl;
import java.io.File;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -82,6 +83,18 @@ abstract public class AbstractImageFileExtractor implements IImageFileExtractor
protected static final String IMAGE_FILE_ACCEPTED = "Image file '%s' was accepted: %s.";
+ // --------
+
+ // comma separated list of channel names, order matters
+ @Deprecated
+ public static final String CHANNEL_NAMES = "channel-names";
+
+ // comma separated list of channel codes, order matters
+ public static final String CHANNEL_CODES = "channel-codes";
+
+ // comma separated list of channel labels, order matters
+ public static final String CHANNEL_LABELS = "channel-labels";
+
// optional, list of the color components (RED, GREEN, BLUE), in the same order as channel names
protected static final String EXTRACT_SINGLE_IMAGE_CHANNELS_PROPERTY =
"extract-single-image-channels";
@@ -104,17 +117,17 @@ abstract public class AbstractImageFileExtractor implements IImageFileExtractor
protected AbstractImageFileExtractor(Properties properties, boolean skipChannelsWithoutImages)
{
- this(extractChannelDescriptions(properties), getWellGeometry(properties),
+ this(tryExtractChannelDescriptions(properties), getWellGeometry(properties),
skipChannelsWithoutImages, properties);
}
/**
* @param skipChannelsWithoutImages if true channel names are derived from a set of channel
* codes in the extracted images. In this way we do not restrict available channel
- * codes and we have no channels without images. Channel labels are taken from
+ * codes and each channel has at least one image. Channel labels are taken from
* channel descriptions anyway. Should be set to true only for microscopy, in HCS
* each dataset of one experiment should have the same set of channels even if they
- * are not present in some datasets.
+ * are not present in some datasets (exceptions: image overlays, test screens).
*/
protected AbstractImageFileExtractor(List<ChannelDescription> channelDescriptionsOrNull,
Geometry wellGeometry, boolean skipChannelsWithoutImages, Properties properties)
@@ -165,7 +178,7 @@ abstract public class AbstractImageFileExtractor implements IImageFileExtractor
if (skipChannelsWithoutImages == false && channelDescriptionsOrNull == null)
{
throw ConfigurationFailureException
- .fromTemplate("Channel names are not specified and extraction of channels from images is switched off!");
+ .fromTemplate("Expected channels are not specified and extraction of channels from images is switched off!");
}
if (channelColorComponentsOrNull != null && channelDescriptionsOrNull == null)
{
@@ -352,7 +365,67 @@ abstract public class AbstractImageFileExtractor implements IImageFileExtractor
protected final static List<ChannelDescription> extractChannelDescriptions(
final Properties properties)
{
- return PlateStorageProcessor.extractChannelDescriptions(properties);
+ List<ChannelDescription> channelDescriptions = tryExtractChannelDescriptions(properties);
+ if (channelDescriptions == null)
+ {
+ throw new ConfigurationFailureException(String.format(
+ "Both '%s' and '%s' should be configured", CHANNEL_CODES, CHANNEL_LABELS));
+ }
+ return channelDescriptions;
+ }
+
+ private final static List<ChannelDescription> tryExtractChannelDescriptions(
+ final Properties properties)
+ {
+ List<String> names = PropertyUtils.tryGetList(properties, CHANNEL_NAMES);
+ List<String> codes = PropertyUtils.tryGetList(properties, CHANNEL_CODES);
+ List<String> labels = tryGetListOfLabels(properties, CHANNEL_LABELS);
+ if (names != null && (codes != null || labels != null))
+ {
+ throw new ConfigurationFailureException(String.format(
+ "Configure either '%s' or ('%s','%s') but not both.", CHANNEL_NAMES,
+ CHANNEL_CODES, CHANNEL_LABELS));
+ }
+ if (names != null)
+ {
+ List<ChannelDescription> descriptions = new ArrayList<ChannelDescription>();
+ for (String name : names)
+ {
+ descriptions.add(new ChannelDescription(name));
+ }
+ return descriptions;
+ }
+ if (codes == null || labels == null)
+ {
+ return null;
+ }
+ if (codes.size() != labels.size())
+ {
+ throw new ConfigurationFailureException(String.format(
+ "Number of configured '%s' should be the same as number of '%s'.",
+ CHANNEL_CODES, CHANNEL_LABELS));
+ }
+ List<ChannelDescription> descriptions = new ArrayList<ChannelDescription>();
+ for (int i = 0; i < codes.size(); i++)
+ {
+ descriptions.add(new ChannelDescription(codes.get(i), labels.get(i)));
+ }
+ return descriptions;
+ }
+
+ private final static List<String> tryGetListOfLabels(Properties properties, String propertyKey)
+ {
+ String itemsList = PropertyUtils.getProperty(properties, propertyKey);
+ if (itemsList == null)
+ {
+ return null;
+ }
+ String[] items = itemsList.split(",");
+ for (int i = 0; i < items.length; i++)
+ {
+ items[i] = items[i].trim();
+ }
+ return Arrays.asList(items);
}
protected final static void ensureChannelExist(
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/AbstractImageStorageProcessor.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/AbstractImageStorageProcessor.java
index bb5ae6b1252..52164f21bfe 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/AbstractImageStorageProcessor.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/AbstractImageStorageProcessor.java
@@ -19,7 +19,6 @@ package ch.systemsx.cisd.openbis.dss.etl;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Properties;
@@ -29,13 +28,14 @@ import javax.sql.DataSource;
import net.lemnik.eodsql.QueryTool;
+import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang.time.DurationFormatUtils;
import org.apache.log4j.Logger;
import ch.systemsx.cisd.bds.hcs.Geometry;
import ch.systemsx.cisd.common.collections.CollectionUtils;
-import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.exceptions.UserFailureException;
import ch.systemsx.cisd.common.filesystem.FileOperations;
import ch.systemsx.cisd.common.filesystem.FileUtilities;
@@ -50,6 +50,7 @@ import ch.systemsx.cisd.etlserver.AbstractStorageProcessor;
import ch.systemsx.cisd.etlserver.ITypeExtractor;
import ch.systemsx.cisd.etlserver.hdf5.Hdf5Container;
import ch.systemsx.cisd.etlserver.hdf5.HierarchicalStructureDuplicatorFileToHdf5;
+import ch.systemsx.cisd.etlserver.utils.Unzipper;
import ch.systemsx.cisd.openbis.dss.Constants;
import ch.systemsx.cisd.openbis.dss.etl.dataaccess.IImagingQueryDAO;
import ch.systemsx.cisd.openbis.dss.etl.dto.ImageSeriesPoint;
@@ -136,16 +137,6 @@ abstract class AbstractImageStorageProcessor extends AbstractStorageProcessor
protected static final String FILE_EXTRACTOR_PROPERTY = "file-extractor";
- // comma separated list of channel names, order matters
- @Deprecated
- public static final String CHANNEL_NAMES = "channel-names";
-
- // comma separated list of channel codes, order matters
- public static final String CHANNEL_CODES = "channel-codes";
-
- // comma separated list of channel labels, order matters
- public static final String CHANNEL_LABELS = "channel-labels";
-
// how the original data should be stored
private static enum OriginalDataStorageFormat
{
@@ -178,8 +169,6 @@ abstract class AbstractImageStorageProcessor extends AbstractStorageProcessor
protected final Geometry spotGeometry;
- protected final List<ChannelDescription> channelDescriptions;
-
// --- internal state -------------
private IImagingQueryDAO currentTransaction;
@@ -188,17 +177,14 @@ abstract class AbstractImageStorageProcessor extends AbstractStorageProcessor
public AbstractImageStorageProcessor(final Properties properties)
{
- this(getMandatorySpotGeometry(properties), extractChannelDescriptions(properties),
- tryCreateImageExtractor(properties), properties);
+ this(getMandatorySpotGeometry(properties), tryCreateImageExtractor(properties), properties);
}
protected AbstractImageStorageProcessor(Geometry spotGeometry,
- List<ChannelDescription> channelDescriptions, IImageFileExtractor imageFileExtractor,
- Properties properties)
+ IImageFileExtractor imageFileExtractor, Properties properties)
{
super(properties);
this.spotGeometry = spotGeometry;
- this.channelDescriptions = channelDescriptions;
this.imageFileExtractor = imageFileExtractor;
this.thumbnailMaxWidth =
PropertyUtils.getInt(properties, THUMBNAIL_MAX_WIDTH_PROPERTY,
@@ -245,61 +231,6 @@ abstract class AbstractImageStorageProcessor extends AbstractStorageProcessor
return OriginalDataStorageFormat.valueOf(textValue.toUpperCase());
}
- private final static List<String> tryGetListOfLabels(Properties properties, String propertyKey)
- {
- String itemsList = PropertyUtils.getProperty(properties, propertyKey);
- if (itemsList == null)
- {
- return null;
- }
- String[] items = itemsList.split(",");
- for (int i = 0; i < items.length; i++)
- {
- items[i] = items[i].trim();
- }
- return Arrays.asList(items);
- }
-
- public final static List<ChannelDescription> extractChannelDescriptions(
- final Properties properties)
- {
- List<String> names = PropertyUtils.tryGetList(properties, CHANNEL_NAMES);
- List<String> codes = PropertyUtils.tryGetList(properties, CHANNEL_CODES);
- List<String> labels = tryGetListOfLabels(properties, CHANNEL_LABELS);
- if (names != null && (codes != null || labels != null))
- {
- throw new ConfigurationFailureException(String.format(
- "Configure either '%s' or ('%s','%s') but not both.", CHANNEL_NAMES,
- CHANNEL_CODES, CHANNEL_LABELS));
- }
- if (names != null)
- {
- List<ChannelDescription> descriptions = new ArrayList<ChannelDescription>();
- for (String name : names)
- {
- descriptions.add(new ChannelDescription(name));
- }
- return descriptions;
- }
- if (codes == null || labels == null)
- {
- throw new ConfigurationFailureException(String.format(
- "Both '%s' and '%s' should be configured", CHANNEL_CODES, CHANNEL_LABELS));
- }
- if (codes.size() != labels.size())
- {
- throw new ConfigurationFailureException(String.format(
- "Number of configured '%s' should be the same as number of '%s'.",
- CHANNEL_CODES, CHANNEL_LABELS));
- }
- List<ChannelDescription> descriptions = new ArrayList<ChannelDescription>();
- for (int i = 0; i < codes.size(); i++)
- {
- descriptions.add(new ChannelDescription(codes.get(i), labels.get(i)));
- }
- return descriptions;
- }
-
private IImagingQueryDAO createQuery()
{
return QueryTool.getQuery(dataSource, IImagingQueryDAO.class);
@@ -315,6 +246,13 @@ abstract class AbstractImageStorageProcessor extends AbstractStorageProcessor
assert incomingDataSetDirectory != null : "Incoming data set directory can not be null.";
assert typeExtractor != null : "Unspecified IProcedureAndDataTypeExtractor implementation.";
+ File unzipedFolder = tryUnzipToFolder(incomingDataSetDirectory);
+ if (unzipedFolder != null)
+ {
+ return storeData(dataSetInformation, typeExtractor, mailClient, unzipedFolder,
+ rootDirectory);
+ }
+
ImageFileExtractionResult extractionResult =
extractImages(dataSetInformation, incomingDataSetDirectory);
@@ -330,6 +268,23 @@ abstract class AbstractImageStorageProcessor extends AbstractStorageProcessor
return rootDirectory;
}
+ private File tryUnzipToFolder(File incomingDataSetDirectory)
+ {
+ if (isZipFile(incomingDataSetDirectory) == false)
+ {
+ return null;
+ }
+ String outputDirName = FilenameUtils.getBaseName(incomingDataSetDirectory.getName());
+ File output = new File(incomingDataSetDirectory.getParentFile(), outputDirName);
+ Status status = Unzipper.unzip(incomingDataSetDirectory, output, true);
+ if (status.isError())
+ {
+ throw EnvironmentFailureException.fromTemplate("Cannot unzip '%s': %s",
+ incomingDataSetDirectory.getName(), status.tryGetErrorMessage());
+ }
+ return output;
+ }
+
private void processImages(final File rootDirectory, List<AcquiredSingleImage> plateImages,
File imagesInStoreFolder)
{
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSImageDatasetInfo.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSImageDatasetInfo.java
index 6a1e9c56dd5..5dbfb7b4603 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSImageDatasetInfo.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSImageDatasetInfo.java
@@ -23,12 +23,15 @@ package ch.systemsx.cisd.openbis.dss.etl;
*/
public class HCSImageDatasetInfo extends HCSContainerDatasetInfo
{
+ private final boolean storeChannelsOnExperimentLevel;
+
private final int tileRows, tileColumns;
// has any well timepoints or depth stack images?
private final boolean hasImageSeries;
- public HCSImageDatasetInfo(HCSContainerDatasetInfo info, int tileRows, int tileColumns,
+ public HCSImageDatasetInfo(HCSContainerDatasetInfo info,
+ boolean storeChannelsOnExperimentLevel, int tileRows, int tileColumns,
boolean hasImageSeries)
{
super.setContainerRows(info.getContainerRows());
@@ -36,6 +39,7 @@ public class HCSImageDatasetInfo extends HCSContainerDatasetInfo
super.setContainerPermId(info.getContainerPermId());
super.setDatasetPermId(info.getDatasetPermId());
super.setExperimentPermId(info.getExperimentPermId());
+ this.storeChannelsOnExperimentLevel = storeChannelsOnExperimentLevel;
this.tileRows = tileRows;
this.tileColumns = tileColumns;
this.hasImageSeries = hasImageSeries;
@@ -55,4 +59,9 @@ public class HCSImageDatasetInfo extends HCSContainerDatasetInfo
{
return hasImageSeries;
}
+
+ public boolean isStoreChannelsOnExperimentLevel()
+ {
+ return storeChannelsOnExperimentLevel;
+ }
}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSImageDatasetUploader.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSImageDatasetUploader.java
index c516b94d50f..387cc0088be 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSImageDatasetUploader.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSImageDatasetUploader.java
@@ -46,16 +46,29 @@ public class HCSImageDatasetUploader extends AbstractImageDatasetUploader
private void upload(HCSImageDatasetInfo info, List<AcquiredSingleImage> images,
List<ImageFileExtractionResult.Channel> channels)
{
- ExperimentWithChannelsAndContainer basicStruct =
- ImagingDatabaseHelper.getOrCreateExperimentWithChannelsAndContainer(
- dao, info, channels);
- long contId = basicStruct.getContainerId();
- ImagingChannelsMap channelsMap = basicStruct.getChannelsMap();
+ long contId;
+ ImagingChannelsMap channelsMap = null;
+ if (info.isStoreChannelsOnExperimentLevel())
+ {
+ ExperimentWithChannelsAndContainer basicStruct =
+ ImagingDatabaseHelper.getOrCreateExperimentWithChannelsAndContainer(dao, info,
+ channels);
+ contId = basicStruct.getContainerId();
+ channelsMap = basicStruct.getChannelsMap();
+ } else
+ {
+ contId = ImagingDatabaseHelper.getOrCreateExperimentAndContainer(dao, info);
+ }
Long[][] spotIds = getOrCreateSpots(contId, info, images);
ISpotProvider spotProvider = getSpotProvider(spotIds);
long datasetId = createDataset(contId, info);
+ if (info.isStoreChannelsOnExperimentLevel() == false)
+ {
+ channelsMap = ImagingDatabaseHelper.createDatasetChannels(dao, datasetId, channels);
+ }
+ assert channelsMap != null;
createImages(images, spotProvider, channelsMap, datasetId);
}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSImageFileExtractor.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSImageFileExtractor.java
index 60f98e67ae3..7b55b51818c 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSImageFileExtractor.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSImageFileExtractor.java
@@ -49,11 +49,17 @@ public class HCSImageFileExtractor extends AbstractImageFileExtractor
public HCSImageFileExtractor(final Properties properties)
{
- super(properties, false);
+ super(properties, extractSkipChannelsWithoutImages(properties));
this.shouldValidatePlateName =
PropertyUtils.getBoolean(properties, CHECK_PLATE_NAME_FLAG_PROPERTY_NAME, true);
}
+ private static boolean extractSkipChannelsWithoutImages(Properties properties)
+ {
+ return PropertyUtils.getBoolean(properties,
+ PlateStorageProcessor.CHANNELS_PER_EXPERIMENT_PROPERTY, true) == false;
+ }
+
/**
* Extracts the plate location from argument. Returns <code>null</code> if the operation fails.
*/
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/MicroscopyBlackboxSeriesStorageProcessor.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/MicroscopyBlackboxSeriesStorageProcessor.java
index 35afb96026c..703d95c2406 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/MicroscopyBlackboxSeriesStorageProcessor.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/MicroscopyBlackboxSeriesStorageProcessor.java
@@ -59,7 +59,7 @@ public class MicroscopyBlackboxSeriesStorageProcessor extends AbstractImageStora
public MicroscopyBlackboxSeriesStorageProcessor(Properties properties)
{
- super(DEFAULT_WELL_GEOMETRY, DEFAULT_CHANNELS, new BlackboxSeriesImageFileExtractor(
+ super(DEFAULT_WELL_GEOMETRY, new BlackboxSeriesImageFileExtractor(
properties), properties);
}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/PlateStorageProcessor.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/PlateStorageProcessor.java
index c34381e90b4..7c6eb63c052 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/PlateStorageProcessor.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/PlateStorageProcessor.java
@@ -34,6 +34,7 @@ import ch.systemsx.cisd.common.exceptions.UserFailureException;
import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.common.mail.IMailClient;
import ch.systemsx.cisd.common.utilities.ClassUtils;
+import ch.systemsx.cisd.common.utilities.PropertyUtils;
import ch.systemsx.cisd.etlserver.IHCSImageFileAccepter;
import ch.systemsx.cisd.openbis.dss.etl.HCSImageCheckList.FullLocation;
import ch.systemsx.cisd.openbis.dss.etl.dataaccess.IImagingQueryDAO;
@@ -54,10 +55,18 @@ public final class PlateStorageProcessor extends AbstractImageStorageProcessor
// a class of the old-style image extractor
private static final String DEPRECATED_FILE_EXTRACTOR_PROPERTY = "deprecated-file-extractor";
+ /**
+ * Optional boolean property. Defines if all image datasets in one experiment have the same
+ * channels or if each imported dataset can have different channels. By default true.
+ */
+ static final String CHANNELS_PER_EXPERIMENT_PROPERTY = "define-channels-per-experiment";
+
// ---
private final ch.systemsx.cisd.etlserver.IHCSImageFileExtractor deprecatedImageFileExtractor;
+ private final boolean storeChannelsOnExperimentLevel;
+
public PlateStorageProcessor(Properties properties)
{
super(properties);
@@ -71,6 +80,8 @@ public final class PlateStorageProcessor extends AbstractImageStorageProcessor
{
this.deprecatedImageFileExtractor = null;
}
+ this.storeChannelsOnExperimentLevel =
+ PropertyUtils.getBoolean(properties, CHANNELS_PER_EXPERIMENT_PROPERTY, true);
}
private static final class HCSImageFileAccepter implements IHCSImageFileAccepter
@@ -160,7 +171,8 @@ public final class PlateStorageProcessor extends AbstractImageStorageProcessor
protected void validateImages(DataSetInformation dataSetInformation, IMailClient mailClient,
File incomingDataSetDirectory, ImageFileExtractionResult extractionResult)
{
- HCSImageCheckList imageCheckList = createImageCheckList(dataSetInformation);
+ HCSImageCheckList imageCheckList =
+ createImageCheckList(dataSetInformation, extractionResult.getChannels());
checkImagesForDuplicates(extractionResult, imageCheckList);
if (extractionResult.getInvalidFiles().size() > 0)
{
@@ -193,13 +205,14 @@ public final class PlateStorageProcessor extends AbstractImageStorageProcessor
return HCSContainerDatasetInfo.getPlateGeometry(dataSetInformation);
}
- private HCSImageCheckList createImageCheckList(DataSetInformation dataSetInformation)
+ private HCSImageCheckList createImageCheckList(DataSetInformation dataSetInformation,
+ List<ch.systemsx.cisd.openbis.dss.etl.ImageFileExtractionResult.Channel> channels)
{
PlateDimension plateGeometry = getPlateGeometry(dataSetInformation);
List<String> channelCodes = new ArrayList<String>();
- for (ChannelDescription cd : channelDescriptions)
+ for (ch.systemsx.cisd.openbis.dss.etl.ImageFileExtractionResult.Channel channel : channels)
{
- channelCodes.add(cd.getCode());
+ channelCodes.add(channel.getCode());
}
return new HCSImageCheckList(channelCodes, plateGeometry, spotGeometry);
}
@@ -250,6 +263,8 @@ public final class PlateStorageProcessor extends AbstractImageStorageProcessor
IImageFileExtractor extractor = imageFileExtractor;
if (extractor == null)
{
+ List<ChannelDescription> channelDescriptions =
+ AbstractImageFileExtractor.extractChannelDescriptions(properties);
extractor =
adapt(deprecatedImageFileExtractor, incomingDataSetDirectory,
channelDescriptions);
@@ -297,7 +312,7 @@ public final class PlateStorageProcessor extends AbstractImageStorageProcessor
HCSContainerDatasetInfo info =
HCSContainerDatasetInfo.createScreeningDatasetInfo(dataSetInformation);
boolean hasImageSeries = hasImageSeries(acquiredImages);
- return new HCSImageDatasetInfo(info, spotGeometry.getRows(), spotGeometry.getColumns(),
- hasImageSeries);
+ return new HCSImageDatasetInfo(info, storeChannelsOnExperimentLevel,
+ spotGeometry.getRows(), spotGeometry.getColumns(), hasImageSeries);
}
}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/ImagingDatasetLoader.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/ImagingDatasetLoader.java
index 7b972f37a7f..c5ac54f8df1 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/ImagingDatasetLoader.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/ImagingDatasetLoader.java
@@ -125,14 +125,17 @@ public class ImagingDatasetLoader extends HCSDatasetLoader implements IImagingDa
private ImgChannelDTO tryLoadChannel(String chosenChannelCode)
{
- if (containerOrNull != null)
+ // first we check if there are some channels defined at the dataset level (even for HCS one
+ // can decide in configuration about that)
+ ImgChannelDTO channel = query.tryGetChannelForDataset(dataset.getId(), chosenChannelCode);
+ // if not, we check at the experiment level
+ if (channel == null && containerOrNull != null)
{
- return query.tryGetChannelForExperiment(containerOrNull.getExperimentId(),
- chosenChannelCode);
- } else
- {
- return query.tryGetChannelForDataset(dataset.getId(), chosenChannelCode);
+ channel =
+ query.tryGetChannelForExperiment(containerOrNull.getExperimentId(),
+ chosenChannelCode);
}
+ return channel;
}
private void validateChannelStackReference(ImageChannelStackReference channelStackReference)
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/generic/server/images/ImageChannelsUtils.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/generic/server/images/ImageChannelsUtils.java
index f1b54ba8a10..b16f7d65062 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/generic/server/images/ImageChannelsUtils.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/generic/server/images/ImageChannelsUtils.java
@@ -116,7 +116,8 @@ public class ImageChannelsUtils
// NOTE: never merges the overlays, draws each channel separately (merging looses
// transparency and is slower)
List<BufferedImage> overlayImages =
- getSingleImages(overlayChannels, overlaySize, datasetDirectoryProvider);
+ getSingleImagesSkipNonExisting(overlayChannels, overlaySize,
+ datasetDirectoryProvider);
for (BufferedImage overlayImage : overlayImages)
{
if (image != null)
@@ -131,7 +132,7 @@ public class ImageChannelsUtils
return createResponseContentStream(image, null);
}
- private static List<BufferedImage> getSingleImages(
+ private static List<BufferedImage> getSingleImagesSkipNonExisting(
DatasetAcquiredImagesReference imagesReference, RequestedImageSize imageSize,
IDatasetDirectoryProvider datasetDirectoryProvider)
{
@@ -139,7 +140,7 @@ public class ImageChannelsUtils
createImageChannelsUtils(imagesReference, datasetDirectoryProvider, imageSize);
boolean mergeAllChannels = utils.isMergeAllChannels(imagesReference);
List<AbsoluteImageReference> imageContents =
- utils.fetchImageContents(imagesReference, mergeAllChannels);
+ utils.fetchImageContents(imagesReference, mergeAllChannels, true);
return calculateSingleImages(imageContents);
}
@@ -198,7 +199,7 @@ public class ImageChannelsUtils
{
boolean mergeAllChannels = isMergeAllChannels(imageChannels);
List<AbsoluteImageReference> imageContents =
- fetchImageContents(imageChannels, mergeAllChannels);
+ fetchImageContents(imageChannels, mergeAllChannels, false);
return mergeChannels(imageContents, transform, mergeAllChannels);
}
@@ -227,16 +228,30 @@ public class ImageChannelsUtils
return true;
}
+ /**
+ * @param skipNonExisting if true references to non-existing images are ignored, otherwise an
+ * exception is thrown
+ */
private List<AbsoluteImageReference> fetchImageContents(
- DatasetAcquiredImagesReference imagesReference, boolean mergeAllChannels)
+ DatasetAcquiredImagesReference imagesReference, boolean mergeAllChannels,
+ boolean skipNonExisting)
{
List<String> channelCodes = getChannelCodes(imagesReference);
List<AbsoluteImageReference> images = new ArrayList<AbsoluteImageReference>();
for (String channelCode : channelCodes)
{
+ ImageChannelStackReference channelStackReference =
+ imagesReference.getChannelStackReference();
AbsoluteImageReference image =
- getImageReference(imagesReference.getChannelStackReference(), channelCode);
- images.add(image);
+ imageAccessor.tryGetImage(channelCode, channelStackReference, imageSizeLimit);
+ if (image == null && skipNonExisting == false)
+ {
+ throw createImageNotFoundException(channelStackReference, channelCode);
+ }
+ if (image != null)
+ {
+ images.add(image);
+ }
}
// Optimization for a case where all channels are on one image
@@ -331,7 +346,7 @@ public class ImageChannelsUtils
new ImageChannelsUtils(imageAccessor, imageSizeLimitOrNull);
boolean mergeAllChannels = imageChannelsUtils.isMergeAllChannels(imagesReference);
List<AbsoluteImageReference> imageContents =
- imageChannelsUtils.fetchImageContents(imagesReference, mergeAllChannels);
+ imageChannelsUtils.fetchImageContents(imagesReference, mergeAllChannels, false);
IContent rawContent = tryGetRawContent(convertToPng, imageContents);
if (rawContent != null)
@@ -594,24 +609,13 @@ public class ImageChannelsUtils
// --------- common
- /**
- * @throw {@link EnvironmentFailureException} when image does not exist
- */
- private AbsoluteImageReference getImageReference(
+ private EnvironmentFailureException createImageNotFoundException(
ImageChannelStackReference channelStackReference, String chosenChannelCode)
{
- AbsoluteImageReference image =
- imageAccessor.tryGetImage(chosenChannelCode, channelStackReference, imageSizeLimit);
- if (image != null)
- {
- return image;
- } else
- {
- throw EnvironmentFailureException.fromTemplate(
- "No " + (imageSizeLimit.isThumbnailRequired() ? "thumbnail" : "image")
- + " found for channel stack %s and channel %s", channelStackReference,
- chosenChannelCode);
- }
+ return EnvironmentFailureException.fromTemplate(
+ "No " + (imageSizeLimit.isThumbnailRequired() ? "thumbnail" : "image")
+ + " found for channel stack %s and channel %s", channelStackReference,
+ chosenChannelCode);
}
/**
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/imaging/HCSDatasetLoader.java b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/imaging/HCSDatasetLoader.java
index d0e56ba8747..72422a5d8f0 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/imaging/HCSDatasetLoader.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/imaging/HCSDatasetLoader.java
@@ -82,13 +82,12 @@ public class HCSDatasetLoader implements IImageDatasetLoader
private List<ImgChannelDTO> loadChannels()
{
- if (containerOrNull != null)
- {
- return query.getChannelsByExperimentId(containerOrNull.getExperimentId());
- } else
+ List<ImgChannelDTO> myChannels = query.getChannelsByDatasetId(dataset.getId());
+ if (myChannels.size() == 0 && containerOrNull != null)
{
- return query.getChannelsByDatasetId(dataset.getId());
+ myChannels = query.getChannelsByExperimentId(containerOrNull.getExperimentId());
}
+ return myChannels;
}
private String tryGetImageTransformerFactorySignatureForMergedChannels()
--
GitLab