From 14a75d281ccc4ef411d55fa55eb599bb9152bfed Mon Sep 17 00:00:00 2001
From: tpylak <tpylak>
Date: Tue, 8 Nov 2011 14:26:51 +0000
Subject: [PATCH] LMS-2562 store thumbnails in separate contained datasets
(main chunk)
SVN: 23586
---
.../dss/etl/AbstractImageDatasetUploader.java | 17 +-
.../dss/etl/AbstractImageFileExtractor.java | 49 +--
.../etl/AbstractImageStorageProcessor.java | 286 ++++++++----------
.../openbis/dss/etl/AcquiredSingleImage.java | 9 +-
.../dss/etl/HCSContainerDatasetInfo.java | 53 ++--
.../openbis/dss/etl/HCSImageCheckList.java | 4 +-
.../openbis/dss/etl/HCSImageDatasetInfo.java | 2 +-
.../dss/etl/HCSImageDatasetUploader.java | 2 +-
.../dss/etl/Hdf5ThumbnailGenerator.java | 107 +++++--
.../dss/etl/ImageFileExtractionResult.java | 20 +-
.../cisd/openbis/dss/etl/ImageValidator.java | 16 +-
.../dss/etl/ImagingDatabaseHelper.java | 2 +-
...roscopyBlackboxSeriesStorageProcessor.java | 19 +-
.../dss/etl/MicroscopyImageChecklist.java | 4 +-
.../dss/etl/MicroscopyStorageProcessor.java | 18 +-
.../dss/etl/PlateStorageProcessor.java | 235 +++++++-------
.../dss/etl/RelativeImageReference.java | 2 +-
.../dss/etl/dataaccess/IImagingQueryDAO.java | 5 +
.../etl/dataaccess/ImagingDatasetLoader.java | 61 +++-
.../openbis/dss/etl/dto/ImageDatasetInfo.java | 12 +-
.../openbis/dss/etl/dto/ImageZoomLevel.java | 46 +++
.../dss/etl/dto/RelativeImageFile.java | 64 ++++
.../dto/api/impl/ImageDataSetInformation.java | 120 ++++++++
.../ImageDataSetStructure.java} | 36 ++-
.../etl/dto/api/impl/ThumbnailFilePaths.java | 56 ++++
...ImagingDataSetRegistrationTransaction.java | 44 +++
.../dto/api/v1/IImagingDatasetFactory.java | 9 +-
.../etl/dto/api/v1/SimpleImageDataConfig.java | 6 +-
.../FeatureVectorStorageProcessor.java | 2 +-
.../etl/genedata/FeatureStorageProcessor.java | 23 +-
.../etl/genedata/HCSImageFileExtractor.java | 108 -------
.../etl/jython/JythonPlateDataSetHandler.java | 280 +++++++++++++++--
.../jython/SimpleImageDataSetRegistrator.java | 26 +-
.../detailviewers/ImagingDatasetGuiUtils.java | 7 +-
.../utils/EntityTypeLabelUtils.java | 12 +-
.../server/logic/HCSImageDatasetLoader.java | 12 +-
.../server/logic/LogicalImageLoader.java | 7 +-
.../server/logic/PlateContentLoader.java | 36 +--
.../server/logic/ScreeningUtils.java | 108 ++++++-
.../shared/basic/dto/ScreeningConstants.java | 32 +-
.../shared/imaging/HCSDatasetLoader.java | 15 +-
.../dataaccess/IImagingReadonlyQueryDAO.java | 4 +
.../dataaccess/ImgImageZoomLevelDTO.java | 80 +++++
.../java/BiozentrumMatLabApiTest.java | 222 ++++++++++++++
.../etl/dataaccess/ImagingQueryDAOTest.java | 16 +
.../FeatureVectorUploaderTest.java | 2 +-
.../SimpleImageDataSetRegistratorTest.java | 37 +--
47 files changed, 1674 insertions(+), 659 deletions(-)
create mode 100644 screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/ImageZoomLevel.java
create mode 100644 screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/RelativeImageFile.java
create mode 100644 screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/ImageDataSetInformation.java
rename screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/{v1/ImageDataSetInformation.java => impl/ImageDataSetStructure.java} (75%)
create mode 100644 screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/ThumbnailFilePaths.java
create mode 100644 screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/IImagingDataSetRegistrationTransaction.java
delete mode 100644 screening/source/java/ch/systemsx/cisd/openbis/dss/etl/genedata/HCSImageFileExtractor.java
create mode 100644 screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/imaging/dataaccess/ImgImageZoomLevelDTO.java
create mode 100644 screening/sourceTest/java/BiozentrumMatLabApiTest.java
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/AbstractImageDatasetUploader.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/AbstractImageDatasetUploader.java
index ad81c489097..a182385cdbf 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/AbstractImageDatasetUploader.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/AbstractImageDatasetUploader.java
@@ -32,11 +32,13 @@ import ch.systemsx.cisd.openbis.dss.etl.ImagingDatabaseHelper.ImagingChannelsMap
import ch.systemsx.cisd.openbis.dss.etl.dataaccess.IImagingQueryDAO;
import ch.systemsx.cisd.openbis.dss.etl.dto.ImageDatasetInfo;
import ch.systemsx.cisd.openbis.dss.etl.dto.ImageLibraryInfo;
+import ch.systemsx.cisd.openbis.dss.etl.dto.ImageZoomLevel;
import ch.systemsx.cisd.openbis.generic.shared.basic.CodeNormalizer;
import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgAcquiredImageDTO;
import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgChannelStackDTO;
import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgImageDTO;
import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgImageDatasetDTO;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgImageZoomLevelDTO;
/**
* Abstract superclass for uploaders of image datasets into the imaging database.
@@ -261,7 +263,7 @@ abstract class AbstractImageDatasetUploader
{
ImgImageDTO dto =
- new ImgImageDTO(dao.createImageId(), imageReferenceOrNull.getRelativeImagePath(),
+ new ImgImageDTO(dao.createImageId(), imageReferenceOrNull.getImageRelativePath(),
imageReferenceOrNull.tryGetImageID(),
imageReferenceOrNull.tryGetColorComponent());
return dto;
@@ -272,7 +274,18 @@ abstract class AbstractImageDatasetUploader
{
ImgImageDatasetDTO dataset =
createImageDatasetDTO(datasetPermId, imageDatasetInfo, containerIdOrNull);
- return dao.addImageDataset(dataset);
+ long imageContainerDatasetId = dao.addImageDataset(dataset);
+ for (ImageZoomLevel imageZoomLevel : imageDatasetInfo.getImageZoomLevels())
+ {
+ dao.addImageZoomLevel(convert(imageContainerDatasetId, imageZoomLevel));
+ }
+ return imageContainerDatasetId;
+ }
+
+ private ImgImageZoomLevelDTO convert(long imageContainerDatasetId, ImageZoomLevel imageZoomLevel)
+ {
+ return new ImgImageZoomLevelDTO(imageZoomLevel.getPhysicalDatasetPermId(),
+ imageZoomLevel.isOriginal(), imageContainerDatasetId);
}
private static ImgImageDatasetDTO createImageDatasetDTO(String datasetPermId,
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/AbstractImageFileExtractor.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/AbstractImageFileExtractor.java
index bea6a0687dd..9a6c7c0e54b 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/AbstractImageFileExtractor.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/AbstractImageFileExtractor.java
@@ -40,12 +40,15 @@ import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.utilities.PropertyUtils;
+import ch.systemsx.cisd.openbis.dss.etl.dto.RelativeImageFile;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.ThumbnailFilePaths;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.Channel;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageFileInfo;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageIdentifier;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifier;
import ch.systemsx.cisd.openbis.plugin.screening.shared.basic.dto.ChannelDescription;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.basic.dto.ScreeningConstants;
import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ColorComponent;
/**
@@ -56,6 +59,9 @@ import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.Color
* <br>
* If 'extract-single-image-channels' property is specified for storage processor then the channels
* are extracted from the color components and the channel in the image file name is ignored.
+ * <p>
+ * Deprecated, use python dropboxes instead! Image datasets registered with the extractor 1. cannot
+ * have any thumbnails and 2. will not be saved in 'original' directory!
*
* @author Tomasz Pylak
*/
@@ -243,7 +249,10 @@ abstract public class AbstractImageFileExtractor implements IImageFileExtractor
invalidFiles.add(imageFile);
}
}
- return new ImageFileExtractionResult(acquiredImages,
+ File datasetRelativeImagesFolderPath =
+ new File(ScreeningConstants.ORIGINAL_DATA_DIR + File.separator
+ + incomingDataSetDirectory.getName());
+ return new ImageFileExtractionResult(acquiredImages, datasetRelativeImagesFolderPath,
Collections.unmodifiableList(invalidFiles), getAllChannels(acquiredImages),
tileGeometry, null, null);
@@ -260,13 +269,14 @@ abstract public class AbstractImageFileExtractor implements IImageFileExtractor
{
ColorComponent colorComponent = channelColorComponentsOrNull.get(i);
ChannelDescription channelDescription = channelDescriptionsOrNull.get(i);
- images.add(createImage(imageInfo, channelDescription.getCode(), colorComponent));
+ images.add(createImage(imageInfo, channelDescription.getCode(), colorComponent,
+ null));
}
return images;
} else
{
ensureChannelExist(channelDescriptionsOrNull, imageInfo.getChannelCode());
- return createImagesWithNoColorComponent(imageInfo);
+ return createImagesWithNoColorComponent(imageInfo, null);
}
}
@@ -373,18 +383,6 @@ abstract public class AbstractImageFileExtractor implements IImageFileExtractor
return channels;
}
- protected final static List<ChannelDescription> extractChannelDescriptions(
- final Properties properties)
- {
- List<ChannelDescription> channelDescriptions = tryExtractChannelDescriptions(properties);
- if (channelDescriptions == null)
- {
- throw new ConfigurationFailureException(String.format(
- "Both '%s' and '%s' should be configured", CHANNEL_CODES, CHANNEL_LABELS));
- }
- return channelDescriptions;
- }
-
private final static List<ChannelDescription> tryExtractChannelDescriptions(
final Properties properties)
{
@@ -459,20 +457,32 @@ abstract public class AbstractImageFileExtractor implements IImageFileExtractor
}
public final static List<AcquiredSingleImage> createImagesWithNoColorComponent(
- ImageFileInfo imageInfo)
+ ImageFileInfo imageInfo, ThumbnailFilePaths thumbnailFilePathsOrNull)
{
List<AcquiredSingleImage> images = new ArrayList<AcquiredSingleImage>();
- images.add(createImage(imageInfo, imageInfo.getChannelCode(), null));
+ images.add(createImage(imageInfo, imageInfo.getChannelCode(), null,
+ thumbnailFilePathsOrNull));
return images;
}
public final static AcquiredSingleImage createImage(ImageFileInfo imageInfo,
- String channelCode, ColorComponent colorComponentOrNull)
+ String channelCode, ColorComponent colorComponentOrNull,
+ ThumbnailFilePaths thumbnailFilePathsOrNull)
{
RelativeImageReference relativeImageRef =
new RelativeImageReference(imageInfo.getImageRelativePath(),
getUniqueStringIdentifier(imageInfo.tryGetImageIdentifier()),
colorComponentOrNull);
+
+ RelativeImageReference relativeThumbnailRef = null;
+ if (thumbnailFilePathsOrNull != null)
+ {
+ String relativeThumbnailPath =
+ thumbnailFilePathsOrNull.getThumbnailPath(RelativeImageFile.create(imageInfo));
+ relativeThumbnailRef =
+ new RelativeImageReference(relativeThumbnailPath, null, colorComponentOrNull);
+ }
+
Location wellLoc = null;
if (imageInfo.hasWellLocation())
{
@@ -484,7 +494,8 @@ abstract public class AbstractImageFileExtractor implements IImageFileExtractor
Location.createLocationFromRowAndColumn(imageInfo.getTileRow(),
imageInfo.getTileColumn());
return new AcquiredSingleImage(wellLoc, tileLoc, channelCode, imageInfo.tryGetTimepoint(),
- imageInfo.tryGetDepth(), imageInfo.tryGetSeriesNumber(), relativeImageRef);
+ imageInfo.tryGetDepth(), imageInfo.tryGetSeriesNumber(), relativeImageRef,
+ relativeThumbnailRef);
}
private static String getUniqueStringIdentifier(ImageIdentifier identifier)
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/AbstractImageStorageProcessor.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/AbstractImageStorageProcessor.java
index 891854d4da3..54a3bd5a55a 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/AbstractImageStorageProcessor.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/AbstractImageStorageProcessor.java
@@ -30,7 +30,6 @@ import javax.sql.DataSource;
import net.lemnik.eodsql.QueryTool;
import org.apache.commons.io.FilenameUtils;
-import org.apache.commons.lang.time.DurationFormatUtils;
import org.apache.log4j.Logger;
import ch.systemsx.cisd.base.image.IImageTransformerFactory;
@@ -58,21 +57,22 @@ import ch.systemsx.cisd.etlserver.IDataSetInfoExtractor;
import ch.systemsx.cisd.etlserver.ITypeExtractor;
import ch.systemsx.cisd.etlserver.utils.Unzipper;
import ch.systemsx.cisd.openbis.dss.Constants;
+import ch.systemsx.cisd.openbis.dss.etl.PlateStorageProcessor.DatasetOwnerInformation;
+import ch.systemsx.cisd.openbis.dss.etl.PlateStorageProcessor.ImageDatasetOwnerInformation;
import ch.systemsx.cisd.openbis.dss.etl.dataaccess.IImagingQueryDAO;
-import ch.systemsx.cisd.openbis.dss.etl.dto.ImageLibraryInfo;
import ch.systemsx.cisd.openbis.dss.etl.dto.ImageSeriesPoint;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.ImageDataSetInformation;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.ImageDataSetStructure;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.ThumbnailFilePaths;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.Channel;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ChannelColorComponent;
-import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageDataSetInformation;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageFileInfo;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageStorageConfiguraton;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.OriginalDataStorageFormat;
-import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ThumbnailsStorageFormat;
import ch.systemsx.cisd.openbis.dss.etl.jython.JythonPlateDataSetHandler;
import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
import ch.systemsx.cisd.openbis.plugin.screening.shared.basic.dto.ChannelDescription;
-import ch.systemsx.cisd.openbis.plugin.screening.shared.basic.dto.ScreeningConstants;
import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ColorComponent;
/**
@@ -123,21 +123,21 @@ abstract class AbstractImageStorageProcessor extends AbstractStorageProcessor im
* @param dao should not be commited or rollbacked, it's done outside of this method.
*/
abstract protected void storeInDatabase(IImagingQueryDAO dao,
- DataSetInformation dataSetInformation, ImageFileExtractionResult extractedImages);
+ ImageDatasetOwnerInformation dataSetInformation,
+ ImageFileExtractionResult extractedImages);
/**
* Additional image validation (e.g. are there all images that were expected?). Prints warnings
* to the log, does not throw exceptions.
+ *
+ * @return true if the images are 'complete'.
*/
- abstract protected void validateImages(DataSetInformation dataSetInformation,
+ abstract protected boolean validateImages(DatasetOwnerInformation dataSetInformation,
IMailClient mailClient, File incomingDataSetDirectory,
ImageFileExtractionResult extractionResult);
// --------------------------------------------
- /** The directory where <i>original</i> data could be found. */
- private static final String DIR_ORIGINAL = ScreeningConstants.ORIGINAL_DATA_DIR;
-
protected static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
PlateStorageProcessor.class);
@@ -151,14 +151,6 @@ abstract class AbstractImageStorageProcessor extends AbstractStorageProcessor im
private final static String ORIGINAL_DATA_STORAGE_FORMAT_PROPERTY =
"original-data-storage-format";
- private static final String GENERATE_THUMBNAILS_PROPERTY = "generate-thumbnails";
-
- private final static String COMPRESS_THUMBNAILS_PROPERTY = "compress-thumbnails";
-
- private static final String THUMBNAIL_MAX_WIDTH_PROPERTY = "thumbnail-max-width";
-
- private static final String THUMBNAIL_MAX_HEIGHT_PROPERTY = "thumbnail-max-height";
-
// ---
private final DataSource dataSource;
@@ -194,38 +186,11 @@ abstract class AbstractImageStorageProcessor extends AbstractStorageProcessor im
private static ImageStorageConfiguraton getGlobalImageStorageConfiguraton(Properties properties)
{
ImageStorageConfiguraton storageFormatParameters = new ImageStorageConfiguraton();
- storageFormatParameters
- .setThumbnailsStorageFormat(tryCreateThumbnailsStorageFormat(properties));
storageFormatParameters
.setOriginalDataStorageFormat(getOriginalDataStorageFormat(properties));
return storageFormatParameters;
}
- private static ThumbnailsStorageFormat tryCreateThumbnailsStorageFormat(Properties properties)
- {
- boolean generateThumbnails =
- PropertyUtils.getBoolean(properties, GENERATE_THUMBNAILS_PROPERTY, false);
- if (generateThumbnails == false)
- {
- return null;
- }
- ThumbnailsStorageFormat thumbnailsStorageFormat = new ThumbnailsStorageFormat();
- int thumbnailMaxWidth =
- PropertyUtils.getInt(properties, THUMBNAIL_MAX_WIDTH_PROPERTY,
- ThumbnailsStorageFormat.DEFAULT_THUMBNAIL_MAX_SIZE);
- int thumbnailMaxHeight =
- PropertyUtils.getInt(properties, THUMBNAIL_MAX_HEIGHT_PROPERTY,
- ThumbnailsStorageFormat.DEFAULT_THUMBNAIL_MAX_SIZE);
- boolean areThumbnailsCompressed =
- PropertyUtils.getBoolean(properties, COMPRESS_THUMBNAILS_PROPERTY,
- ThumbnailsStorageFormat.DEFAULT_COMPRESS_THUMBNAILS);
-
- thumbnailsStorageFormat.setMaxWidth(thumbnailMaxWidth);
- thumbnailsStorageFormat.setMaxHeight(thumbnailMaxHeight);
- thumbnailsStorageFormat.setStoreCompressed(areThumbnailsCompressed);
- return thumbnailsStorageFormat;
- }
-
private static OriginalDataStorageFormat getOriginalDataStorageFormat(
final Properties properties)
{
@@ -294,33 +259,66 @@ abstract class AbstractImageStorageProcessor extends AbstractStorageProcessor im
this.incomingDataSetDirectory = unzipedFolder;
return getStoredDataDirectory();
}
+ if (isImageDataset() == false)
+ {
+ plainMoveToStore();
+ return rootDirectory;
+ }
ImageFileExtractionWithConfig extractionResultWithConfig =
processor.extractImages(dataSetInformation, incomingDataSetDirectory);
ImageFileExtractionResult extractionResult =
extractionResultWithConfig.getExtractionResult();
- processor.validateImages(dataSetInformation, mailClient, incomingDataSetDirectory,
- extractionResult);
+ validateImages(mailClient, extractionResultWithConfig, extractionResult);
+
List<AcquiredSingleImage> plateImages = extractionResult.getImages();
ImageStorageConfiguraton imageStorageConfiguraton =
extractionResultWithConfig.getImageStorageConfiguraton();
- File imagesInStoreFolder =
- processor.moveToStore(incomingDataSetDirectory, rootDirectory);
- this.storedDataDirectory = rootDirectory;
- // NOTE: plateImages will be changed by reference
- processImages(rootDirectory, plateImages, imagesInStoreFolder, imageStorageConfiguraton);
+ File datasetRelativeImagesFolderPath =
+ extractionResultWithConfig.getExtractionResult()
+ .getDatasetRelativeImagesFolderPath();
+ plainMoveToStore();
+ processImages(rootDirectory, plateImages, datasetRelativeImagesFolderPath,
+ imageStorageConfiguraton);
shouldDeleteOriginalDataOnCommit =
imageStorageConfiguraton.getOriginalDataStorageFormat().isHdf5();
dbTransaction = processor.createQuery();
- processor.storeInDatabase(dbTransaction, dataSetInformation, extractionResult);
+ processor.storeInDatabase(dbTransaction,
+ extractionResultWithConfig.getImageDatasetOwner(), extractionResult);
return rootDirectory;
}
+ private void validateImages(final IMailClient mailClient,
+ ImageFileExtractionWithConfig extractionResultWithConfig,
+ ImageFileExtractionResult extractionResult)
+ {
+ boolean isComplete =
+ processor.validateImages(extractionResultWithConfig.getImageDatasetOwner(),
+ mailClient, incomingDataSetDirectory, extractionResult);
+ dataSetInformation.setComplete(isComplete);
+ }
+
+ private boolean isImageDataset()
+ {
+ return (processor.imageFileExtractorOrNull != null)
+ || dataSetInformation instanceof ImageDataSetInformation;
+ }
+
+ // moves the incoming folder to the store
+ private File plainMoveToStore()
+ {
+ File destinationDir =
+ AbstractImageStorageProcessor.moveFileToDirectory(incomingDataSetDirectory,
+ rootDirectory);
+ this.storedDataDirectory = rootDirectory;
+ return destinationDir;
+ }
+
@Override
protected void executeCommit()
{
@@ -330,7 +328,10 @@ abstract class AbstractImageStorageProcessor extends AbstractStorageProcessor im
}
// commit the database transaction
- dbTransaction.close(true);
+ if (dbTransaction != null)
+ {
+ dbTransaction.close(true);
+ }
}
@Override
@@ -359,7 +360,7 @@ abstract class AbstractImageStorageProcessor extends AbstractStorageProcessor im
{
assert storedDataDirectory != null : "Unspecified stored data directory. Please call storeData(...)";
- File originalFolder = getOriginalFolder(storedDataDirectory);
+ File originalFolder = storedDataDirectory;
File[] content = originalFolder.listFiles();
if (content == null || content.length == 0)
{
@@ -390,7 +391,7 @@ abstract class AbstractImageStorageProcessor extends AbstractStorageProcessor im
storedDataDirectory = rootDirectory;
}
checkParameters(incomingDataSetDirectory, storedDataDirectory);
-
+
final File originalDataFile = tryGetProprietaryData();
if (originalDataFile == null)
{
@@ -452,20 +453,30 @@ abstract class AbstractImageStorageProcessor extends AbstractStorageProcessor im
private final class ImageFileExtractionWithConfig
{
+ private final ImageDatasetOwnerInformation imageDatasetOwner;
+
private final ImageFileExtractionResult extractionResult;
private final ImageStorageConfiguraton imageStorageConfiguraton;
- public ImageFileExtractionWithConfig(ImageFileExtractionResult extractionResult,
+ public ImageFileExtractionWithConfig(ImageDatasetOwnerInformation imageDatasetOwner,
+ ImageFileExtractionResult extractionResult,
ImageStorageConfiguraton imageStorageConfiguraton)
{
+ assert imageDatasetOwner != null : "imageDatasetOwner is null";
assert extractionResult != null : "extractionResult is null";
assert imageStorageConfiguraton != null : "imageStorageConfiguraton is null";
+ this.imageDatasetOwner = imageDatasetOwner;
this.extractionResult = extractionResult;
this.imageStorageConfiguraton = imageStorageConfiguraton;
}
+ public ImageDatasetOwnerInformation getImageDatasetOwner()
+ {
+ return imageDatasetOwner;
+ }
+
public ImageFileExtractionResult getExtractionResult()
{
return extractionResult;
@@ -494,16 +505,13 @@ abstract class AbstractImageStorageProcessor extends AbstractStorageProcessor im
return output;
}
- private static void processImages(final File rootDirectory,
- List<AcquiredSingleImage> plateImages, File imagesInStoreFolder,
- ImageStorageConfiguraton imageStorageConfiguraton)
+ private static void processImages(final File rootDirectory, List<AcquiredSingleImage> images,
+ File datasetRelativeImagesFolderPath, ImageStorageConfiguraton imageStorageConfiguraton)
{
- generateThumbnails(plateImages, rootDirectory, imagesInStoreFolder,
- imageStorageConfiguraton);
String relativeImagesDirectory =
- packageImagesIfNecessary(rootDirectory, plateImages, imagesInStoreFolder,
+ packageImagesIfNecessary(rootDirectory, images, datasetRelativeImagesFolderPath,
imageStorageConfiguraton);
- updateImagesRelativePath(relativeImagesDirectory, plateImages);
+ updateImagesRelativePath(relativeImagesDirectory, images);
}
// returns the prefix which should be added before each image path to create a path relative to
@@ -514,19 +522,19 @@ abstract class AbstractImageStorageProcessor extends AbstractStorageProcessor im
{
OriginalDataStorageFormat originalDataStorageFormat =
imageStorageConfiguraton.getOriginalDataStorageFormat();
+ File absolutePath = new File(rootDirectory, imagesInStoreFolder.getPath());
if (originalDataStorageFormat.isHdf5())
{
File hdf5OriginalContainer = getHdf5OriginalContainer(rootDirectory);
boolean isDataCompressed =
originalDataStorageFormat == OriginalDataStorageFormat.HDF5_COMPRESSED;
- String pathInHdf5Container = "/" + imagesInStoreFolder.getName() + "/";
- saveInHdf5(imagesInStoreFolder, pathInHdf5Container, hdf5OriginalContainer,
- isDataCompressed);
+ String pathInHdf5Container = "/" + absolutePath.getName() + "/";
+ saveInHdf5(absolutePath, pathInHdf5Container, hdf5OriginalContainer, isDataCompressed);
String hdf5ArchivePathPrefix = hdf5OriginalContainer.getName() + ARCHIVE_DELIMITER;
return hdf5ArchivePathPrefix + pathInHdf5Container;
} else
{
- return getRelativeImagesDirectory(rootDirectory, imagesInStoreFolder) + "/";
+ return imagesInStoreFolder.getPath() + "/";
}
}
@@ -544,65 +552,16 @@ abstract class AbstractImageStorageProcessor extends AbstractStorageProcessor im
pathInHdf5Container));
}
- private File moveToStore(File incomingDataSetDirectory, File rootDirectory)
- {
- File originalFolder = getOriginalFolder(rootDirectory);
- originalFolder.mkdirs();
- if (originalFolder.exists() == false)
- {
- throw new UserFailureException("Cannot create a directory: " + originalFolder);
- }
- return moveFileToDirectory(incomingDataSetDirectory, originalFolder);
-
- }
-
- // modifies plateImages by setting the path to thumbnails
- private static void generateThumbnails(final List<AcquiredSingleImage> plateImages,
- final File rootDirectory, final File imagesInStoreFolder,
- ImageStorageConfiguraton imageStorageConfiguraton)
- {
- final File thumbnailsFile =
- new File(rootDirectory, Constants.HDF5_CONTAINER_THUMBNAILS_FILE_NAME);
- final String relativeThumbnailFilePath =
- getRelativeImagesDirectory(rootDirectory, thumbnailsFile);
-
- ThumbnailsStorageFormat thumbnailsStorageFormatOrNull =
- imageStorageConfiguraton.getThumbnailsStorageFormat();
- if (thumbnailsStorageFormatOrNull != null)
- {
- HDF5Container container = new HDF5Container(thumbnailsFile);
- ImageLibraryInfo imageLibrary = imageStorageConfiguraton.tryGetImageLibrary();
- Hdf5ThumbnailGenerator thumbnailsGenerator =
- new Hdf5ThumbnailGenerator(plateImages, imagesInStoreFolder,
- thumbnailsStorageFormatOrNull, imageLibrary, relativeThumbnailFilePath,
- operationLog);
- container.runWriterClient(thumbnailsStorageFormatOrNull.isStoreCompressed(),
- thumbnailsGenerator);
- }
- }
-
- private static void updateImagesRelativePath(String folderPathPrefix,
+ private static void updateImagesRelativePath(String pathPrefixToAdd,
final List<AcquiredSingleImage> plateImages)
{
for (AcquiredSingleImage plateImage : plateImages)
{
RelativeImageReference imageReference = plateImage.getImageReference();
- imageReference.setRelativeImageFolder(folderPathPrefix);
+ imageReference.setRelativeImageFolder(pathPrefixToAdd);
}
}
- private static String getRelativeImagesDirectory(File rootDirectory, File imagesInStoreFolder)
- {
- String root = rootDirectory.getAbsolutePath();
- String imgDir = imagesInStoreFolder.getAbsolutePath();
- if (imgDir.startsWith(root) == false)
- {
- throw UserFailureException.fromTemplate(
- "Directory %s should be a subdirectory of directory %s.", imgDir, root);
- }
- return imgDir.substring(root.length());
- }
-
/**
* @return true if the dataset has been enriched before and already contains all the information
* about images.
@@ -615,27 +574,35 @@ abstract class AbstractImageStorageProcessor extends AbstractStorageProcessor im
private ImageFileExtractionWithConfig extractImages(
final DataSetInformation dataSetInformation, final File incomingDataSetDirectory)
{
- long extractionStart = System.currentTimeMillis();
- IImageFileExtractor extractor = tryGetImageFileExtractor(incomingDataSetDirectory);
- if (extractor == null)
+ if (imageFileExtractorOrNull == null)
{
return extractImagesFromDatasetInfoOrDie(dataSetInformation);
+ } else
+ {
+ return deprecatedExtractImages(dataSetInformation, incomingDataSetDirectory,
+ imageFileExtractorOrNull);
}
+ }
+
+ // handle deprecated non-jython way of importing images
+ private ImageFileExtractionWithConfig deprecatedExtractImages(
+ final DataSetInformation dataSetInformation, final File incomingDataSetDirectory,
+ IImageFileExtractor extractor)
+ {
ImageFileExtractionResult result =
extractor.extract(incomingDataSetDirectory, dataSetInformation);
-
- if (operationLog.isInfoEnabled())
- {
- long duration = System.currentTimeMillis() - extractionStart;
- operationLog.info(String.format("Extraction of %d files took %s.", result.getImages()
- .size(), DurationFormatUtils.formatDurationHMS(duration)));
- }
if (result.getImages().size() == 0)
{
throw new UserFailureException("No images found in the incoming diretcory: "
+ incomingDataSetDirectory);
}
- return new ImageFileExtractionWithConfig(result, globalImageStorageConfiguraton);
+ // no container dataset will be created in this case, having thumbnails will also not be
+ // allowed
+ ImageDatasetOwnerInformation imageDatasetOwner =
+ ImageDatasetOwnerInformation.create(dataSetInformation.getDataSetCode(),
+ dataSetInformation, null);
+ return new ImageFileExtractionWithConfig(imageDatasetOwner, result,
+ globalImageStorageConfiguraton);
}
private ImageFileExtractionWithConfig extractImagesFromDatasetInfoOrDie(
@@ -655,24 +622,26 @@ abstract class AbstractImageStorageProcessor extends AbstractStorageProcessor im
}
private ImageFileExtractionWithConfig extractImagesFromDatasetInfo(
- ImageDataSetInformation imageDataSetInfo)
+ ImageDataSetInformation dataSetInformation)
{
- if (imageDataSetInfo.isValid() == false)
+ ImageDataSetStructure imageDataSetStructure = dataSetInformation.getImageDataSetStructure();
+ if (imageDataSetStructure.isValid() == false)
{
throw ConfigurationFailureException
.fromTemplate("Invalid image dataset info object, check if your jython script fills all the required fields. "
+ "Or maybe the recognized files extensions is set incorrectly? Dataset: "
- + imageDataSetInfo);
+ + imageDataSetStructure);
}
Geometry tileGeometry =
- new Geometry(imageDataSetInfo.getTileRowsNumber(),
- imageDataSetInfo.getTileColumnsNumber());
+ new Geometry(imageDataSetStructure.getTileRowsNumber(),
+ imageDataSetStructure.getTileColumnsNumber());
- List<AcquiredSingleImage> images = convertImages(imageDataSetInfo);
+ ThumbnailFilePaths thumbnailFilePaths = dataSetInformation.tryGetThumbnailFilePaths();
+ List<AcquiredSingleImage> images = convertImages(imageDataSetStructure, thumbnailFilePaths);
List<File> invalidFiles = new ArrayList<File>(); // handles in an earlier phase
ImageStorageConfiguraton imageStorageConfiguraton =
- imageDataSetInfo.getImageStorageConfiguraton();
+ imageDataSetStructure.getImageStorageConfiguraton();
if (imageStorageConfiguraton == null)
{
imageStorageConfiguraton = globalImageStorageConfiguraton;
@@ -681,18 +650,29 @@ abstract class AbstractImageStorageProcessor extends AbstractStorageProcessor im
setPerImageTransformationIfNeeded(images, imageStorageConfiguraton);
ImageFileExtractionResult extractionResult =
- new ImageFileExtractionResult(images, invalidFiles, imageDataSetInfo.getChannels(),
- tileGeometry, imageStorageConfiguraton.getStoreChannelsOnExperimentLevel(),
+ new ImageFileExtractionResult(images,
+ dataSetInformation.getDatasetRelativeImagesFolderPath(), invalidFiles,
+ imageDataSetStructure.getChannels(), tileGeometry,
+ imageStorageConfiguraton.getStoreChannelsOnExperimentLevel(),
imageStorageConfiguraton.tryGetImageLibrary());
- return new ImageFileExtractionWithConfig(extractionResult, imageStorageConfiguraton);
+
+ String thumbnailDatasetPermIdOrNull =
+ (thumbnailFilePaths == null) ? null : thumbnailFilePaths
+ .getThumbnailPhysicalDatasetPermId();
+ ImageDatasetOwnerInformation imageDatasetOwner =
+ ImageDatasetOwnerInformation.create(dataSetInformation.getContainerDatasetPermId(),
+ dataSetInformation, thumbnailDatasetPermIdOrNull);
+ return new ImageFileExtractionWithConfig(imageDatasetOwner, extractionResult,
+ imageStorageConfiguraton);
}
- private static List<AcquiredSingleImage> convertImages(ImageDataSetInformation imageDataSetInfo)
+ private static List<AcquiredSingleImage> convertImages(
+ ImageDataSetStructure imageDataSetStructure, ThumbnailFilePaths thumbnailFilePathsOrNull)
{
- List<ImageFileInfo> imageInfos = imageDataSetInfo.getImages();
+ List<ImageFileInfo> imageInfos = imageDataSetStructure.getImages();
List<ChannelColorComponent> channelColorComponentsOrNull =
- imageDataSetInfo.getChannelColorComponents();
- List<Channel> channels = imageDataSetInfo.getChannels();
+ imageDataSetStructure.getChannelColorComponents();
+ List<Channel> channels = imageDataSetStructure.getChannels();
List<AcquiredSingleImage> images = new ArrayList<AcquiredSingleImage>();
for (ImageFileInfo imageInfo : imageInfos)
@@ -706,13 +686,13 @@ abstract class AbstractImageStorageProcessor extends AbstractStorageProcessor im
Channel channel = channels.get(i);
AcquiredSingleImage image =
AbstractImageFileExtractor.createImage(imageInfo, channel.getCode(),
- colorComponent);
+ colorComponent, thumbnailFilePathsOrNull);
images.add(image);
}
} else
{
- images.addAll(AbstractImageFileExtractor
- .createImagesWithNoColorComponent(imageInfo));
+ images.addAll(AbstractImageFileExtractor.createImagesWithNoColorComponent(
+ imageInfo, thumbnailFilePathsOrNull));
}
}
return images;
@@ -738,14 +718,9 @@ abstract class AbstractImageStorageProcessor extends AbstractStorageProcessor im
}
}
- protected IImageFileExtractor tryGetImageFileExtractor(File incomingDataSetDirectory)
- {
- return imageFileExtractorOrNull;
- }
-
private static void commitHdf5StorageFormatChanges(File storedDataDirectory)
{
- File originalFolder = getOriginalFolder(storedDataDirectory);
+ File originalFolder = storedDataDirectory;
File hdf5OriginalContainer = getHdf5OriginalContainer(storedDataDirectory);
if (hdf5OriginalContainer.exists()) // this should be always true
{
@@ -833,11 +808,6 @@ abstract class AbstractImageStorageProcessor extends AbstractStorageProcessor im
}
}
- private static File getOriginalFolder(File storedDataDirectory)
- {
- return new File(storedDataDirectory, DIR_ORIGINAL);
- }
-
protected static List<String> extractChannelCodes(final List<ChannelDescription> descriptions)
{
List<String> channelCodes = new ArrayList<String>();
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/AcquiredSingleImage.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/AcquiredSingleImage.java
index 51a6657e0b1..64c233ccbf8 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/AcquiredSingleImage.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/AcquiredSingleImage.java
@@ -51,7 +51,8 @@ public class AcquiredSingleImage extends AbstractHashable
public AcquiredSingleImage(Location wellLocationOrNull, Location tileLocation,
String channelCode, Float timePointOrNull, Float depthOrNull,
- Integer seriesNumberOrNull, RelativeImageReference imageFilePath)
+ Integer seriesNumberOrNull, RelativeImageReference imageFilePath,
+ RelativeImageReference thumbnailFilePathOrNull)
{
this.wellLocationOrNull = wellLocationOrNull;
this.tileLocation = tileLocation;
@@ -60,6 +61,7 @@ public class AcquiredSingleImage extends AbstractHashable
this.depthOrNull = depthOrNull;
this.seriesNumberOrNull = seriesNumberOrNull;
this.imageFilePath = imageFilePath;
+ this.thumbnailFilePathOrNull = thumbnailFilePathOrNull;
}
public Location tryGetWellLocation()
@@ -123,11 +125,6 @@ public class AcquiredSingleImage extends AbstractHashable
// ---- setters
- public final void setThumbnailFilePathOrNull(RelativeImageReference thumbnailFilePathOrNull)
- {
- this.thumbnailFilePathOrNull = thumbnailFilePathOrNull;
- }
-
public void setSeriesNumber(int seriesNumber)
{
this.seriesNumberOrNull = seriesNumber;
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSContainerDatasetInfo.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSContainerDatasetInfo.java
index 205e8ef0882..0fad8f0f0cf 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSContainerDatasetInfo.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSContainerDatasetInfo.java
@@ -17,6 +17,7 @@
package ch.systemsx.cisd.openbis.dss.etl;
import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.openbis.dss.etl.PlateStorageProcessor.DatasetOwnerInformation;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Experiment;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IEntityProperty;
@@ -34,11 +35,11 @@ public class HCSContainerDatasetInfo
{
private String experimentPermId;
- private String containerPermId;
+ private String containerSamplePermId;
- private String datasetPermId;
+ private int containerSampleRows, containerSampleColumns;
- private int containerRows, containerColumns;
+ private String datasetPermId;
public String getExperimentPermId()
{
@@ -50,14 +51,14 @@ public class HCSContainerDatasetInfo
this.experimentPermId = experimentPermId;
}
- public String getContainerPermId()
+ public String getContainerSamplePermId()
{
- return containerPermId;
+ return containerSamplePermId;
}
- public void setContainerPermId(String containerPermId)
+ public void setContainerSamplePermId(String containerSamplePermId)
{
- this.containerPermId = containerPermId;
+ this.containerSamplePermId = containerSamplePermId;
}
public String getDatasetPermId()
@@ -72,33 +73,40 @@ public class HCSContainerDatasetInfo
public int getContainerRows()
{
- return containerRows;
+ return containerSampleRows;
}
public void setContainerRows(int containerRows)
{
- this.containerRows = containerRows;
+ this.containerSampleRows = containerRows;
}
public int getContainerColumns()
{
- return containerColumns;
+ return containerSampleColumns;
}
public void setContainerColumns(int containerColumns)
{
- this.containerColumns = containerColumns;
+ this.containerSampleColumns = containerColumns;
}
public Geometry getContainerGeometry()
{
- return Geometry.createFromRowColDimensions(containerRows, containerColumns);
+ return Geometry.createFromRowColDimensions(containerSampleRows, containerSampleColumns);
+ }
+
+ public static HCSContainerDatasetInfo createScreeningDatasetInfoWithSample(
+ DataSetInformation dataSetInformation, Sample sampleOrNull)
+ {
+ return createScreeningDatasetInfoWithSample(
+ DatasetOwnerInformation.create(dataSetInformation), sampleOrNull);
}
public static HCSContainerDatasetInfo createScreeningDatasetInfo(
- DataSetInformation dataSetInformation)
+ DatasetOwnerInformation dataSetInformation)
{
- Sample sample = dataSetInformation.tryToGetSample();
+ Sample sample = dataSetInformation.tryGetSample();
assert sample != null : "no sample connected to a dataset";
PlateDimension plateGeometry = getPlateGeometry(dataSetInformation);
HCSContainerDatasetInfo info =
@@ -110,7 +118,7 @@ public class HCSContainerDatasetInfo
* Create a screening data set info given sample.
*/
public static HCSContainerDatasetInfo createScreeningDatasetInfoWithSample(
- DataSetInformation dataSetInformation, Sample containingSample)
+ DatasetOwnerInformation dataSetInformation, Sample containingSample)
{
Sample sample = containingSample;
assert sample != null : "no sample connected to a dataset";
@@ -121,12 +129,12 @@ public class HCSContainerDatasetInfo
}
private static HCSContainerDatasetInfo createBasicScreeningDataSetInfo(
- DataSetInformation dataSetInformation, Sample sample, PlateDimension plateGeometry)
+ DatasetOwnerInformation dataSetInformation, Sample sample, PlateDimension plateGeometry)
{
- Experiment experiment = dataSetInformation.tryToGetExperiment();
+ Experiment experiment = dataSetInformation.tryGetExperiment();
HCSContainerDatasetInfo info = new HCSContainerDatasetInfo();
info.setExperimentPermId(experiment.getPermId());
- info.setContainerPermId(sample.getPermId());
+ info.setContainerSamplePermId(sample.getPermId());
info.setDatasetPermId(dataSetInformation.getDataSetCode());
info.setContainerRows(plateGeometry.getRowsNum());
info.setContainerColumns(plateGeometry.getColsNum());
@@ -148,14 +156,14 @@ public class HCSContainerDatasetInfo
return plateDimension;
}
- public static PlateDimension getPlateGeometry(final DataSetInformation dataSetInformation)
+ public static PlateDimension getPlateGeometry(final DatasetOwnerInformation dataSetInformation)
{
- IEntityProperty[] sampleProperties = dataSetInformation.getProperties();
+ IEntityProperty[] sampleProperties = dataSetInformation.getSampleProperties();
if ((sampleProperties == null || sampleProperties.length == 0)
- && dataSetInformation.tryToGetSample() != null)
+ && dataSetInformation.tryGetSample() != null)
{
sampleProperties =
- dataSetInformation.tryToGetSample().getProperties()
+ dataSetInformation.tryGetSample().getProperties()
.toArray(new IEntityProperty[0]);
}
final PlateDimension plateDimension =
@@ -168,5 +176,4 @@ public class HCSContainerDatasetInfo
}
return plateDimension;
}
-
}
\ No newline at end of file
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSImageCheckList.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSImageCheckList.java
index 0c1c9f86efc..83c84220ca1 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSImageCheckList.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSImageCheckList.java
@@ -387,11 +387,11 @@ public final class HCSImageCheckList extends AbstractImageChecklist
sb.append(" mapped to the following images:");
sb.append("\n\t");
- sb.append(primaryImage.getImageReference().getRelativeImagePath());
+ sb.append(primaryImage.getImageReference().getImageRelativePath());
for (AcquiredSingleImage duplicateImage : duplicateImages)
{
sb.append("\n\t");
- sb.append(duplicateImage.getImageReference().getRelativeImagePath());
+ sb.append(duplicateImage.getImageReference().getImageRelativePath());
}
return sb.toString();
}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSImageDatasetInfo.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSImageDatasetInfo.java
index 11c9f2a85c0..f0712ff692d 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSImageDatasetInfo.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSImageDatasetInfo.java
@@ -34,7 +34,7 @@ public class HCSImageDatasetInfo extends HCSContainerDatasetInfo
{
super.setContainerRows(info.getContainerRows());
super.setContainerColumns(info.getContainerColumns());
- super.setContainerPermId(info.getContainerPermId());
+ super.setContainerSamplePermId(info.getContainerSamplePermId());
super.setDatasetPermId(info.getDatasetPermId());
super.setExperimentPermId(info.getExperimentPermId());
this.storeChannelsOnExperimentLevel = storeChannelsOnExperimentLevel;
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSImageDatasetUploader.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSImageDatasetUploader.java
index 05ffce8b16e..0cff373ca56 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSImageDatasetUploader.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSImageDatasetUploader.java
@@ -101,7 +101,7 @@ public class HCSImageDatasetUploader extends AbstractImageDatasetUploader
List<ImgSpotDTO> oldSpots = dao.listSpots(contId);
List<ImgSpotDTO> newSpots =
createNewSpots(contId, images, oldSpots, info.getContainerRows(),
- info.getContainerColumns(), info.getContainerPermId());
+ info.getContainerColumns(), info.getContainerSamplePermId());
newSpots.addAll(oldSpots);
return makeTechIdMatrix(newSpots, info.getContainerRows(), info.getContainerColumns());
}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/Hdf5ThumbnailGenerator.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/Hdf5ThumbnailGenerator.java
index a4a5281496d..b30e716206c 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/Hdf5ThumbnailGenerator.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/Hdf5ThumbnailGenerator.java
@@ -34,6 +34,7 @@ import ch.systemsx.cisd.common.concurrent.FailureRecord;
import ch.systemsx.cisd.common.concurrent.ITaskExecutor;
import ch.systemsx.cisd.common.concurrent.ParallelizedExecutor;
import ch.systemsx.cisd.common.exceptions.Status;
+import ch.systemsx.cisd.common.hdf5.HDF5Container;
import ch.systemsx.cisd.common.hdf5.HDF5Container.IHDF5WriterClient;
import ch.systemsx.cisd.common.hdf5.IHDF5ContainerWriter;
import ch.systemsx.cisd.common.io.FileBasedContentNode;
@@ -42,7 +43,10 @@ import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.process.ProcessExecutionHelper;
import ch.systemsx.cisd.common.process.ProcessIOStrategy;
import ch.systemsx.cisd.common.process.ProcessResult;
+import ch.systemsx.cisd.openbis.dss.etl.dto.RelativeImageFile;
import ch.systemsx.cisd.openbis.dss.etl.dto.ImageLibraryInfo;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.ThumbnailFilePaths;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageStorageConfiguraton;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ThumbnailsStorageFormat;
import ch.systemsx.cisd.openbis.dss.generic.shared.utils.ImageUtil;
@@ -51,18 +55,59 @@ import ch.systemsx.cisd.openbis.dss.generic.shared.utils.ImageUtil;
*
* @author Chandrasekhar Ramakrishnan
*/
-class Hdf5ThumbnailGenerator implements IHDF5WriterClient
+public class Hdf5ThumbnailGenerator implements IHDF5WriterClient
{
+ /**
+ * Generates thumbnails of specified images whose paths are relative to the specified image
+ * parent directory.
+ *
+ * @param images path to the images relative to the imagesParentDirectory
+ * @param thumbnailFilePath absolute path to the file where thumbnails will be saved
+ * @param imageStorageConfiguraton describes how the thumbnails should be generated
+ * @return null if thumbnails generation was not requested
+ */
+ public static ThumbnailFilePaths tryGenerateThumbnails(List<RelativeImageFile> images,
+ File imagesParentDirectory, String thumbnailFilePath,
+ ImageStorageConfiguraton imageStorageConfiguraton, String thumbnailPhysicalDatasetPermId)
+ {
+ ThumbnailsStorageFormat thumbnailsStorageFormatOrNull =
+ imageStorageConfiguraton.getThumbnailsStorageFormat();
+ if (thumbnailsStorageFormatOrNull != null)
+ {
+ ThumbnailFilePaths thumbnailPaths =
+ new ThumbnailFilePaths(thumbnailPhysicalDatasetPermId);
+
+ File thumbnailsFile = new File(thumbnailFilePath);
+ final String relativeThumbnailFilePath = thumbnailsFile.getName();
+
+ HDF5Container container = new HDF5Container(thumbnailsFile);
+ ImageLibraryInfo imageLibrary = imageStorageConfiguraton.tryGetImageLibrary();
+ Hdf5ThumbnailGenerator thumbnailsGenerator =
+ new Hdf5ThumbnailGenerator(images, imagesParentDirectory,
+ thumbnailsStorageFormatOrNull, imageLibrary, relativeThumbnailFilePath,
+ thumbnailPaths, operationLog);
+ container.runWriterClient(thumbnailsStorageFormatOrNull.isStoreCompressed(),
+ thumbnailsGenerator);
+ return thumbnailPaths;
+ } else
+ {
+ return null;
+ }
+ }
+
private static final File convertUtilityOrNull = OSUtilities.findExecutable("convert");
private static final Logger machineLog = LogFactory.getLogger(LogCategory.MACHINE,
Hdf5ThumbnailGenerator.class);
+ private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
+ Hdf5ThumbnailGenerator.class);
+
private static final int MAX_RETRY_OF_FAILED_GENERATION = 3;
- private final List<AcquiredSingleImage> plateImages;
+ private final List<RelativeImageFile> images;
- private final File imagesInStoreFolder;
+ private final File imagesParentDirectory;
private final ThumbnailsStorageFormat thumbnailsStorageFormat;
@@ -70,18 +115,22 @@ class Hdf5ThumbnailGenerator implements IHDF5WriterClient
private final String relativeThumbnailFilePath;
- private final Logger operationLog;
+ private final ThumbnailFilePaths thumbnailPathCollector;
+
+ private final Logger logger;
- Hdf5ThumbnailGenerator(List<AcquiredSingleImage> plateImages, File imagesInStoreFolder,
+ private Hdf5ThumbnailGenerator(List<RelativeImageFile> images, File imagesParentDirectory,
ThumbnailsStorageFormat thumbnailsStorageFormat, ImageLibraryInfo imageLibraryOrNull,
- String relativeThumbnailFilePath, Logger operationLog)
+ String relativeThumbnailFilePath, ThumbnailFilePaths thumbnailPathCollector,
+ Logger operationLog)
{
- this.plateImages = plateImages;
- this.imagesInStoreFolder = imagesInStoreFolder;
+ this.images = images;
+ this.imagesParentDirectory = imagesParentDirectory;
this.thumbnailsStorageFormat = thumbnailsStorageFormat;
this.imageLibraryOrNull = imageLibraryOrNull;
this.relativeThumbnailFilePath = relativeThumbnailFilePath;
- this.operationLog = operationLog;
+ this.thumbnailPathCollector = thumbnailPathCollector;
+ this.logger = operationLog;
}
/**
@@ -89,29 +138,27 @@ class Hdf5ThumbnailGenerator implements IHDF5WriterClient
* the thumbnail. Using it allows not to allocate memory each time when a thumbnail
* is generated.
*/
- private Status generateThumbnail(IHDF5ContainerWriter writer, AcquiredSingleImage plateImage,
+ private Status generateThumbnail(IHDF5ContainerWriter writer, RelativeImageFile image,
ByteArrayOutputStream bufferOutputStream)
{
- RelativeImageReference imageReference = plateImage.getImageReference();
- String imagePath = imageReference.getRelativeImagePath();
- String thumbnailPath = createThumbnailPath(imageReference);
- File img = new File(imagesInStoreFolder, imagePath);
+ String imagePath = image.getImageRelativePath();
+ String thumbnailPath = createThumbnailPath(image);
+ File img = new File(imagesParentDirectory, imagePath);
try
{
long start = System.currentTimeMillis();
- String imageIdOrNull = imageReference.tryGetImageID();
+ String imageIdOrNull = image.tryGetImageID();
byte[] byteArray = generateThumbnail(bufferOutputStream, img, imageIdOrNull);
String path =
relativeThumbnailFilePath + AbstractImageStorageProcessor.ARCHIVE_DELIMITER
+ thumbnailPath;
- plateImage.setThumbnailFilePathOrNull(new RelativeImageReference(path, null,
- imageReference.tryGetColorComponent()));
+ thumbnailPathCollector.saveThumbnailPath(image, path);
- if (operationLog.isDebugEnabled())
+ if (logger.isDebugEnabled())
{
long now = System.currentTimeMillis();
- operationLog.debug(Thread.currentThread().getName() + " thumbnail " + thumbnailPath
+ logger.debug(Thread.currentThread().getName() + " thumbnail " + thumbnailPath
+ " (" + byteArray.length + " bytes) generated in " + (now - start)
+ " msec");
}
@@ -127,16 +174,16 @@ class Hdf5ThumbnailGenerator implements IHDF5WriterClient
return Status.OK;
}
- private String createThumbnailPath(RelativeImageReference imageReference)
+ private String createThumbnailPath(RelativeImageFile plateImage)
{
- String imagePath = imageReference.getRelativeImagePath();
+ String imagePath = plateImage.getImageRelativePath();
String newImagePath = imagePath;
int lastIndex = imagePath.lastIndexOf('.');
if (lastIndex > 0)
{
newImagePath = imagePath.substring(0, lastIndex);
}
- String imageIdOrNull = imageReference.tryGetImageID();
+ String imageIdOrNull = plateImage.tryGetImageID();
if (imageIdOrNull != null)
{
newImagePath += "_" + imageIdOrNull;
@@ -174,7 +221,7 @@ class Hdf5ThumbnailGenerator implements IHDF5WriterClient
}
params.add("png:-");
final ProcessResult result =
- ProcessExecutionHelper.run(params, operationLog, machineLog,
+ ProcessExecutionHelper.run(params, logger, machineLog,
ConcurrencyUtilities.NO_TIMEOUT,
ProcessIOStrategy.BINARY_DISCARD_STDERR_IO_STRATEGY, false);
if (result.isOK() == false)
@@ -208,15 +255,15 @@ class Hdf5ThumbnailGenerator implements IHDF5WriterClient
private Status createStatus(String thumbnailPath, IOException ex)
{
- operationLog.warn("Retriable error when creating thumbnail '" + thumbnailPath + "'", ex);
+ logger.warn("Retriable error when creating thumbnail '" + thumbnailPath + "'", ex);
return Status.createRetriableError(String.format("Could not generate a thumbnail '%s': %s",
thumbnailPath, ex.getMessage()));
}
- private ITaskExecutor<AcquiredSingleImage> createThumbnailGenerator(
+ private ITaskExecutor<RelativeImageFile> createThumbnailGenerator(
final IHDF5ContainerWriter writer)
{
- return new ITaskExecutor<AcquiredSingleImage>()
+ return new ITaskExecutor<RelativeImageFile>()
{
private ThreadLocal<ByteArrayOutputStream> outputStreamBuffers =
new ThreadLocal<ByteArrayOutputStream>()
@@ -228,21 +275,21 @@ class Hdf5ThumbnailGenerator implements IHDF5WriterClient
}
};
- public Status execute(AcquiredSingleImage plateImage)
+ public Status execute(RelativeImageFile image)
{
// each thread will get its own buffer to avoid allocating memory for the
// internal array each time
ByteArrayOutputStream outputStreamBuffer = outputStreamBuffers.get();
outputStreamBuffer.reset();
- return generateThumbnail(writer, plateImage, outputStreamBuffer);
+ return generateThumbnail(writer, image, outputStreamBuffer);
}
};
}
public void runWithSimpleWriter(IHDF5ContainerWriter writer)
{
- Collection<FailureRecord<AcquiredSingleImage>> errors =
- ParallelizedExecutor.process(plateImages, createThumbnailGenerator(writer),
+ Collection<FailureRecord<RelativeImageFile>> errors =
+ ParallelizedExecutor.process(images, createThumbnailGenerator(writer),
thumbnailsStorageFormat.getAllowedMachineLoadDuringGeneration(), 100,
"Thumbnails generation", MAX_RETRY_OF_FAILED_GENERATION, true);
if (errors.size() > 0)
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/ImageFileExtractionResult.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/ImageFileExtractionResult.java
index 0fe5008220c..e9a1e6ce6b4 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/ImageFileExtractionResult.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/ImageFileExtractionResult.java
@@ -32,6 +32,13 @@ public final class ImageFileExtractionResult
/** The images files with description. */
private final List<AcquiredSingleImage> images;
+ /**
+ * Path to the incoming folder with images, relative to the dataset directory. E.g. if the
+ * incoming folder name is X and the transaction's dataset registration code put it inside
+ * 'original' folder, then this path points to "original/X'.
+ */
+ private File datasetRelativeImagesFolderPath;
+
/** The invalid files found. */
private final List<File> invalidFiles;
@@ -44,11 +51,13 @@ public final class ImageFileExtractionResult
private final ImageLibraryInfo imageLibraryOrNull;
- public ImageFileExtractionResult(List<AcquiredSingleImage> images, List<File> invalidFiles,
- List<Channel> channels, Geometry tileGeometry,
- Boolean storeChannelsOnExperimentLevelOrNull, ImageLibraryInfo imageLibraryOrNull)
+ public ImageFileExtractionResult(List<AcquiredSingleImage> images,
+ File datasetRelativeImagesFolderPath, List<File> invalidFiles, List<Channel> channels,
+ Geometry tileGeometry, Boolean storeChannelsOnExperimentLevelOrNull,
+ ImageLibraryInfo imageLibraryOrNull)
{
this.images = images;
+ this.datasetRelativeImagesFolderPath = datasetRelativeImagesFolderPath;
this.invalidFiles = invalidFiles;
this.channels = channels;
this.tileGeometry = tileGeometry;
@@ -61,6 +70,11 @@ public final class ImageFileExtractionResult
return images;
}
+ public File getDatasetRelativeImagesFolderPath()
+ {
+ return datasetRelativeImagesFolderPath;
+ }
+
public List<File> getInvalidFiles()
{
return invalidFiles;
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/ImageValidator.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/ImageValidator.java
index e976242c804..381aee3e54f 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/ImageValidator.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/ImageValidator.java
@@ -28,8 +28,8 @@ import ch.systemsx.cisd.common.collections.CollectionUtils;
import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
import ch.systemsx.cisd.common.exceptions.UserFailureException;
import ch.systemsx.cisd.common.mail.IMailClient;
+import ch.systemsx.cisd.openbis.dss.etl.PlateStorageProcessor.DatasetOwnerInformation;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.Channel;
-import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Experiment;
import ch.systemsx.cisd.openbis.plugin.screening.shared.dto.PlateDimension;
@@ -40,7 +40,7 @@ import ch.systemsx.cisd.openbis.plugin.screening.shared.dto.PlateDimension;
*/
public class ImageValidator
{
- private final DataSetInformation dataSetInformation;
+ private final DatasetOwnerInformation dataSetInformation;
private final IMailClient mailClient;
@@ -65,7 +65,7 @@ public class ImageValidator
* @param extractionResult
* @param operationLog
*/
- public ImageValidator(DataSetInformation dataSetInformation, IMailClient mailClient,
+ public ImageValidator(DatasetOwnerInformation dataSetInformation, IMailClient mailClient,
File incomingDataSetDirectory, ImageFileExtractionResult extractionResult,
Logger operationLog, Logger notificationLog, boolean notifyIfPlateIncomplete)
{
@@ -82,7 +82,7 @@ public class ImageValidator
/**
* Validate the images and throw exceptions if they are not valid.
*/
- public void validateImages()
+ public boolean validateImages()
{
initializeImageCheckList();
checkImagesForDuplicates();
@@ -98,7 +98,7 @@ public class ImageValidator
+ " Have you changed your naming convention?",
incomingDataSetDirectory.getAbsolutePath());
}
- checkCompleteness();
+ return checkCompleteness();
}
private void initializeImageCheckList()
@@ -134,18 +134,17 @@ public class ImageValidator
imageCheckList.checkForDuplicates();
}
- private void checkCompleteness()
+ private boolean checkCompleteness()
{
String dataSetFileName = incomingDataSetDirectory.getName();
final boolean complete = imageCheckList.getCheckedOnFullLocationsSize() == 0;
- dataSetInformation.setComplete(complete);
if (complete == false)
{
final String message = imageCheckList.getIncompleteDataSetErrorMessage(dataSetFileName);
operationLog.warn(message);
if (mailClient != null && notifyIfPlateIncomplete)
{
- Experiment experiment = dataSetInformation.tryToGetExperiment();
+ Experiment experiment = dataSetInformation.tryGetExperiment();
assert experiment != null : "dataset not connected to an experiment: "
+ dataSetInformation;
String email = null;
@@ -171,6 +170,7 @@ public class ImageValidator
}
}
}
+ return complete;
}
private PlateDimension getPlateGeometry()
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/ImagingDatabaseHelper.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/ImagingDatabaseHelper.java
index 34114dc89d0..7ad9d9451c2 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/ImagingDatabaseHelper.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/ImagingDatabaseHelper.java
@@ -128,7 +128,7 @@ public class ImagingDatabaseHelper
private static CreatedOrFetchedEntity getOrCreateContainer(IImagingQueryDAO dao,
HCSContainerDatasetInfo info, long expId)
{
- String containerPermId = info.getContainerPermId();
+ String containerPermId = info.getContainerSamplePermId();
Long containerId = dao.tryGetContainerIdPermId(containerPermId);
if (containerId != null)
{
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/MicroscopyBlackboxSeriesStorageProcessor.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/MicroscopyBlackboxSeriesStorageProcessor.java
index 0a892e4d913..92de6dc08cd 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/MicroscopyBlackboxSeriesStorageProcessor.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/MicroscopyBlackboxSeriesStorageProcessor.java
@@ -26,11 +26,12 @@ import java.util.Properties;
import ch.systemsx.cisd.bds.hcs.Geometry;
import ch.systemsx.cisd.bds.hcs.Location;
import ch.systemsx.cisd.common.mail.IMailClient;
+import ch.systemsx.cisd.openbis.dss.etl.PlateStorageProcessor.DatasetOwnerInformation;
+import ch.systemsx.cisd.openbis.dss.etl.PlateStorageProcessor.ImageDatasetOwnerInformation;
import ch.systemsx.cisd.openbis.dss.etl.dataaccess.IImagingQueryDAO;
import ch.systemsx.cisd.openbis.dss.etl.dto.ImageDatasetInfo;
import ch.systemsx.cisd.openbis.dss.etl.dto.ImageLibraryInfo;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageFileInfo;
-import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifier;
import ch.systemsx.cisd.openbis.plugin.screening.shared.basic.dto.ChannelDescription;
@@ -85,7 +86,8 @@ public class MicroscopyBlackboxSeriesStorageProcessor extends AbstractImageStora
}
@Override
- protected void storeInDatabase(IImagingQueryDAO dao, DataSetInformation dataSetInformation,
+ protected void storeInDatabase(IImagingQueryDAO dao,
+ ImageDatasetOwnerInformation dataSetInformation,
ImageFileExtractionResult extractedImages)
{
List<AcquiredSingleImage> images = extractedImages.getImages();
@@ -118,27 +120,30 @@ public class MicroscopyBlackboxSeriesStorageProcessor extends AbstractImageStora
private String getPath(AcquiredSingleImage o1)
{
- return o1.getImageReference().getRelativeImagePath();
+ return o1.getImageReference().getImageRelativePath();
}
};
}
private MicroscopyImageDatasetInfo createMicroscopyImageDatasetInfo(
- DataSetInformation dataSetInformation, List<AcquiredSingleImage> images,
+ ImageDatasetOwnerInformation dataSetInformation, List<AcquiredSingleImage> images,
Geometry tileGeometry, ImageLibraryInfo imageLibraryInfoOrNull)
{
boolean hasImageSeries = hasImageSeries(images);
ImageDatasetInfo imageDatasetInfo =
new ImageDatasetInfo(tileGeometry.getRows(), tileGeometry.getColumns(),
- hasImageSeries, imageLibraryInfoOrNull);
+ hasImageSeries, imageLibraryInfoOrNull,
+ dataSetInformation.getImageZoomLevels());
return new MicroscopyImageDatasetInfo(dataSetInformation.getDataSetCode(), imageDatasetInfo);
}
@Override
- protected void validateImages(DataSetInformation dataSetInformation, IMailClient mailClient,
- File incomingDataSetDirectory, ImageFileExtractionResult extractionResult)
+ protected boolean validateImages(DatasetOwnerInformation dataSetInformation,
+ IMailClient mailClient, File incomingDataSetDirectory,
+ ImageFileExtractionResult extractionResult)
{
// do nothing - for now we do not have good examples of real data
+ return true;
}
}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/MicroscopyImageChecklist.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/MicroscopyImageChecklist.java
index c1e2074b7ee..06ec4ef0d87 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/MicroscopyImageChecklist.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/MicroscopyImageChecklist.java
@@ -350,11 +350,11 @@ public final class MicroscopyImageChecklist extends AbstractImageChecklist
sb.append(" mapped to the following images:");
sb.append("\n\t");
- sb.append(primaryImage.getImageReference().getRelativeImagePath());
+ sb.append(primaryImage.getImageReference().getImageRelativePath());
for (AcquiredSingleImage duplicateImage : duplicateImages)
{
sb.append("\n\t");
- sb.append(duplicateImage.getImageReference().getRelativeImagePath());
+ sb.append(duplicateImage.getImageReference().getImageRelativePath());
}
return sb.toString();
}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/MicroscopyStorageProcessor.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/MicroscopyStorageProcessor.java
index 9e90cb2c713..48dc0b45509 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/MicroscopyStorageProcessor.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/MicroscopyStorageProcessor.java
@@ -22,10 +22,11 @@ import java.util.Properties;
import ch.systemsx.cisd.bds.hcs.Geometry;
import ch.systemsx.cisd.common.mail.IMailClient;
+import ch.systemsx.cisd.openbis.dss.etl.PlateStorageProcessor.DatasetOwnerInformation;
+import ch.systemsx.cisd.openbis.dss.etl.PlateStorageProcessor.ImageDatasetOwnerInformation;
import ch.systemsx.cisd.openbis.dss.etl.dataaccess.IImagingQueryDAO;
import ch.systemsx.cisd.openbis.dss.etl.dto.ImageDatasetInfo;
import ch.systemsx.cisd.openbis.dss.etl.dto.ImageLibraryInfo;
-import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
/**
* Storage processor which stores microscopy images in a special-purpose imaging database.
@@ -43,7 +44,8 @@ public class MicroscopyStorageProcessor extends AbstractImageStorageProcessor
}
@Override
- protected void storeInDatabase(IImagingQueryDAO dao, DataSetInformation dataSetInformation,
+ protected void storeInDatabase(IImagingQueryDAO dao,
+ ImageDatasetOwnerInformation dataSetInformation,
ImageFileExtractionResult extractedImages)
{
List<AcquiredSingleImage> images = extractedImages.getImages();
@@ -55,24 +57,26 @@ public class MicroscopyStorageProcessor extends AbstractImageStorageProcessor
}
private MicroscopyImageDatasetInfo createMicroscopyImageDatasetInfo(
- DataSetInformation dataSetInformation, List<AcquiredSingleImage> images,
+ ImageDatasetOwnerInformation dataSetInformation, List<AcquiredSingleImage> images,
Geometry tileGeometry, ImageLibraryInfo imageLibraryInfoOrNull)
{
boolean hasImageSeries = hasImageSeries(images);
ImageDatasetInfo imageDatasetInfo =
new ImageDatasetInfo(tileGeometry.getRows(), tileGeometry.getColumns(),
- hasImageSeries, imageLibraryInfoOrNull);
+ hasImageSeries, imageLibraryInfoOrNull,
+ dataSetInformation.getImageZoomLevels());
return new MicroscopyImageDatasetInfo(dataSetInformation.getDataSetCode(), imageDatasetInfo);
}
@Override
- protected void validateImages(DataSetInformation dataSetInformation, IMailClient mailClient,
- File incomingDataSetDirectory, ImageFileExtractionResult extractionResult)
+ protected boolean validateImages(DatasetOwnerInformation dataSetInformation,
+ IMailClient mailClient, File incomingDataSetDirectory,
+ ImageFileExtractionResult extractionResult)
{
ImageValidator validator =
new ImageValidator(dataSetInformation, mailClient, incomingDataSetDirectory,
extractionResult, operationLog, notificationLog, false);
validator.validateImages();
+ return true;
}
-
}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/PlateStorageProcessor.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/PlateStorageProcessor.java
index 6b78ef921e7..5fd8d4e23a7 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/PlateStorageProcessor.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/PlateStorageProcessor.java
@@ -20,25 +20,21 @@ import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
-import java.util.Set;
import ch.systemsx.cisd.bds.hcs.Geometry;
-import ch.systemsx.cisd.bds.hcs.Location;
-import ch.systemsx.cisd.bds.storage.IFile;
-import ch.systemsx.cisd.bds.storage.filesystem.NodeFactory;
-import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.common.mail.IMailClient;
-import ch.systemsx.cisd.common.utilities.ClassUtils;
+import ch.systemsx.cisd.common.utilities.AbstractHashable;
import ch.systemsx.cisd.common.utilities.PropertyUtils;
-import ch.systemsx.cisd.etlserver.IHCSImageFileAccepter;
-import ch.systemsx.cisd.etlserver.IHCSImageFileExtractor;
import ch.systemsx.cisd.openbis.dss.etl.dataaccess.IImagingQueryDAO;
import ch.systemsx.cisd.openbis.dss.etl.dto.ImageDatasetInfo;
import ch.systemsx.cisd.openbis.dss.etl.dto.ImageLibraryInfo;
-import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.Channel;
+import ch.systemsx.cisd.openbis.dss.etl.dto.ImageZoomLevel;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Experiment;
-import ch.systemsx.cisd.openbis.plugin.screening.shared.basic.dto.ChannelDescription;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IEntityProperty;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Sample;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.ExperimentIdentifier;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifier;
/**
* Storage processor which stores HCS plate images in a special-purpose imaging database.
@@ -49,9 +45,6 @@ import ch.systemsx.cisd.openbis.plugin.screening.shared.basic.dto.ChannelDescrip
*/
public final class PlateStorageProcessor extends AbstractImageStorageProcessor
{
- // a class of the old-style image extractor
- private static final String DEPRECATED_FILE_EXTRACTOR_PROPERTY = "deprecated-file-extractor";
-
/**
* Optional boolean property. Defines if all image datasets in one experiment have the same
* channels or if each imported dataset can have different channels. By default true.
@@ -66,8 +59,6 @@ public final class PlateStorageProcessor extends AbstractImageStorageProcessor
// ---
- private final ch.systemsx.cisd.etlserver.IHCSImageFileExtractor deprecatedImageFileExtractorOrNull;
-
// can be overwritten for each dataset
private final boolean globalStoreChannelsOnExperimentLevel;
@@ -76,141 +67,134 @@ public final class PlateStorageProcessor extends AbstractImageStorageProcessor
public PlateStorageProcessor(Properties properties)
{
super(properties);
- this.deprecatedImageFileExtractorOrNull = tryCreateDeprecatedFileExtractor();
this.globalStoreChannelsOnExperimentLevel =
PropertyUtils.getBoolean(properties, CHANNELS_PER_EXPERIMENT_PROPERTY, true);
this.notifyIfPlateIncomplete =
PropertyUtils.getBoolean(properties, NOTIFY_IF_INCOMPLETE_PROPERTY, true);
}
- private IHCSImageFileExtractor tryCreateDeprecatedFileExtractor()
+ public static class DatasetOwnerInformation extends AbstractHashable
{
- if (imageFileExtractorOrNull == null)
+ public static DatasetOwnerInformation create(DataSetInformation dataSetInformation)
{
- String fileExtractorClass = properties.getProperty(DEPRECATED_FILE_EXTRACTOR_PROPERTY);
- if (fileExtractorClass != null)
- {
- return ClassUtils.create(ch.systemsx.cisd.etlserver.IHCSImageFileExtractor.class,
- fileExtractorClass, properties);
- }
+ return new DatasetOwnerInformation(dataSetInformation.getDataSetCode(),
+ dataSetInformation);
}
- return null;
- }
- private static final class HCSImageFileAccepter implements IHCSImageFileAccepter
- {
- private final List<AcquiredSingleImage> images = new ArrayList<AcquiredSingleImage>();
+ private final Experiment experiment;
- private final File imageFileRootDirectory;
+ private final SampleIdentifier sampleIdentifier;
- private final List<String> channelCodes;
+ private final ExperimentIdentifier experimentIdentifier;
+
+ private final Sample sample;
+
+ private final String dataSetCode;
+
+ private final IEntityProperty[] sampleProperties;
+
+ protected DatasetOwnerInformation(String dataSetCode, DataSetInformation dataSetOwner)
+ {
+ this(dataSetCode, dataSetOwner.tryToGetSample(), dataSetOwner.getSampleIdentifier(),
+ dataSetOwner.getProperties(), dataSetOwner.tryToGetExperiment(), dataSetOwner
+ .getExperimentIdentifier());
+ }
- public HCSImageFileAccepter(File imageFileRootDirectory, List<String> channelCodes)
+ private DatasetOwnerInformation(String dataSetCode, Sample sample,
+ SampleIdentifier sampleIdentifier, IEntityProperty[] sampleProperties,
+ Experiment experiment, ExperimentIdentifier experimentIdentifier)
{
- this.imageFileRootDirectory = imageFileRootDirectory;
- this.channelCodes = channelCodes;
+ this.dataSetCode = dataSetCode;
+ this.sample = sample;
+ this.sampleIdentifier = sampleIdentifier;
+ this.sampleProperties = sampleProperties;
+ this.experiment = experiment;
+ this.experimentIdentifier = experimentIdentifier;
}
- public final void accept(final int channel, final Location wellLocation,
- final Location tileLocation, final IFile imageFile)
+ public String getDataSetCode()
{
- final String imageRelativePath =
- FileUtilities.getRelativeFilePath(imageFileRootDirectory,
- new File(imageFile.getPath()));
- assert imageRelativePath != null : "Image relative path should not be null.";
- String channelCode = getChannelCodeOrLabel(channelCodes, channel);
- AcquiredSingleImage imageDesc =
- new AcquiredSingleImage(wellLocation, tileLocation, channelCode, null, null,
- null, new RelativeImageReference(imageRelativePath, null, null));
- images.add(imageDesc);
+ return dataSetCode;
}
- public List<AcquiredSingleImage> getImages()
+ public SampleIdentifier getSampleIdentifier()
{
- return images;
+ return sampleIdentifier;
+ }
+
+ public Sample tryGetSample()
+ {
+ return sample;
+ }
+
+ public IEntityProperty[] getSampleProperties()
+ {
+ return sampleProperties;
+ }
+
+ public Experiment tryGetExperiment()
+ {
+ return experiment;
+ }
+
+ public ExperimentIdentifier getExperimentIdentifier()
+ {
+ return experimentIdentifier;
}
}
- // adapts old-style image extractor to the new one which is stateless
- private static IImageFileExtractor adapt(
- final ch.systemsx.cisd.etlserver.IHCSImageFileExtractor extractor,
- final File imageFileRootDirectory, final List<ChannelDescription> descriptions,
- final Geometry tileGeometry)
+ public static class ImageDatasetOwnerInformation extends DatasetOwnerInformation
{
- return new IImageFileExtractor()
+ public static ImageDatasetOwnerInformation create(String containerDatasetPermId,
+ DataSetInformation originalDataset, String thumbnailDatasetPermIdOrNull)
+ {
+ return new ImageDatasetOwnerInformation(containerDatasetPermId, originalDataset,
+ thumbnailDatasetPermIdOrNull);
+ }
+
+ private final List<ImageZoomLevel> imageZoomLevels;
+
+ private ImageDatasetOwnerInformation(String containerDatasetPermId,
+ DataSetInformation originalDataset, String thumbnailDatasetPermIdOrNull)
+ {
+ super(containerDatasetPermId, originalDataset);
+ this.imageZoomLevels = createZoomLevels(originalDataset, thumbnailDatasetPermIdOrNull);
+ }
+
+ private static List<ImageZoomLevel> createZoomLevels(DataSetInformation originalDataset,
+ String thumbnailDatasetPermIdOrNull)
+ {
+ List<ImageZoomLevel> zoomLevels = new ArrayList<ImageZoomLevel>();
+ ImageZoomLevel originalZoomLevel =
+ new ImageZoomLevel(originalDataset.getDataSetCode(), true);
+ zoomLevels.add(originalZoomLevel);
+ if (thumbnailDatasetPermIdOrNull != null)
{
- public ImageFileExtractionResult extract(File incomingDataSetDirectory,
- DataSetInformation dataSetInformation)
- {
- HCSImageFileAccepter accepter =
- new HCSImageFileAccepter(imageFileRootDirectory,
- extractChannelCodes(descriptions));
- ch.systemsx.cisd.etlserver.HCSImageFileExtractionResult originalResult =
- extractor.process(
- NodeFactory.createDirectoryNode(incomingDataSetDirectory),
- dataSetInformation, accepter);
- List<Channel> channels = convert(originalResult.getChannels());
- return new ImageFileExtractionResult(accepter.getImages(),
- asRelativePaths(originalResult.getInvalidFiles()), channels,
- tileGeometry, true, null);
- }
-
- private List<Channel> convert(Set<ch.systemsx.cisd.bds.hcs.Channel> channels)
- {
- List<Channel> result = new ArrayList<Channel>();
- for (ch.systemsx.cisd.bds.hcs.Channel channel : channels)
- {
- String channelCode =
- getChannelCodeOrLabel(extractChannelCodes(descriptions),
- channel.getCounter());
- String channelLabel =
- getChannelCodeOrLabel(extractChannelLabels(descriptions),
- channel.getCounter());
- Channel convertedChannel = new Channel(channelCode, channelLabel);
- result.add(convertedChannel);
- }
- return result;
- }
-
- private List<File> asRelativePaths(List<IFile> files)
- {
- List<File> result = new ArrayList<File>();
- for (IFile file : files)
- {
- result.add(new File(file.getPath()));
- }
- return result;
- }
- };
+ ImageZoomLevel thumbnailZoomLevel =
+ new ImageZoomLevel(thumbnailDatasetPermIdOrNull, false);
+ zoomLevels.add(thumbnailZoomLevel);
+ }
+ return zoomLevels;
+ }
+
+ public List<ImageZoomLevel> getImageZoomLevels()
+ {
+ return imageZoomLevels;
+ }
}
@Override
- protected void validateImages(DataSetInformation dataSetInformation, IMailClient mailClient,
- File incomingDataSetDirectory, ImageFileExtractionResult extractionResult)
+ protected boolean validateImages(DatasetOwnerInformation dataSetInformation,
+ IMailClient mailClient, File incomingDataSetDirectory,
+ ImageFileExtractionResult extractionResult)
{
ImageValidator validator =
new ImageValidator(dataSetInformation, mailClient, incomingDataSetDirectory,
extractionResult, operationLog, notificationLog, notifyIfPlateIncomplete);
- validator.validateImages();
+ return validator.validateImages();
}
-
- @Override
- protected IImageFileExtractor tryGetImageFileExtractor(File incomingDataSetDirectory)
- {
- IImageFileExtractor extractor = imageFileExtractorOrNull;
- if (extractor == null && deprecatedImageFileExtractorOrNull != null)
- {
- List<ChannelDescription> channelDescriptions =
- AbstractImageFileExtractor.extractChannelDescriptions(properties);
- Geometry tileGeometry = AbstractImageFileExtractor.getMandatoryTileGeometry(properties);
- extractor =
- adapt(deprecatedImageFileExtractorOrNull, incomingDataSetDirectory,
- channelDescriptions, tileGeometry);
- }
- return extractor;
- }
-
- private void checkDataSetInformation(final DataSetInformation dataSetInformation)
+
+ private void checkDataSetInformation(final DatasetOwnerInformation dataSetInformation)
{
assert dataSetInformation != null : "Unspecified data set information";
assert dataSetInformation.getSampleIdentifier() != null : "Unspecified sample identifier";
@@ -218,7 +202,7 @@ public final class PlateStorageProcessor extends AbstractImageStorageProcessor
final ch.systemsx.cisd.openbis.generic.shared.dto.identifier.ExperimentIdentifier experimentIdentifier =
dataSetInformation.getExperimentIdentifier();
assert experimentIdentifier != null : "Unspecified experiment identifier";
- assert dataSetInformation.tryToGetExperiment() != null : "experiment not set";
+ assert dataSetInformation.tryGetExperiment() != null : "experiment not set";
checkExperimentIdentifier(experimentIdentifier);
}
@@ -231,12 +215,13 @@ public final class PlateStorageProcessor extends AbstractImageStorageProcessor
}
@Override
- protected void storeInDatabase(IImagingQueryDAO dao, DataSetInformation dataSetInformation,
+ protected void storeInDatabase(IImagingQueryDAO dao,
+ ImageDatasetOwnerInformation dataSetInformation,
ImageFileExtractionResult extractedImages)
{
checkDataSetInformation(dataSetInformation);
- Experiment experiment = dataSetInformation.tryToGetExperiment();
+ Experiment experiment = dataSetInformation.tryGetExperiment();
assert experiment != null : "experiment is null";
List<AcquiredSingleImage> images = extractedImages.getImages();
boolean storeChannelsOnExperimentLevel = globalStoreChannelsOnExperimentLevel;
@@ -253,16 +238,18 @@ public final class PlateStorageProcessor extends AbstractImageStorageProcessor
}
private HCSImageDatasetInfo createImageDatasetInfo(Experiment experiment,
- DataSetInformation dataSetInformation, List<AcquiredSingleImage> acquiredImages,
- Geometry tileGeometry, ImageLibraryInfo imageLibraryInfoOrNull,
- boolean storeChannelsOnExperimentLevel)
+ ImageDatasetOwnerInformation dataSetInformation,
+ List<AcquiredSingleImage> acquiredImages, Geometry tileGeometry,
+ ImageLibraryInfo imageLibraryInfoOrNull, boolean storeChannelsOnExperimentLevel)
{
HCSContainerDatasetInfo info =
HCSContainerDatasetInfo.createScreeningDatasetInfo(dataSetInformation);
boolean hasImageSeries = hasImageSeries(acquiredImages);
ImageDatasetInfo imageDatasetInfo =
new ImageDatasetInfo(tileGeometry.getRows(), tileGeometry.getColumns(),
- hasImageSeries, imageLibraryInfoOrNull);
+ hasImageSeries, imageLibraryInfoOrNull,
+ dataSetInformation.getImageZoomLevels());
return new HCSImageDatasetInfo(info, imageDatasetInfo, storeChannelsOnExperimentLevel);
}
+
}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/RelativeImageReference.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/RelativeImageReference.java
index 2b3cd563fad..322134855b1 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/RelativeImageReference.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/RelativeImageReference.java
@@ -34,7 +34,7 @@ public class RelativeImageReference extends AbstractImageReference
this.imageRelativePath = relativePath;
}
- public String getRelativeImagePath()
+ public String getImageRelativePath()
{
return imageRelativePath;
}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/IImagingQueryDAO.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/IImagingQueryDAO.java
index 35240cb9d09..74f282d0140 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/IImagingQueryDAO.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/IImagingQueryDAO.java
@@ -35,6 +35,7 @@ import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgFe
import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgImageDTO;
import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgImageDatasetDTO;
import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgImageTransformationDTO;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgImageZoomLevelDTO;
import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgSpotDTO;
/**
@@ -89,6 +90,10 @@ public interface IImagingQueryDAO extends TransactionQuery, IImagingReadonlyQuer
+ "?{1.imageLibraryName}, ?{1.imageReaderName}) returning ID")
public long addImageDataset(ImgImageDatasetDTO dataset);
+ @Select("insert into image_zoom_levels (physical_dataset_perm_id, is_original, container_dataset_id) "
+ + "values(?{1.physicalDatasetPermId}, ?{1.isOriginal}, ?{1.containerDatasetId}) returning ID")
+ public long addImageZoomLevel(ImgImageZoomLevelDTO dataset);
+
@Select("insert into ANALYSIS_DATA_SETS (PERM_ID, CONT_ID) "
+ "values(?{1.permId}, ?{1.containerId}) returning ID")
public long addAnalysisDataset(ImgAnalysisDatasetDTO dataset);
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/ImagingDatasetLoader.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/ImagingDatasetLoader.java
index ac98f2c1738..b62240ceff7 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/ImagingDatasetLoader.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/ImagingDatasetLoader.java
@@ -96,6 +96,12 @@ public class ImagingDatasetLoader extends HCSDatasetLoader implements IImagingDa
{
// get the image content from the original image
imageDTO = tryGetOriginalImage(chosenChannelId, channelStackReference, datasetId);
+ // Thumbnail is not required, but original image is not there.
+ // We have to fetch the thumbnail anyway.
+ if (imageDTO == null && imageSize.isThumbnailRequired() == false)
+ {
+ imageDTO = tryGetThumbnail(chosenChannelId, channelStackReference, datasetId);
+ }
}
if (imageDTO == null)
{
@@ -259,20 +265,23 @@ public class ImagingDatasetLoader extends HCSDatasetLoader implements IImagingDa
HCSChannelStackByLocationReference hcsRef = channelStackReference.tryGetHCSChannelStack();
MicroscopyChannelStackByLocationReference micRef =
channelStackReference.tryGetMicroscopyChannelStack();
+ ImgImageDTO image;
if (hcsRef != null)
{
- return query.tryGetHCSImage(channelId, datasetId, hcsRef.getTileLocation(),
- hcsRef.getWellLocation());
+ image =
+ query.tryGetHCSImage(channelId, datasetId, hcsRef.getTileLocation(),
+ hcsRef.getWellLocation());
} else if (micRef != null)
{
- return query.tryGetMicroscopyImage(channelId, datasetId, micRef.getTileLocation());
+ image = query.tryGetMicroscopyImage(channelId, datasetId, micRef.getTileLocation());
} else
{
Long channelStackId = channelStackReference.tryGetChannelStackId();
assert channelStackId != null : "invalid specification of the channel stack: "
+ channelStackReference;
- return query.tryGetImage(channelId, channelStackId, datasetId);
+ image = query.tryGetImage(channelId, channelStackId, datasetId);
}
+ return checkAccessability(image);
}
private ImgImageDTO tryGetThumbnail(long channelId,
@@ -281,19 +290,44 @@ public class ImagingDatasetLoader extends HCSDatasetLoader implements IImagingDa
HCSChannelStackByLocationReference hcsRef = channelStackReference.tryGetHCSChannelStack();
MicroscopyChannelStackByLocationReference micRef =
channelStackReference.tryGetMicroscopyChannelStack();
+ ImgImageDTO image;
if (hcsRef != null)
{
- return query.tryGetHCSThumbnail(channelId, datasetId, hcsRef.getTileLocation(),
- hcsRef.getWellLocation());
+ image =
+ query.tryGetHCSThumbnail(channelId, datasetId, hcsRef.getTileLocation(),
+ hcsRef.getWellLocation());
} else if (micRef != null)
{
- return query.tryGetMicroscopyThumbnail(channelId, datasetId, micRef.getTileLocation());
+ image = query.tryGetMicroscopyThumbnail(channelId, datasetId, micRef.getTileLocation());
} else
{
Long channelStackId = channelStackReference.tryGetChannelStackId();
assert channelStackId != null : "invalid specification of the channel stack: "
+ channelStackReference;
- return query.tryGetThumbnail(channelId, channelStackId, datasetId);
+ image = query.tryGetThumbnail(channelId, channelStackId, datasetId);
+ }
+ return checkAccessability(image);
+ }
+
+ private ImgImageDTO checkAccessability(ImgImageDTO image)
+ {
+ return isFileAccessible(image) ? image : null;
+ }
+
+ private boolean isFileAccessible(ImgImageDTO image)
+ {
+ String filePath = image.getFilePath();
+ try
+ {
+ content.getNode(filePath);
+ return true;
+ } catch (IllegalArgumentException ex)
+ {
+ operationLog
+ .warn(String
+ .format("Path '%s' is unaccessible, probably the dataset has been deleted and the imaging db is not yet synchronized.",
+ filePath));
+ return false;
}
}
@@ -326,7 +360,7 @@ public class ImagingDatasetLoader extends HCSDatasetLoader implements IImagingDa
query.tryGetHCSRepresentativeImage(datasetId, wellLocationOrNull, channelId);
}
}
- return image;
+ return checkAccessability(image);
}
public AbsoluteImageReference tryFindAnyOriginalImage()
@@ -373,14 +407,17 @@ public class ImagingDatasetLoader extends HCSDatasetLoader implements IImagingDa
private ImgImageDTO tryGetRepresentativeThumbnailImageDTO(long channelId,
Location wellLocationOrNull)
{
+ ImgImageDTO image;
if (wellLocationOrNull == null)
{
- return query.tryGetMicroscopyRepresentativeThumbnail(dataset.getId(), channelId);
+ image = query.tryGetMicroscopyRepresentativeThumbnail(dataset.getId(), channelId);
} else
{
- return query.tryGetHCSRepresentativeThumbnail(dataset.getId(), wellLocationOrNull,
- channelId);
+ image =
+ query.tryGetHCSRepresentativeThumbnail(dataset.getId(), wellLocationOrNull,
+ channelId);
}
+ return checkAccessability(image);
}
public AbsoluteImageReference tryFindAnyThumbnail()
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/ImageDatasetInfo.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/ImageDatasetInfo.java
index 9638540a5b0..1ae45a9b9ea 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/ImageDatasetInfo.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/ImageDatasetInfo.java
@@ -16,6 +16,8 @@
package ch.systemsx.cisd.openbis.dss.etl.dto;
+import java.util.List;
+
/**
* Information about the dataset specific for the image datasets (hcs and microscopy).
@@ -31,13 +33,16 @@ public class ImageDatasetInfo
private final ImageLibraryInfo imageLibraryOrNull;
+ private final List<ImageZoomLevel> imageZoomLevels;
+
public ImageDatasetInfo(int tileRows, int tileColumns, boolean hasImageSeries,
- ImageLibraryInfo imageLibraryOrNull)
+ ImageLibraryInfo imageLibraryOrNull, List<ImageZoomLevel> imageZoomLevels)
{
this.tileRows = tileRows;
this.tileColumns = tileColumns;
this.hasImageSeries = hasImageSeries;
this.imageLibraryOrNull = imageLibraryOrNull;
+ this.imageZoomLevels = imageZoomLevels;
}
public int getTileRows()
@@ -59,4 +64,9 @@ public class ImageDatasetInfo
{
return imageLibraryOrNull;
}
+
+ public List<ImageZoomLevel> getImageZoomLevels()
+ {
+ return imageZoomLevels;
+ }
}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/ImageZoomLevel.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/ImageZoomLevel.java
new file mode 100644
index 00000000000..46844c2b064
--- /dev/null
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/ImageZoomLevel.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.etl.dto;
+
+/**
+ * Describes an image dataset with 1. all images resized in the same way or 2. original images.
+ *
+ * @author Tomasz Pylak
+ */
+public class ImageZoomLevel
+{
+ private final String physicalDatasetPermId;
+
+ private final boolean isOriginal;
+
+ public ImageZoomLevel(String physicalDatasetPermId, boolean isOriginal)
+ {
+ this.physicalDatasetPermId = physicalDatasetPermId;
+ this.isOriginal = isOriginal;
+ }
+
+ public String getPhysicalDatasetPermId()
+ {
+ return physicalDatasetPermId;
+ }
+
+ public boolean isOriginal()
+ {
+ return isOriginal;
+ }
+
+}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/RelativeImageFile.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/RelativeImageFile.java
new file mode 100644
index 00000000000..53df7a99792
--- /dev/null
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/RelativeImageFile.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.etl.dto;
+
+import ch.systemsx.cisd.common.utilities.AbstractHashable;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageFileInfo;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageIdentifier;
+
+/**
+ * Points to one image on the file system.
+ *
+ * @author Tomasz Pylak
+ */
+public class RelativeImageFile extends AbstractHashable
+{
+ private final String imageRelativePath;
+
+ private final String imageIDOrNull;
+
+ public static RelativeImageFile create(ImageFileInfo imageFileInfo)
+ {
+ ImageIdentifier imageIdentifier = imageFileInfo.tryGetImageIdentifier();
+ String imageIDOrNull =
+ (imageIdentifier == null) ? null : imageIdentifier.getUniqueStringIdentifier();
+ return new RelativeImageFile(imageFileInfo.getImageRelativePath(), imageIDOrNull);
+ }
+
+ public RelativeImageFile(String imageRelativePath, String imageIDOrNull)
+ {
+ assert imageRelativePath != null : "imageRelativePath is null";
+
+ this.imageRelativePath = imageRelativePath;
+ this.imageIDOrNull = imageIDOrNull;
+ }
+
+ /** Path relative to the folder with all the images. */
+ public String getImageRelativePath()
+ {
+ return imageRelativePath;
+ }
+
+ /**
+ * Not null if the file pointed by {@link #getImageRelativePath()} is an image container with
+ * potentially many images inside. In this case image id determines the image in the container.
+ */
+ public String tryGetImageID()
+ {
+ return imageIDOrNull;
+ }
+}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/ImageDataSetInformation.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/ImageDataSetInformation.java
new file mode 100644
index 00000000000..02ede10eda7
--- /dev/null
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/ImageDataSetInformation.java
@@ -0,0 +1,120 @@
+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.etl.dto.api.impl;
+
+import java.io.File;
+
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.BasicDataSetInformation;
+import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
+import ch.systemsx.cisd.openbis.generic.shared.IServer;
+
+/**
+ * Extends {@link DataSetInformation} with information about images needed in HCS/Microscopy.
+ *
+ * @author Tomasz Pylak
+ */
+public class ImageDataSetInformation extends BasicDataSetInformation
+{
+ private static final long serialVersionUID = IServer.VERSION;
+
+ private File incomingDirectory;
+
+ /**
+ * Path to the incoming folder with images, relative to the dataset directory. E.g. if the
+ * incoming folder name is X and the transaction's dataset registration code put it inside
+ * 'original' folder, then this path points to "original/X'.
+ */
+ private File datasetRelativeImagesFolderPath;
+
+ private ImageDataSetStructure imageDataSetStructure;
+
+ private ThumbnailFilePaths thumbnailFilePathsOrNull;
+
+ private String containerDatasetPermId;
+
+ public File getIncomingDirectory()
+ {
+ return incomingDirectory;
+ }
+
+ public String getContainerDatasetPermId()
+ {
+ return containerDatasetPermId;
+ }
+
+ /** @returns null if no thumbnails will be stored */
+ public ThumbnailFilePaths tryGetThumbnailFilePaths()
+ {
+ return thumbnailFilePathsOrNull;
+ }
+
+ public ImageDataSetStructure getImageDataSetStructure()
+ {
+ return imageDataSetStructure;
+ }
+
+ public File getDatasetRelativeImagesFolderPath()
+ {
+ return datasetRelativeImagesFolderPath;
+ }
+
+ /** Sets the folder where all original images are located initially. */
+ public void setIncomingDirectory(File incomingDirectory)
+ {
+ this.incomingDirectory = incomingDirectory;
+ }
+
+ public void setDatasetRelativeImagesFolderPath(File datasetRelativeImagesFolderPath)
+ {
+ this.datasetRelativeImagesFolderPath = datasetRelativeImagesFolderPath;
+ }
+
+ public void setContainerDatasetPermId(String containerDatasetPermId)
+ {
+ this.containerDatasetPermId = containerDatasetPermId;
+ }
+
+ public void setThumbnailFilePaths(ThumbnailFilePaths thumbnailFilePathsOrNull)
+ {
+ this.thumbnailFilePathsOrNull = thumbnailFilePathsOrNull;
+ }
+
+ public void setImageDataSetStructure(ImageDataSetStructure imageDataSetStructure)
+ {
+ this.imageDataSetStructure = imageDataSetStructure;
+ }
+
+ // -------
+
+ @Override
+ public String toString()
+ {
+ final StringBuilder buffer = new StringBuilder(super.toString());
+ appendNameAndObject(buffer, "images structure", imageDataSetStructure.toString());
+ appendNameAndObject(buffer, "container dataset", containerDatasetPermId);
+ appendNameAndObject(buffer, "original dataset", this.getDataSetCode());
+ if (this.tryGetThumbnailFilePaths() != null)
+ {
+ appendNameAndObject(buffer, "thumbnail", this.tryGetThumbnailFilePaths());
+ } else
+ {
+ appendNameAndObject(buffer, "thumbnail", "none");
+ }
+ return buffer.toString();
+ }
+
+}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/ImageDataSetInformation.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/ImageDataSetStructure.java
similarity index 75%
rename from screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/ImageDataSetInformation.java
rename to screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/ImageDataSetStructure.java
index c1e40ad4f2f..2c48a25dda8 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/ImageDataSetInformation.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/ImageDataSetStructure.java
@@ -14,23 +14,25 @@
* limitations under the License.
*/
-package ch.systemsx.cisd.openbis.dss.etl.dto.api.v1;
+package ch.systemsx.cisd.openbis.dss.etl.dto.api.impl;
import java.util.List;
import ch.systemsx.cisd.common.collections.CollectionUtils;
-import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
-import ch.systemsx.cisd.openbis.generic.shared.IServer;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.Channel;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ChannelColorComponent;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageFileInfo;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageStorageConfiguraton;
+import ch.systemsx.cisd.openbis.dss.generic.shared.utils.ToStringUtil;
/**
- * Extends {@link DataSetInformation} with information about images needed in HCS/Microscopy.
+ * Information about images needed in HCS/Microscopy. Does not contain information about datasets
+ * entities and their metadata.
*
* @author Tomasz Pylak
*/
-public class ImageDataSetInformation extends BasicDataSetInformation
+public class ImageDataSetStructure
{
- private static final long serialVersionUID = IServer.VERSION;
-
private List<ImageFileInfo> images;
private List<Channel> channels;
@@ -75,6 +77,11 @@ public class ImageDataSetInformation extends BasicDataSetInformation
return imageStorageConfiguratonOrNull;
}
+ public boolean areThumbnailsGenerated()
+ {
+ return getImageStorageConfiguraton().getThumbnailsStorageFormat() != null;
+ }
+
// ------ setters
/** Sets location of the tile (a.k.a. filed or side) on the 'well matrix'. */
@@ -122,8 +129,6 @@ public class ImageDataSetInformation extends BasicDataSetInformation
this.imageStorageConfiguratonOrNull = imageStorageConfiguratonOrNull;
}
- // -------
-
/** are all necessary fields filled? */
public boolean isValid()
{
@@ -133,12 +138,13 @@ public class ImageDataSetInformation extends BasicDataSetInformation
@Override
public String toString()
{
- final StringBuilder buffer = new StringBuilder(super.toString());
- appendNameAndObject(buffer, "config", imageStorageConfiguratonOrNull);
- appendNameAndObject(buffer, "tile", tileRowsNumber + "x" + tileColumnsNumber);
- appendNameAndObject(buffer, "channels", CollectionUtils.abbreviate(channels, -1));
- appendNameAndObject(buffer, "number of images", images.size());
+ final StringBuilder buffer = new StringBuilder();
+ ToStringUtil.appendNameAndObject(buffer, "config", imageStorageConfiguratonOrNull);
+ ToStringUtil.appendNameAndObject(buffer, "tile", tileRowsNumber + "x" + tileColumnsNumber);
+ ToStringUtil.appendNameAndObject(buffer, "channels",
+ CollectionUtils.abbreviate(channels, -1));
+ ToStringUtil.appendNameAndObject(buffer, "number of images", images.size());
return buffer.toString();
}
-}
+}
\ No newline at end of file
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/ThumbnailFilePaths.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/ThumbnailFilePaths.java
new file mode 100644
index 00000000000..1cb5d037e62
--- /dev/null
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/ThumbnailFilePaths.java
@@ -0,0 +1,56 @@
+package ch.systemsx.cisd.openbis.dss.etl.dto.api.impl;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import ch.systemsx.cisd.openbis.dss.etl.dto.RelativeImageFile;
+import ch.systemsx.cisd.openbis.dss.generic.shared.utils.ToStringUtil;
+
+/**
+ * Stores the mapping between image and their thumbnails paths. Thread-safe class.
+ *
+ * @author Tomasz Pylak
+ */
+public class ThumbnailFilePaths
+{
+ private final Map<RelativeImageFile, String> imageToThumbnailPathMap;
+
+ private final String thumbnailPhysicalDatasetPermId;
+
+ public ThumbnailFilePaths(String thumbnailPhysicalDatasetPermId)
+ {
+ assert thumbnailPhysicalDatasetPermId != null : "thumbnailPhysicalDatasetPermId is null";
+
+ this.thumbnailPhysicalDatasetPermId = thumbnailPhysicalDatasetPermId;
+ this.imageToThumbnailPathMap = new HashMap<RelativeImageFile, String>();
+ }
+
+ /**
+ * Saves the path to the thumbnail for the specified image. The thumbnail path is relative and
+ * starts with the name of the thumbnail file.
+ */
+ public synchronized void saveThumbnailPath(RelativeImageFile image, String thumbnailRelativePath)
+ {
+ imageToThumbnailPathMap.put(image, thumbnailRelativePath);
+ }
+
+ public synchronized String getThumbnailPath(RelativeImageFile image)
+ {
+ return imageToThumbnailPathMap.get(image);
+ }
+
+ public String getThumbnailPhysicalDatasetPermId()
+ {
+ return thumbnailPhysicalDatasetPermId;
+ }
+
+ @Override
+ public String toString()
+ {
+ final StringBuilder buffer = new StringBuilder();
+ ToStringUtil.appendNameAndObject(buffer, "number of thumbnails",
+ imageToThumbnailPathMap.size());
+ ToStringUtil.appendNameAndObject(buffer, "dataset perm id", thumbnailPhysicalDatasetPermId);
+ return buffer.toString();
+ }
+}
\ No newline at end of file
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/IImagingDataSetRegistrationTransaction.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/IImagingDataSetRegistrationTransaction.java
new file mode 100644
index 00000000000..ac07922e63e
--- /dev/null
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/IImagingDataSetRegistrationTransaction.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.etl.dto.api.v1;
+
+import java.io.File;
+
+import ch.systemsx.cisd.etlserver.registrator.DataSetRegistrationDetails;
+import ch.systemsx.cisd.etlserver.registrator.api.v1.IDataSet;
+import ch.systemsx.cisd.etlserver.registrator.api.v1.IDataSetRegistrationTransaction;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.ImageDataSetInformation;
+
+/**
+ * Interface for imaging data set registration transaction. See
+ * {@link IDataSetRegistrationTransaction}.
+ *
+ * @author Tomasz Pylak
+ */
+public interface IImagingDataSetRegistrationTransaction extends IDataSetRegistrationTransaction
+{
+ /**
+ * Creates a new image dataset. See {@link SimpleImageDataConfig} documentation for
+ * configuration details.
+ */
+ IDataSet createNewImageDataSet(SimpleImageDataConfig imageDataSet, File incomingFolderWithImages);
+
+ @Deprecated
+ IDataSet createNewImageDataSet(
+ DataSetRegistrationDetails<ImageDataSetInformation> imageRegistrationDetails);
+
+}
\ No newline at end of file
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/IImagingDatasetFactory.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/IImagingDatasetFactory.java
index 138da241a34..2c0fcff7b0f 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/IImagingDatasetFactory.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/IImagingDatasetFactory.java
@@ -24,6 +24,7 @@ import ch.systemsx.cisd.common.exceptions.UserFailureException;
import ch.systemsx.cisd.etlserver.registrator.DataSetRegistrationDetails;
import ch.systemsx.cisd.etlserver.registrator.DataSetRegistrationService;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.FeatureVectorDataSetInformation;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.ImageDataSetInformation;
/**
* Factory for defining image and feature vector datasets.
@@ -39,6 +40,7 @@ public interface IImagingDatasetFactory
* 2. {@link SimpleImageDataConfig} is not sufficient to describe the dataset and further
* modifications on the dataset detail object are required.
*/
+ @Deprecated
DataSetRegistrationDetails<ImageDataSetInformation> createImageRegistrationDetails(
SimpleImageDataConfig imageDataSet, File incomingDatasetFolder);
@@ -55,20 +57,21 @@ public interface IImagingDatasetFactory
*
* @param imageDatasetDetails advanced dataset specification
*/
+ @Deprecated
boolean registerImageDataset(
DataSetRegistrationDetails<ImageDataSetInformation> imageDatasetDetails,
File incomingDatasetFolder, DataSetRegistrationService<ImageDataSetInformation> service);
/**
* Allows to define feature vectors of one image analysis dataset. Used to define a dataset
- * details with {@link #createFeatureVectorRegistrationDetails(IFeaturesBuilder)}.
+ * details with {@link #createFeatureVectorDatasetDetails(IFeaturesBuilder)}.
*/
IFeaturesBuilder createFeaturesBuilder();
/**
* Creates feature vector dataset details by using the specified builder.
*/
- DataSetRegistrationDetails<FeatureVectorDataSetInformation> createFeatureVectorRegistrationDetailsNew(
+ DataSetRegistrationDetails<FeatureVectorDataSetInformation> createFeatureVectorDatasetDetails(
IFeaturesBuilder featureBuilder);
/**
@@ -89,7 +92,7 @@ public interface IImagingDatasetFactory
*
* @throws IOException if file cannot be parsed
*/
- DataSetRegistrationDetails<FeatureVectorDataSetInformation> createFeatureVectorRegistrationDetails(
+ DataSetRegistrationDetails<FeatureVectorDataSetInformation> createFeatureVectorDatasetDetails(
String csvFilePath, Properties properties) throws IOException;
/**
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/SimpleImageDataConfig.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/SimpleImageDataConfig.java
index d56c35821a6..e58045e06e8 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/SimpleImageDataConfig.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/SimpleImageDataConfig.java
@@ -187,7 +187,7 @@ abstract public class SimpleImageDataConfig
// --- auxiliary structures ----------------------------------------------
- private String datasetTypeCode;
+ private String mainDatasetTypeCode;
private String fileFormatCode = ScreeningConstants.UNKNOWN_FILE_FORMAT;
@@ -613,7 +613,7 @@ abstract public class SimpleImageDataConfig
/** Sets the type of the dataset. */
public void setDataSetType(String datasetTypeCode)
{
- this.datasetTypeCode = datasetTypeCode;
+ this.mainDatasetTypeCode = datasetTypeCode;
}
/** Sets the file type of the dataset. */
@@ -642,7 +642,7 @@ abstract public class SimpleImageDataConfig
public String getDataSetType()
{
- return datasetTypeCode;
+ return mainDatasetTypeCode;
}
public String getFileFormatType()
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/featurevector/FeatureVectorStorageProcessor.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/featurevector/FeatureVectorStorageProcessor.java
index 0ae11404a55..c1191f5e669 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/featurevector/FeatureVectorStorageProcessor.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/featurevector/FeatureVectorStorageProcessor.java
@@ -37,7 +37,7 @@ import ch.systemsx.cisd.openbis.dss.etl.HCSContainerDatasetInfo;
import ch.systemsx.cisd.openbis.dss.etl.dataaccess.IImagingQueryDAO;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.FeatureDefinition;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.FeatureVectorDataSetInformation;
-import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageDataSetInformation;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.ImageDataSetInformation;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/genedata/FeatureStorageProcessor.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/genedata/FeatureStorageProcessor.java
index 2b0bcc844de..75344de65ec 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/genedata/FeatureStorageProcessor.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/genedata/FeatureStorageProcessor.java
@@ -166,7 +166,6 @@ public class FeatureStorageProcessor extends AbstractDelegatingStorageProcessor
return ServiceProvider.getOpenBISService();
}
-
@Override
public IStorageProcessorTransaction createTransaction(
StorageProcessorTransactionParameters parameters)
@@ -176,8 +175,7 @@ public class FeatureStorageProcessor extends AbstractDelegatingStorageProcessor
}
protected void transform(IImagingQueryDAO dataAccessObject, File originalDataSet,
- File targetFolderForTransformedDataSet,
- DataSetInformation dataSetInformation)
+ File targetFolderForTransformedDataSet, DataSetInformation dataSetInformation)
{
List<String> lines = FileUtilities.loadToStringList(originalDataSet);
if (lines.isEmpty())
@@ -202,8 +200,7 @@ public class FeatureStorageProcessor extends AbstractDelegatingStorageProcessor
}
private void loadDataSetIntoDatabase(IImagingQueryDAO dataAccessObject, List<String> lines,
- DataSetInformation dataSetInformation)
- throws IOException
+ DataSetInformation dataSetInformation) throws IOException
{
GenedataFormatToCanonicalFeatureVector convertor =
new GenedataFormatToCanonicalFeatureVector(lines, LAYER_PREFIX);
@@ -215,8 +212,7 @@ public class FeatureStorageProcessor extends AbstractDelegatingStorageProcessor
uploader.uploadFeatureVectors(fvecs);
}
- private HCSContainerDatasetInfo createScreeningDatasetInfo(
- DataSetInformation dataSetInformation)
+ private HCSContainerDatasetInfo createScreeningDatasetInfo(DataSetInformation dataSetInformation)
{
Sample sampleOrNull = tryFindSampleForDataSet(dataSetInformation);
if (sampleOrNull == null)
@@ -225,8 +221,8 @@ public class FeatureStorageProcessor extends AbstractDelegatingStorageProcessor
"Cannot find a sample to which a plate should be (directly or indirectly) connected: "
+ dataSetInformation);
}
- return HCSContainerDatasetInfo.createScreeningDatasetInfoWithSample(
- dataSetInformation, sampleOrNull);
+ return HCSContainerDatasetInfo.createScreeningDatasetInfoWithSample(dataSetInformation,
+ sampleOrNull);
}
private Sample tryFindSampleForDataSet(DataSetInformation dataSetInformation)
@@ -332,7 +328,7 @@ public class FeatureStorageProcessor extends AbstractDelegatingStorageProcessor
return new UserFailureException("Error in line " + lineIndex + 1 + ": " + reason + ": "
+ line);
}
-
+
private static class FeatureStorageProcessorTransaction extends
AbstractDelegatingStorageProcessorTransaction
{
@@ -352,14 +348,12 @@ public class FeatureStorageProcessor extends AbstractDelegatingStorageProcessor
@Override
protected File executeStoreData(ITypeExtractor typeExtractor, IMailClient mailClient)
{
- nestedTransaction
- .storeData(typeExtractor, mailClient, incomingDataSetDirectory);
+ nestedTransaction.storeData(typeExtractor, mailClient, incomingDataSetDirectory);
dataAccessObject = processor.createDAO();
File storedDataSet = nestedTransaction.getStoredDataDirectory();
File originalDir = DefaultStorageProcessor.getOriginalDirectory(storedDataSet);
- File targetFile =
- new File(originalDir, incomingDataSetDirectory.getName());
+ File targetFile = new File(originalDir, incomingDataSetDirectory.getName());
processor.transform(dataAccessObject, targetFile, storedDataSet, dataSetInformation);
return nestedTransaction.getStoredDataDirectory();
}
@@ -399,6 +393,5 @@ public class FeatureStorageProcessor extends AbstractDelegatingStorageProcessor
dataAccessObject = null;
}
}
-
}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/genedata/HCSImageFileExtractor.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/genedata/HCSImageFileExtractor.java
deleted file mode 100644
index aeb3ddefbb1..00000000000
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/genedata/HCSImageFileExtractor.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Copyright 2010 ETH Zuerich, CISD
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package ch.systemsx.cisd.openbis.dss.etl.genedata;
-
-import java.util.Properties;
-
-import ch.rinn.restrictions.Private;
-import ch.systemsx.cisd.bds.hcs.Geometry;
-import ch.systemsx.cisd.bds.hcs.Location;
-import ch.systemsx.cisd.bds.storage.IDirectory;
-import ch.systemsx.cisd.etlserver.HCSImageFileExtractionResult;
-import ch.systemsx.cisd.etlserver.IHCSImageFileAccepter;
-import ch.systemsx.cisd.etlserver.IHCSImageFileExtractor;
-import ch.systemsx.cisd.etlserver.plugins.AbstractHCSImageFileExtractor;
-import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
-
-/**
- * @author Franz-Josef Elmer
- */
-public class HCSImageFileExtractor extends AbstractHCSImageFileExtractor implements
- IHCSImageFileExtractor
-{
- public HCSImageFileExtractor(final Properties properties)
- {
- super(properties);
- }
-
- @Private
- HCSImageFileExtractor(Geometry wellGeometry)
- {
- super(wellGeometry);
- }
-
- public HCSImageFileExtractionResult process(IDirectory incomingDataSetDirectory,
- DataSetInformation dataSetInformation, IHCSImageFileAccepter accepter)
- {
- assert incomingDataSetDirectory != null;
- return process(incomingDataSetDirectory.listFiles(null, false), dataSetInformation,
- accepter);
- }
-
- @Override
- protected String[] tryToSplitIntoTokens(String imageFileName)
- {
- String[] tokens = new String[4];
- int indexOfChannelSeparator = imageFileName.indexOf('-');
- if (indexOfChannelSeparator < 0)
- {
- return null;
- }
- tokens[3] = imageFileName.substring(indexOfChannelSeparator + 1);
- String wellAndTileCoordinates = imageFileName.substring(0, indexOfChannelSeparator);
- if (wellAndTileCoordinates.length() != 9)
- {
- return null;
- }
- tokens[1] = wellAndTileCoordinates.substring(0, 6);
- tokens[2] = wellAndTileCoordinates.substring(6);
- return tokens;
- }
-
- @Override
- protected int getChannelWavelength(String channel)
- {
- try
- {
- return Integer.parseInt(channel) + 1;
- } catch (NumberFormatException ex)
- {
- return 0;
- }
- }
-
- @Override
- protected Location tryGetWellLocation(String wellLocation)
- {
- return new Location(1, 1);
- }
-
- @Override
- protected Location tryGetPlateLocation(String plateLocation)
- {
- try
- {
- int row = Integer.parseInt(plateLocation.substring(0, 3));
- int column = Integer.parseInt(plateLocation.substring(3));
- return new Location(column, row);
- } catch (NumberFormatException ex)
- {
- return null;
- }
- }
-
-}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/JythonPlateDataSetHandler.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/JythonPlateDataSetHandler.java
index 1ea87917f7e..1d0731ee386 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/JythonPlateDataSetHandler.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/JythonPlateDataSetHandler.java
@@ -4,6 +4,7 @@ import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.List;
import java.util.Properties;
@@ -11,6 +12,8 @@ import org.python.util.PythonInterpreter;
import ch.systemsx.cisd.bds.hcs.Location;
import ch.systemsx.cisd.common.exceptions.UserFailureException;
+import ch.systemsx.cisd.common.utilities.IDelegatedActionWithResult;
+import ch.systemsx.cisd.etlserver.ITopLevelDataSetRegistratorDelegate;
import ch.systemsx.cisd.etlserver.TopLevelDataSetRegistratorGlobalState;
import ch.systemsx.cisd.etlserver.registrator.DataSetRegistrationDetails;
import ch.systemsx.cisd.etlserver.registrator.DataSetRegistrationService;
@@ -18,13 +21,19 @@ import ch.systemsx.cisd.etlserver.registrator.IDataSetRegistrationDetailsFactory
import ch.systemsx.cisd.etlserver.registrator.JythonTopLevelDataSetHandler;
import ch.systemsx.cisd.etlserver.registrator.api.v1.IDataSet;
import ch.systemsx.cisd.etlserver.registrator.api.v1.impl.DataSetRegistrationTransaction;
+import ch.systemsx.cisd.openbis.dss.Constants;
+import ch.systemsx.cisd.openbis.dss.etl.Hdf5ThumbnailGenerator;
import ch.systemsx.cisd.openbis.dss.etl.PlateGeometryOracle;
+import ch.systemsx.cisd.openbis.dss.etl.dto.RelativeImageFile;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.FeatureDefinition;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.FeatureVectorDataSetInformation;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.FeaturesBuilder;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.ImageDataSetInformation;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.ImageDataSetStructure;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.ThumbnailFilePaths;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.IFeaturesBuilder;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.IImagingDataSetRegistrationTransaction;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.IImagingDatasetFactory;
-import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageDataSetInformation;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageFileInfo;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.SimpleImageDataConfig;
import ch.systemsx.cisd.openbis.dss.etl.featurevector.CsvFeatureVectorParser;
@@ -135,8 +144,10 @@ public class JythonPlateDataSetHandler extends JythonTopLevelDataSetHandler<Data
public String figureGeometry(
DataSetRegistrationDetails<ImageDataSetInformation> registrationDetails)
{
- List<Location> locations =
- extractLocations(registrationDetails.getDataSetInformation().getImages());
+ List<ImageFileInfo> images =
+ registrationDetails.getDataSetInformation().getImageDataSetStructure()
+ .getImages();
+ List<Location> locations = extractLocations(images);
List<String> plateGeometries =
loadPlateGeometries(registratorState.getGlobalState().getOpenBisService());
return PlateGeometryOracle.figureGeometry(locations, plateGeometries);
@@ -171,22 +182,7 @@ public class JythonPlateDataSetHandler extends JythonTopLevelDataSetHandler<Data
return new FeaturesBuilder();
}
- /**
- * This method exists just for backward compatibility. It used to have the second parameter,
- * which is now ignored.
- *
- * @deprecated use {@link #createFeatureVectorRegistrationDetails(IFeaturesBuilder)}
- * instead.
- */
- @SuppressWarnings("unused")
- @Deprecated
- public DataSetRegistrationDetails<FeatureVectorDataSetInformation> createFeatureVectorRegistrationDetails(
- IFeaturesBuilder featureBuilder, Object incomingDatasetFolder)
- {
- return createFeatureVectorRegistrationDetailsNew(featureBuilder);
- }
-
- public DataSetRegistrationDetails<FeatureVectorDataSetInformation> createFeatureVectorRegistrationDetailsNew(
+ public DataSetRegistrationDetails<FeatureVectorDataSetInformation> createFeatureVectorDatasetDetails(
IFeaturesBuilder featureBuilder)
{
FeaturesBuilder myFeatureBuilder = (FeaturesBuilder) featureBuilder;
@@ -212,7 +208,7 @@ public class JythonPlateDataSetHandler extends JythonTopLevelDataSetHandler<Data
*
* @throws IOException if file cannot be parsed
*/
- public DataSetRegistrationDetails<FeatureVectorDataSetInformation> createFeatureVectorRegistrationDetails(
+ public DataSetRegistrationDetails<FeatureVectorDataSetInformation> createFeatureVectorDatasetDetails(
String csvFilePath, Properties properties) throws IOException
{
List<FeatureDefinition> featureDefinitions =
@@ -234,5 +230,249 @@ public class JythonPlateDataSetHandler extends JythonTopLevelDataSetHandler<Data
return registrationDetails;
}
+ // -------- backward compatibility methods
+
+ /**
+ * This method exists just for backward compatibility. It used to have the second parameter,
+ * which is now ignored.
+ *
+ * @deprecated use {@link #createFeatureVectorDatasetDetails(IFeaturesBuilder)} instead.
+ */
+ @SuppressWarnings("unused")
+ @Deprecated
+ public DataSetRegistrationDetails<FeatureVectorDataSetInformation> createFeatureVectorRegistrationDetails(
+ IFeaturesBuilder featureBuilder, Object incomingDatasetFolder)
+ {
+ return createFeatureVectorDatasetDetails(featureBuilder);
+ }
+
+ /**
+ * @deprecated Changed to {@link #createFeatureVectorDatasetDetails(String, Properties)} due
+ * to naming convention change.
+ */
+ @SuppressWarnings("unused")
+ @Deprecated
+ public DataSetRegistrationDetails<FeatureVectorDataSetInformation> createFeatureVectorRegistrationDetails(
+ String csvFilePath, Properties properties) throws IOException
+ {
+ return createFeatureVectorDatasetDetails(csvFilePath, properties);
+ }
+
}
+
+ @Override
+ protected DataSetRegistrationService<DataSetInformation> createDataSetRegistrationService(
+ File incomingDataSetFile, DataSetInformation callerDataSetInformationOrNull,
+ IDelegatedActionWithResult<Boolean> cleanAfterwardsAction,
+ ITopLevelDataSetRegistratorDelegate delegate)
+ {
+ return new JythonDataSetRegistrationService<DataSetInformation>(this, incomingDataSetFile,
+ callerDataSetInformationOrNull, cleanAfterwardsAction, delegate,
+ new PythonInterpreter(), getGlobalState())
+ {
+ @Override
+ protected DataSetRegistrationTransaction<DataSetInformation> createTransaction(
+ File rollBackStackParentFolder,
+ File workingDirectory,
+ File stagingDirectory,
+ IDataSetRegistrationDetailsFactory<DataSetInformation> registrationDetailsFactory)
+ {
+ return new ImagingDataSetRegistrationTransaction(rollBackStackParentFolder,
+ workingDirectory, stagingDirectory, this, registrationDetailsFactory);
+ }
+ };
+ }
+
+ /**
+ * Imaging-specific transactions.
+ */
+ private static class ImagingDataSetRegistrationTransaction extends
+ DataSetRegistrationTransaction<DataSetInformation> implements
+ IImagingDataSetRegistrationTransaction
+ {
+ private final JythonPlateDatasetFactory registrationDetailsFactory;
+
+ public ImagingDataSetRegistrationTransaction(File rollBackStackParentFolder,
+ File workingDirectory, File stagingDirectory,
+ DataSetRegistrationService<DataSetInformation> registrationService,
+ IDataSetRegistrationDetailsFactory<DataSetInformation> registrationDetailsFactory)
+ {
+ super(rollBackStackParentFolder, workingDirectory, stagingDirectory,
+ registrationService, registrationDetailsFactory);
+
+ assert registrationDetailsFactory instanceof JythonPlateDatasetFactory : "JythonPlateDatasetFactory expected, but got: "
+ + registrationDetailsFactory.getClass().getCanonicalName();
+
+ this.registrationDetailsFactory =
+ (JythonPlateDatasetFactory) registrationDetailsFactory;
+ }
+
+ public IDataSet createNewImageDataSet(SimpleImageDataConfig imageDataSet,
+ File incomingFolderWithImages)
+ {
+ DataSetRegistrationDetails<ImageDataSetInformation> details =
+ SimpleImageDataSetRegistrator.createImageDatasetDetails(imageDataSet,
+ incomingFolderWithImages,
+ registrationDetailsFactory.imageDatasetFactory);
+ return createNewImageDataSet(details);
+ }
+
+ public IDataSet createNewImageDataSet(
+ DataSetRegistrationDetails<ImageDataSetInformation> imageRegistrationDetails)
+ {
+ ImageDataSetInformation imageDataSetInformation =
+ imageRegistrationDetails.getDataSetInformation();
+ ImageDataSetStructure imageDataSetStructure =
+ imageDataSetInformation.getImageDataSetStructure();
+ File incomingDirectory = imageDataSetInformation.getIncomingDirectory();
+ List<String> containedDataSetCodes = new ArrayList<String>();
+
+ // create thumbnails dataset if needed
+ IDataSet thumbnailDataset = null;
+ boolean generateThumbnails = imageDataSetStructure.areThumbnailsGenerated();
+ if (generateThumbnails)
+ {
+ thumbnailDataset = createThumbnailDataset();
+
+ ThumbnailFilePaths thumbnailPaths =
+ generateThumbnails(imageDataSetStructure, incomingDirectory,
+ thumbnailDataset);
+ imageDataSetInformation.setThumbnailFilePaths(thumbnailPaths);
+ containedDataSetCodes.add(thumbnailDataset.getDataSetCode());
+ }
+ // create main dataset (with original images)
+ IDataSet mainDataset = super.createNewDataSet(imageRegistrationDetails);
+ String originalDatasetPathPrefix =
+ ScreeningConstants.ORIGINAL_DATA_DIR + File.separator
+ + incomingDirectory.getName();
+ moveFile(incomingDirectory.getAbsolutePath(), mainDataset, originalDatasetPathPrefix);
+ containedDataSetCodes.add(mainDataset.getDataSetCode());
+
+ if (thumbnailDataset != null)
+ {
+ setSameDatasetOwner(mainDataset, thumbnailDataset);
+ }
+
+ IDataSet containerDataset =
+ createImageContainerDataset(mainDataset, imageDataSetInformation,
+ containedDataSetCodes);
+ imageDataSetInformation.setContainerDatasetPermId(containerDataset.getDataSetCode());
+ imageDataSetInformation.setDatasetRelativeImagesFolderPath(new File(
+ originalDatasetPathPrefix));
+
+ return containerDataset;
+ }
+
+ private ThumbnailFilePaths generateThumbnails(ImageDataSetStructure imageDataSetStructure,
+ File incomingDirectory, IDataSet thumbnailDataset)
+ {
+ String thumbnailFile =
+ createNewFile(thumbnailDataset, Constants.HDF5_CONTAINER_THUMBNAILS_FILE_NAME);
+
+ List<RelativeImageFile> images = asRelativeImageFile(imageDataSetStructure.getImages());
+ ThumbnailFilePaths thumbnailPaths =
+ Hdf5ThumbnailGenerator.tryGenerateThumbnails(images, incomingDirectory,
+ thumbnailFile, imageDataSetStructure.getImageStorageConfiguraton(),
+ thumbnailDataset.getDataSetCode());
+ return thumbnailPaths;
+ }
+
+ private static List<RelativeImageFile> asRelativeImageFile(List<ImageFileInfo> images)
+ {
+ List<RelativeImageFile> imageFiles = new ArrayList<RelativeImageFile>();
+ for (ImageFileInfo image : images)
+ {
+ imageFiles.add(RelativeImageFile.create(image));
+ }
+ return imageFiles;
+ }
+
+ private IDataSet createThumbnailDataset()
+ {
+ IDataSet thumbnailDataset =
+ createNewDataSet(ScreeningConstants.DEFAULT_OVERVIEW_IMAGE_DATASET_TYPE);
+ thumbnailDataset
+ .setFileFormatType(ScreeningConstants.DEFAULT_OVERVIEW_IMAGE_DATASET_FILE_FORMAT);
+ thumbnailDataset.setMeasuredData(false);
+
+ return thumbnailDataset;
+ }
+
+ private IDataSet createImageContainerDataset(IDataSet mainDataset,
+ ImageDataSetInformation imageDataSetInformation, List<String> containedDataSetCodes)
+ {
+ String containerDatasetTypeCode = findContainerDatasetTypeCode(imageDataSetInformation);
+ IDataSet containerDataset = createNewDataSet(containerDatasetTypeCode);
+ setSameDatasetOwner(mainDataset, containerDataset);
+ moveDatasetRelations(mainDataset, containerDataset);
+
+ containerDataset.setContainedDataSetCodes(containedDataSetCodes);
+ return containerDataset;
+ }
+
+ // Copies properties and relations to datasets from the main dataset to the container and
+ // resets them in the main dataset.
+ private static void moveDatasetRelations(IDataSet mainDataset, IDataSet containerDataset)
+ {
+ containerDataset.setParentDatasets(mainDataset.getParentDatasets());
+ mainDataset.setParentDatasets(Collections.<String> emptyList());
+
+ for (String propertyCode : mainDataset.getAllPropertyCodes())
+ {
+ containerDataset.setPropertyValue(propertyCode,
+ mainDataset.getPropertyValue(propertyCode));
+ mainDataset.setPropertyValue(propertyCode, null);
+ }
+ }
+
+ private static String findContainerDatasetTypeCode(
+ ImageDataSetInformation imageDataSetInformation)
+ {
+ String mainDatasetTypeCode = imageDataSetInformation.getDataSetType().getCode();
+ String prefix = ScreeningConstants.HCS_IMAGE_DATASET_TYPE_PREFIX;
+ if (mainDatasetTypeCode.toUpperCase().startsWith(prefix) == false)
+ {
+ throw UserFailureException.fromTemplate(
+ "The image dataset type '%s' does not start with '%s'.",
+ mainDatasetTypeCode, prefix);
+ }
+ if (mainDatasetTypeCode.toUpperCase().contains(
+ ScreeningConstants.IMAGE_CONTAINER_DATASET_TYPE_MARKER))
+ {
+ throw UserFailureException
+ .fromTemplate(
+ "The specified image dataset type '%s' should not be of container type, but contains '%s' in the type code.",
+ mainDatasetTypeCode,
+ ScreeningConstants.IMAGE_CONTAINER_DATASET_TYPE_MARKER);
+ }
+ return prefix + ScreeningConstants.IMAGE_CONTAINER_DATASET_TYPE_MARKER
+ + mainDatasetTypeCode.substring(prefix.length());
+ }
+
+ private static void setSameDatasetOwner(IDataSet templateDataset,
+ IDataSet destinationDataset)
+ {
+ destinationDataset.setExperiment(templateDataset.getExperiment());
+ destinationDataset.setSample(templateDataset.getSample());
+
+ }
+
+ @Override
+ public IDataSet createNewDataSet(
+ DataSetRegistrationDetails<? extends DataSetInformation> registrationDetails)
+ {
+ if (registrationDetails.getDataSetInformation() instanceof ImageDataSetInformation)
+ {
+ @SuppressWarnings("unchecked")
+ DataSetRegistrationDetails<ImageDataSetInformation> imageRegistrationDetails =
+ (DataSetRegistrationDetails<ImageDataSetInformation>) registrationDetails;
+ return createNewImageDataSet(imageRegistrationDetails);
+ } else
+ {
+ return super.createNewDataSet(registrationDetails);
+ }
+ }
+
+ }
+
}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/SimpleImageDataSetRegistrator.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/SimpleImageDataSetRegistrator.java
index b65617bae81..0b3b43ea9da 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/SimpleImageDataSetRegistrator.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/SimpleImageDataSetRegistrator.java
@@ -46,8 +46,9 @@ import ch.systemsx.cisd.imagereaders.IImageReader;
import ch.systemsx.cisd.imagereaders.ImageID;
import ch.systemsx.cisd.imagereaders.ImageReaderFactory;
import ch.systemsx.cisd.openbis.dss.etl.dto.ImageLibraryInfo;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.ImageDataSetInformation;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.ImageDataSetStructure;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.Channel;
-import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageDataSetInformation;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageFileInfo;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageIdentifier;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageMetadata;
@@ -529,11 +530,15 @@ public class SimpleImageDataSetRegistrator
dataset.setFileFormatCode(simpleImageConfig.getFileFormatType());
dataset.setMeasured(simpleImageConfig.isMeasuredData());
- String sampleCode = simpleImageConfig.getPlateCode();
- String spaceCode = simpleImageConfig.getPlateSpace();
- dataset.setSample(spaceCode, sampleCode);
- dataset.setMeasured(true);
+ dataset.setSample(simpleImageConfig.getPlateSpace(), simpleImageConfig.getPlateCode());
+ dataset.setIncomingDirectory(incoming);
+ ImageDataSetStructure imageStruct = createImageDataSetStructure(incoming);
+ dataset.setImageDataSetStructure(imageStruct);
+ }
+
+ private ImageDataSetStructure createImageDataSetStructure(File incoming)
+ {
List<File> imageFiles = extractImageFiles(incoming);
IImageReader imageReaderOrNull = tryCreateAndSaveImageReader(imageFiles);
List<ImageTokensWithPath> imageTokensList =
@@ -547,11 +552,14 @@ public class SimpleImageDataSetRegistrator
computeAndAppendCommonIntensityRangeTransformation(images, incoming, channels,
imageReaderOrNull);
- dataset.setImages(images);
- dataset.setChannels(channels);
- dataset.setTileGeometry(tileGeometry.getNumberOfRows(), tileGeometry.getNumberOfColumns());
+ ImageDataSetStructure imageStruct = new ImageDataSetStructure();
+ imageStruct.setImages(images);
+ imageStruct.setChannels(channels);
+ imageStruct.setTileGeometry(tileGeometry.getNumberOfRows(),
+ tileGeometry.getNumberOfColumns());
- dataset.setImageStorageConfiguraton(simpleImageConfig.getImageStorageConfiguration());
+ imageStruct.setImageStorageConfiguraton(simpleImageConfig.getImageStorageConfiguration());
+ return imageStruct;
}
private <T extends DataSetInformation> void setRegistrationDetails(
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/client/web/client/application/detailviewers/ImagingDatasetGuiUtils.java b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/client/web/client/application/detailviewers/ImagingDatasetGuiUtils.java
index 226ea4b5432..7c87465d9b7 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/client/web/client/application/detailviewers/ImagingDatasetGuiUtils.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/client/web/client/application/detailviewers/ImagingDatasetGuiUtils.java
@@ -242,7 +242,9 @@ class ImagingDatasetGuiUtils
sb.append("Dataset Code: ").append(reference.getCode()).append("<BR/>");
sb.append("Dataset Type: ").append(reference.getEntityType().getCode())
.append("<BR/>");
- sb.append("File Type: ").append(reference.getFileTypeCode()).append("<BR/>");
+ String fileTypeCode = reference.getFileTypeCode();
+ fileTypeCode = (fileTypeCode == null) ? "none" : fileTypeCode;
+ sb.append("File Type: ").append(fileTypeCode).append("<BR/>");
sb.append("Registration Date: ").append(reference.getRegistrationDate())
.append("<BR/>");
if (featureVectorDataset.getAnalysisProcedure() != null)
@@ -303,7 +305,8 @@ class ImagingDatasetGuiUtils
private String createDatasetSimpleViewModeHref(
SimpleModelComboBox<FeatureVectorDataset> datasetChooser)
{
- return LinkExtractor.tryExtract(datasetChooser.tryGetChosenItem().getDatasetReference());
+ return LinkExtractor
+ .tryExtract(datasetChooser.tryGetChosenItem().getDatasetReference());
}
}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/client/web/client/application/detailviewers/utils/EntityTypeLabelUtils.java b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/client/web/client/application/detailviewers/utils/EntityTypeLabelUtils.java
index 8fdfc44fd7e..a3ec636db09 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/client/web/client/application/detailviewers/utils/EntityTypeLabelUtils.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/client/web/client/application/detailviewers/utils/EntityTypeLabelUtils.java
@@ -108,7 +108,9 @@ public class EntityTypeLabelUtils
String registrationDate, String analysisProcedure, boolean withDatasetCode)
{
String typeLabel = getDatasetUserFriendlyTypeCode(datasetReference);
- String fileType = withFileType ? " (" + datasetReference.getFileTypeCode() + ")" : "";
+ String fileType =
+ withFileType && datasetReference.getFileTypeCode() != null ? " ("
+ + datasetReference.getFileTypeCode() + ")" : "";
String analysisProcedureRendered =
StringUtils.isBlank(analysisProcedure) ? "" : analysisProcedure + ", ";
return typeLabel + fileType + ", " + analysisProcedureRendered + registrationDate
@@ -127,6 +129,14 @@ public class EntityTypeLabelUtils
String label =
tryWithoutPrefix(datasetTypeCode,
ScreeningConstants.HCS_ANALYSIS_DATASET_TYPE_PREFIX);
+ if (StringUtils.isBlank(label))
+ {
+ label =
+ tryWithoutPrefix(datasetTypeCode,
+ ScreeningConstants.HCS_IMAGE_DATASET_TYPE_PREFIX
+ + ScreeningConstants.IMAGE_CONTAINER_DATASET_TYPE_MARKER);
+ }
+
if (StringUtils.isBlank(label))
{
label =
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/server/logic/HCSImageDatasetLoader.java b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/server/logic/HCSImageDatasetLoader.java
index edc6067f90c..74856465f4d 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/server/logic/HCSImageDatasetLoader.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/server/logic/HCSImageDatasetLoader.java
@@ -242,17 +242,21 @@ class HCSImageDatasetLoader extends PlateDatasetLoader
createChildToParentDataSetsMap(filteredChildrenForParents, parentDataSets,
datasetLister);
- List<ExternalData> childrenDataSets = new ArrayList<ExternalData>();
- for (ExternalData child : filteredChildrenForParents)
+ setParentDatasets(filteredChildrenForParents, childIdToParentDataSetsMap);
+ return filteredChildrenForParents;
+ }
+
+ private static void setParentDatasets(List<ExternalData> childrenDatasets,
+ Map<Long, ExternalData> childIdToParentDataSetsMap)
+ {
+ for (ExternalData child : childrenDatasets)
{
ExternalData parentImageDataset = childIdToParentDataSetsMap.get(child.getId());
if (parentImageDataset != null)
{
child.setParents(Arrays.asList(parentImageDataset));
}
- childrenDataSets.add(child);
}
- return childrenDataSets;
}
private Map<Long, ExternalData> createChildToParentDataSetsMap(
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/server/logic/LogicalImageLoader.java b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/server/logic/LogicalImageLoader.java
index da8dd49c10c..53b28d44e98 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/server/logic/LogicalImageLoader.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/server/logic/LogicalImageLoader.java
@@ -23,7 +23,6 @@ import java.util.List;
import ch.systemsx.cisd.openbis.generic.server.business.bo.IDataBO;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ExternalData;
import ch.systemsx.cisd.openbis.generic.shared.dto.DataPE;
-import ch.systemsx.cisd.openbis.generic.shared.dto.ExternalDataPE;
import ch.systemsx.cisd.openbis.generic.shared.dto.Session;
import ch.systemsx.cisd.openbis.generic.shared.translator.DataSetTranslator;
import ch.systemsx.cisd.openbis.generic.shared.util.EntityHelper;
@@ -104,11 +103,11 @@ public class LogicalImageLoader
return new ImageDatasetEnrichedReference(datasetImagesReference, overlayDatasets);
}
- List<ImageDatasetEnrichedReference> loadImageDatasets(List<ExternalDataPE> datasets)
+ List<ImageDatasetEnrichedReference> loadImageDatasets(List<DataPE> datasets)
{
List<ImageDatasetEnrichedReference> refs = new ArrayList<ImageDatasetEnrichedReference>();
- List<ExternalDataPE> imageDatasets = ScreeningUtils.filterImageDatasets(datasets);
- for (ExternalDataPE imageDataset : imageDatasets)
+ List<DataPE> imageDatasets = ScreeningUtils.filterImageDatasets(datasets);
+ for (DataPE imageDataset : imageDatasets)
{
DatasetImagesReference ref = loadImageDatasetReference(imageDataset);
List<DatasetOverlayImagesReference> overlays = extractImageOverlays(imageDataset);
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/server/logic/PlateContentLoader.java b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/server/logic/PlateContentLoader.java
index 8df31d7244b..f725984c684 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/server/logic/PlateContentLoader.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/server/logic/PlateContentLoader.java
@@ -31,17 +31,14 @@ import ch.systemsx.cisd.openbis.generic.server.business.bo.materiallister.IMater
import ch.systemsx.cisd.openbis.generic.server.business.bo.samplelister.ISampleLister;
import ch.systemsx.cisd.openbis.generic.shared.basic.TechId;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.CodeAndLabel;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ExternalData;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IEntityPropertiesHolder;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IEntityProperty;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ListOrSearchSampleCriteria;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Material;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Sample;
import ch.systemsx.cisd.openbis.generic.shared.dto.DataPE;
-import ch.systemsx.cisd.openbis.generic.shared.dto.ExternalDataPE;
import ch.systemsx.cisd.openbis.generic.shared.dto.SamplePE;
import ch.systemsx.cisd.openbis.generic.shared.dto.Session;
-import ch.systemsx.cisd.openbis.generic.shared.translator.DataSetTranslator;
import ch.systemsx.cisd.openbis.generic.shared.translator.EntityPropertyTranslator;
import ch.systemsx.cisd.openbis.generic.shared.translator.SampleTranslator;
import ch.systemsx.cisd.openbis.generic.shared.util.HibernateUtils;
@@ -170,17 +167,12 @@ public class PlateContentLoader
return dataSet;
}
- private ExternalData translate(ExternalDataPE externalData)
- {
- return DataSetTranslator.translate(externalData, session.getBaseIndexURL());
- }
-
private PlateContent getPlateContent(TechId plateId)
{
IDataSetTable dataSetTable = createDataSetTable();
Sample plate = loadPlate(plateId);
- List<ExternalDataPE> datasets = loadDatasets(plateId, dataSetTable);
+ List<DataPE> datasets = loadDatasets(plateId, dataSetTable);
List<WellMetadata> wells = loadWells(plateId);
List<ImageDatasetEnrichedReference> imageDatasetReferences =
@@ -196,17 +188,16 @@ public class PlateContentLoader
unknownDatasetReferences);
}
- private List<DatasetReference> extractUnknownDatasets(List<ExternalDataPE> datasets)
+ private List<DatasetReference> extractUnknownDatasets(List<DataPE> datasets)
{
- List<ExternalDataPE> unknownDatasets = ScreeningUtils.filterUnknownDatasets(datasets);
+ List<DataPE> unknownDatasets = ScreeningUtils.filterUnknownDatasets(datasets);
List<DatasetReference> unknownDatasetReferences = createDatasetReferences(unknownDatasets);
return unknownDatasetReferences;
}
- private List<FeatureVectorDataset> filterAndFetchFeatureVectors(List<ExternalDataPE> datasets)
+ private List<FeatureVectorDataset> filterAndFetchFeatureVectors(List<DataPE> datasets)
{
- List<ExternalDataPE> analysisDatasets =
- ScreeningUtils.filterImageAnalysisDatasetsPE(datasets);
+ List<DataPE> analysisDatasets = ScreeningUtils.filterImageAnalysisDatasetsPE(datasets);
List<DatasetReference> featureVectorDatasetReferences =
createDatasetReferences(analysisDatasets);
return fetchFeatureVectors(featureVectorDatasetReferences);
@@ -303,12 +294,13 @@ public class PlateContentLoader
return featureVectorDataset;
}
- private List<DatasetReference> createDatasetReferences(List<ExternalDataPE> datasets)
+ private <T extends DataPE> List<DatasetReference> createDatasetReferences(List<T> datasets)
{
List<DatasetReference> datasetReferences = new ArrayList<DatasetReference>();
- for (ExternalDataPE dataset : datasets)
+ for (T dataset : datasets)
{
- datasetReferences.add(ScreeningUtils.createDatasetReference(translate(dataset)));
+ datasetReferences.add(ScreeningUtils.createDatasetReference(dataset,
+ session.getBaseIndexURL()));
}
return datasetReferences;
}
@@ -361,10 +353,10 @@ public class PlateContentLoader
return materials;
}
- protected static List<ExternalDataPE> loadDatasets(TechId plateId, IDataSetTable dataSetTable)
+ protected static List<DataPE> loadDatasets(TechId plateId, IDataSetTable dataSetTable)
{
dataSetTable.loadBySampleTechId(plateId);
- return dataSetTable.getExternalData();
+ return dataSetTable.getDataSets();
}
private static List<WellMetadata> createWells(List<Sample> wellSamples)
@@ -415,11 +407,11 @@ public class PlateContentLoader
{
datasetOwnerSampleId = sampleId;
}
- List<ExternalDataPE> datasets = loadDatasets(datasetOwnerSampleId, createDataSetTable());
- List<ExternalDataPE> imageDatasets = ScreeningUtils.filterImageDatasets(datasets);
+ List<DataPE> datasets = loadDatasets(datasetOwnerSampleId, createDataSetTable());
+ List<DataPE> imageDatasets = ScreeningUtils.filterImageDatasets(datasets);
List<LogicalImageInfo> logicalImages = new ArrayList<LogicalImageInfo>();
- for (ExternalDataPE imageDataset : imageDatasets)
+ for (DataPE imageDataset : imageDatasets)
{
LogicalImageInfo logicalImage =
imageLoader.loadLogicalImageInfo(imageDataset.getCode(), imageDataset
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/server/logic/ScreeningUtils.java b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/server/logic/ScreeningUtils.java
index 38c172dc8a7..889e927fa8d 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/server/logic/ScreeningUtils.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/server/logic/ScreeningUtils.java
@@ -18,16 +18,20 @@ package ch.systemsx.cisd.openbis.plugin.screening.server.logic;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import ch.systemsx.cisd.bds.hcs.Location;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ContainerDataSet;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataStore;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Experiment;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ExternalData;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.FileFormatType;
import ch.systemsx.cisd.openbis.generic.shared.dto.DataPE;
-import ch.systemsx.cisd.openbis.generic.shared.dto.ExternalDataPE;
+import ch.systemsx.cisd.openbis.generic.shared.translator.DataStoreTranslator;
+import ch.systemsx.cisd.openbis.generic.shared.translator.ExperimentTranslator;
import ch.systemsx.cisd.openbis.generic.shared.util.EntityHelper;
import ch.systemsx.cisd.openbis.plugin.screening.server.IScreeningBusinessObjectFactory;
import ch.systemsx.cisd.openbis.plugin.screening.server.dataaccess.AnalysisProcedureResult;
@@ -57,22 +61,54 @@ public class ScreeningUtils
}
}
+ public static DatasetReference createDatasetReference(DataPE dataset, String baseIndexURL)
+ {
+ DataStore dataStore = DataStoreTranslator.translate(dataset.getDataStore());
+ String dataTypeCode = dataset.getDataSetType().getCode();
+ String fileTypeCode = null;
+ Experiment experiment =
+ ExperimentTranslator.translate(dataset.getExperiment(), baseIndexURL);
+ String analysisProcedureOrNull =
+ EntityHelper.tryFindPropertyValue(dataset, ScreeningConstants.ANALYSIS_PROCEDURE);
+ return createDatasetReference(dataset.getId(), dataset.getCode(), analysisProcedureOrNull,
+ dataStore, dataTypeCode, dataset.getRegistrationDate(), fileTypeCode, experiment);
+ }
+
public static DatasetReference createDatasetReference(ExternalData dataset)
{
DataStore dataStore = dataset.getDataStore();
String dataTypeCode = dataset.getDataSetType().getCode();
- @SuppressWarnings("deprecation")
- String fileTypeCode = dataset.getFileFormatType().getCode();
+ String fileTypeCode = tryGetFileTypeCode(dataset);
Experiment experiment = dataset.getExperiment();
String analysisProcedureOrNull =
EntityHelper.tryFindPropertyValue(dataset, ScreeningConstants.ANALYSIS_PROCEDURE);
- return new DatasetReference(dataset.getId(), dataset.getCode(), dataTypeCode,
- dataset.getRegistrationDate(), fileTypeCode, dataStore.getCode(),
- dataStore.getHostUrl(), experiment.getPermId(), experiment.getIdentifier(),
- analysisProcedureOrNull);
+ return createDatasetReference(dataset.getId(), dataset.getCode(), analysisProcedureOrNull,
+ dataStore, dataTypeCode, dataset.getRegistrationDate(), fileTypeCode, experiment);
+ }
+
+ private static String tryGetFileTypeCode(ExternalData dataset)
+ {
+ @SuppressWarnings("deprecation")
+ FileFormatType fileFormat = dataset.getFileFormatType();
+ String fileTypeCode = fileFormat == null ? null : fileFormat.getCode();
+ if (fileTypeCode != null
+ && fileTypeCode.equalsIgnoreCase(ScreeningConstants.UNKNOWN_FILE_FORMAT))
+ {
+ fileTypeCode = null;
+ }
+ return fileTypeCode;
+ }
+
+ private static DatasetReference createDatasetReference(long datasetId, String datasetCode,
+ String analysisProcedureOrNull, DataStore dataStore, String dataTypeCode,
+ Date registrationDate, String fileTypeCode, Experiment experiment)
+ {
+ return new DatasetReference(datasetId, datasetCode, dataTypeCode, registrationDate,
+ fileTypeCode, dataStore.getCode(), dataStore.getHostUrl(), experiment.getPermId(),
+ experiment.getIdentifier(), analysisProcedureOrNull);
}
- public static List<ExternalDataPE> filterImageAnalysisDatasetsPE(List<ExternalDataPE> datasets)
+ public static <T extends DataPE> List<T> filterImageAnalysisDatasetsPE(List<T> datasets)
{
return filterDatasetsByTypePattern(datasets,
ScreeningConstants.HCS_IMAGE_ANALYSIS_DATASET_TYPE_PATTERN);
@@ -102,16 +138,16 @@ public class ScreeningUtils
public static <T extends DataPE> List<T> filterImageOverlayDatasets(Collection<T> datasets)
{
- return filterDatasetsByTypePattern(datasets,
+ return filterNonContainedDatasets(datasets,
ScreeningConstants.HCS_SEGMENTATION_IMAGE_DATASET_TYPE_PATTERN,
ScreeningConstants.MICROSCOPY_SEGMENTATION_IMAGE_DATASET_TYPE_PATTERN);
}
/** excludes overlay image data sets even when they match to the image dataset pattern */
- public static List<ExternalDataPE> filterImageDatasets(List<ExternalDataPE> datasets)
+ public static <T extends DataPE> List<T> filterImageDatasets(List<T> datasets)
{
- List<ExternalDataPE> allDatasets =
- filterDatasetsByTypePattern(datasets,
+ List<T> allDatasets =
+ filterNonContainedDatasets(datasets,
ScreeningConstants.ANY_HCS_IMAGE_DATASET_TYPE_PATTERN,
ScreeningConstants.ANY_MICROSCOPY_IMAGE_DATASET_TYPE_PATTERN);
@@ -122,8 +158,40 @@ public class ScreeningUtils
return allDatasets;
}
+ // returns datasets matching one of the specified types which at the same time do not have a
+ // matching container dataset
+ private static <T extends DataPE> List<T> filterNonContainedDatasets(Collection<T> datasets,
+ String... datasetTypeCodePatterns)
+ {
+ List<T> typeMatchingDatasets =
+ filterDatasetsByTypePattern(datasets, datasetTypeCodePatterns);
+ final List<T> chosenDatasets = new ArrayList<T>();
+ for (T dataset : typeMatchingDatasets)
+ {
+ if (isContainerMatching(dataset, datasetTypeCodePatterns) == false)
+ {
+ chosenDatasets.add(dataset);
+ }
+ }
+ return chosenDatasets;
+ }
+
+ private static boolean isContainerMatching(ExternalData dataset,
+ String... datasetTypeCodePatterns)
+ {
+ ContainerDataSet container = dataset.tryGetContainer();
+ return container != null && isOneOfTypesMatching(container, datasetTypeCodePatterns);
+ }
+
+ private static <T extends DataPE> boolean isContainerMatching(T dataset,
+ String... datasetTypeCodePatterns)
+ {
+ DataPE container = dataset.getContainer();
+ return container != null && isOneOfTypesMatching(container, datasetTypeCodePatterns);
+ }
+
/** chooses datasets of unknown types */
- public static List<ExternalDataPE> filterUnknownDatasets(List<ExternalDataPE> datasets)
+ public static <T extends DataPE> List<T> filterUnknownDatasets(List<T> datasets)
{
return excludeDatasetsByTypePattern(datasets,
ScreeningConstants.HCS_IMAGE_ANALYSIS_DATASET_TYPE_PATTERN,
@@ -191,20 +259,28 @@ public class ScreeningUtils
*/
public static boolean isHcsImageDataset(ExternalData externalData)
{
- return isOneOfTypesMatching(externalData,
+ return isTypeMatchingExcludingContainer(externalData,
ScreeningConstants.ANY_HCS_IMAGE_DATASET_TYPE_PATTERN);
}
+ private static boolean isTypeMatchingExcludingContainer(ExternalData externalData,
+ String typePattern)
+ {
+ return isOneOfTypesMatching(externalData, typePattern)
+ && isContainerMatching(externalData, typePattern) == false;
+ }
+
public static boolean isRawHcsImageDataset(ExternalData externalData)
{
- return isTypeMatching(externalData, ScreeningConstants.HCS_RAW_IMAGE_DATASET_TYPE_PATTERN)
+ return isTypeMatchingExcludingContainer(externalData,
+ ScreeningConstants.HCS_RAW_IMAGE_DATASET_TYPE_PATTERN)
|| ScreeningConstants.HCS_RAW_IMAGE_LEGACY_DATASET_TYPE.equals(externalData
.getDataSetType().getCode());
}
public static boolean isSegmentationHcsImageDataset(ExternalData externalData)
{
- return isOneOfTypesMatching(externalData,
+ return isTypeMatchingExcludingContainer(externalData,
ScreeningConstants.HCS_SEGMENTATION_IMAGE_DATASET_TYPE_PATTERN);
}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/basic/dto/ScreeningConstants.java b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/basic/dto/ScreeningConstants.java
index 4cbe2e1c84d..9e5f1215474 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/basic/dto/ScreeningConstants.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/basic/dto/ScreeningConstants.java
@@ -37,6 +37,13 @@ public class ScreeningConstants
// ---- required entity type patterns -----------
+ /**
+ * All image datasets (both hcs and microscopy) which contain this marker in the dataset type
+ * code are considered to be container datasets with original physical dataset inside.
+ * Optionally the container can contain thumbnail datasets.
+ */
+ public static final String IMAGE_CONTAINER_DATASET_TYPE_MARKER = "_CONTAINER";
+
// --- HCS dataset types
/** Prefix of a dataset's type which stores hcs data. */
@@ -48,11 +55,11 @@ public class ScreeningConstants
/** Type of the dataset which stores plate image overlays. */
public static final String HCS_SEGMENTATION_IMAGE_DATASET_TYPE_PATTERN =
(HCS_IMAGE_DATASET_TYPE_PREFIX + ".*OVERLAY.*") + "|" // legacy
- + (HCS_IMAGE_DATASET_TYPE_PREFIX + "_SEGMENTATION.*");
+ + (HCS_IMAGE_DATASET_TYPE_PREFIX + ".*_SEGMENTATION.*");
/** Type of the dataset which stores raw plate images. */
public static final String HCS_RAW_IMAGE_DATASET_TYPE_PATTERN = HCS_IMAGE_DATASET_TYPE_PREFIX
- + "_RAW.*";
+ + ".*_RAW.*";
/** The plain old legacy type for raw image data sets. */
public static final String HCS_RAW_IMAGE_LEGACY_DATASET_TYPE = HCS_IMAGE_DATASET_TYPE_PREFIX;
@@ -108,21 +115,34 @@ public class ScreeningConstants
// --- Default dataset type codes for screening datasets
- /** type of the new raw image dataset */
+ /** type of the image container dataset type for raw images */
+ public static final String DEFAULT_RAW_IMAGE_CONTAINER_DATASET_TYPE = "HCS_IMAGE_CONTAINER_RAW";
+
+ /** type of the raw image physical dataset */
public static final String DEFAULT_RAW_IMAGE_DATASET_TYPE = "HCS_IMAGE_RAW";
- /** type of the new overview image dataset */
- public static final String DEFAULT_OVERVIEW_IMAGE_DATASET_TYPE = "HCS_IMAGE_OVERVIEW";
+ /** type of the image segmentation (overlay) physical dataset */
+ public static final String DEFAULT_SEGMENTATION_IMAGE_CONTAINER_DATASET_TYPE =
+ "HCS_IMAGE_CONTAINER_SEGMENTATION";
- /** type of the new image segmentation (overlay) dataset */
+ /** type of the image segmentation (overlay) physical dataset */
public static final String DEFAULT_SEGMENTATION_IMAGE_DATASET_TYPE = "HCS_IMAGE_SEGMENTATION";
+ /**
+ * type of the overview (aka thumbnail) image physical dataset. Used for both raw and overview
+ * datasets.
+ */
+ public static final String DEFAULT_OVERVIEW_IMAGE_DATASET_TYPE = "HCS_IMAGE_OVERVIEW";
+
/** type of the new analysis dataset */
public static final String DEFAULT_ANALYSIS_WELL_DATASET_TYPE = "HCS_ANALYSIS_WELL_FEATURES";
/** unknown file format code */
public static final String UNKNOWN_FILE_FORMAT = "UNKNOWN";
+ /** Default file format of thumbnail datasets. */
+ public static final String DEFAULT_OVERVIEW_IMAGE_DATASET_FILE_FORMAT = "PNG";
+
// ----
/** Code of plate geometry vocabulary. */
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/imaging/HCSDatasetLoader.java b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/imaging/HCSDatasetLoader.java
index 7029538a2bb..07c3c6cd5eb 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/imaging/HCSDatasetLoader.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/imaging/HCSDatasetLoader.java
@@ -20,13 +20,17 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.log4j.Logger;
+
import ch.systemsx.cisd.common.collections.CollectionUtils;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.utilities.MD5ChecksumCalculator;
import ch.systemsx.cisd.openbis.generic.shared.basic.utils.GroupByMap;
import ch.systemsx.cisd.openbis.generic.shared.basic.utils.IGroupKeyExtractor;
-import ch.systemsx.cisd.openbis.plugin.screening.shared.basic.dto.InternalImageChannel;
import ch.systemsx.cisd.openbis.plugin.screening.shared.basic.dto.ImageChannelStack;
import ch.systemsx.cisd.openbis.plugin.screening.shared.basic.dto.ImageDatasetParameters;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.basic.dto.InternalImageChannel;
import ch.systemsx.cisd.openbis.plugin.screening.shared.basic.dto.InternalImageTransformationInfo;
import ch.systemsx.cisd.openbis.plugin.screening.shared.basic.dto.WellLocation;
import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.IImagingReadonlyQueryDAO;
@@ -46,6 +50,9 @@ import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgIm
*/
public class HCSDatasetLoader implements IImageDatasetLoader
{
+ static protected final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
+ HCSDatasetLoader.class);
+
protected final IImagingReadonlyQueryDAO query;
protected final ImgImageDatasetDTO dataset;
@@ -238,9 +245,9 @@ public class HCSDatasetLoader implements IImageDatasetLoader
{
String transformationSignature =
tryGetSignature(transformation.getSerializedImageTransformerFactory());
- return new InternalImageTransformationInfo(transformation.getCode(), transformation.getLabel(),
- transformation.getDescription(), transformationSignature,
- transformation.getIsDefault());
+ return new InternalImageTransformationInfo(transformation.getCode(),
+ transformation.getLabel(), transformation.getDescription(),
+ transformationSignature, transformation.getIsDefault());
}
private static String tryGetSignature(byte[] bytesOrNull)
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/imaging/dataaccess/IImagingReadonlyQueryDAO.java b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/imaging/dataaccess/IImagingReadonlyQueryDAO.java
index 7f2e7c4b998..c54e10e35d4 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/imaging/dataaccess/IImagingReadonlyQueryDAO.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/imaging/dataaccess/IImagingReadonlyQueryDAO.java
@@ -235,6 +235,10 @@ public interface IImagingReadonlyQueryDAO extends BaseQuery
{ StringArrayMapper.class }, fetchSize = FETCH_SIZE)
public List<ImgImageDatasetDTO> listImageDatasetsByPermId(String... datasetPermIds);
+ @Select(sql = "select * from IMAGE_ZOOM_LEVELS zoom, IMAGE_DATA_SETS img_ds "
+ + "where zoom.CONTAINER_DATASET_ID = img_ds.id and img_ds.PERM_ID = ?{1}", fetchSize = FETCH_SIZE)
+ public List<ImgImageZoomLevelDTO> listImageZoomLevels(String containerDatasetPermId);
+
@Select(sql = "select * from ANALYSIS_DATA_SETS where PERM_ID = any(?{1})", parameterBindings =
{ StringArrayMapper.class }, fetchSize = FETCH_SIZE)
public List<ImgAnalysisDatasetDTO> listAnalysisDatasetsByPermId(String... datasetPermIds);
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/imaging/dataaccess/ImgImageZoomLevelDTO.java b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/imaging/dataaccess/ImgImageZoomLevelDTO.java
new file mode 100644
index 00000000000..cb006c85e95
--- /dev/null
+++ b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/imaging/dataaccess/ImgImageZoomLevelDTO.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess;
+
+import net.lemnik.eodsql.ResultColumn;
+
+/**
+ * Describes an image dataset with 1. all images resized in the same way or 2. original images.
+ *
+ * @author Tomasz Pylak
+ */
+public class ImgImageZoomLevelDTO extends AbstractImgIdentifiable
+{
+ @ResultColumn("physical_dataset_perm_id")
+ private String physicalDatasetPermId;
+
+ @ResultColumn("is_original")
+ private boolean isOriginal;
+
+ @ResultColumn("container_dataset_id")
+ private long containerDatasetId;
+
+ @SuppressWarnings("unused")
+ private ImgImageZoomLevelDTO()
+ {
+ // All Data-Object classes must have a default constructor.
+ }
+
+ public ImgImageZoomLevelDTO(String physicalDatasetPermId, boolean isOriginal,
+ long containerDatasetId)
+ {
+ this.physicalDatasetPermId = physicalDatasetPermId;
+ this.isOriginal = isOriginal;
+ this.containerDatasetId = containerDatasetId;
+ }
+
+ public String getPhysicalDatasetPermId()
+ {
+ return physicalDatasetPermId;
+ }
+
+ public void setPhysicalDatasetPermId(String physicalDatasetPermId)
+ {
+ this.physicalDatasetPermId = physicalDatasetPermId;
+ }
+
+ public boolean getIsOriginal()
+ {
+ return isOriginal;
+ }
+
+ public void setOriginal(boolean isOriginal)
+ {
+ this.isOriginal = isOriginal;
+ }
+
+ public long getContainerDatasetId()
+ {
+ return containerDatasetId;
+ }
+
+ public void setContainerDatasetId(long containerDatasetId)
+ {
+ this.containerDatasetId = containerDatasetId;
+ }
+}
diff --git a/screening/sourceTest/java/BiozentrumMatLabApiTest.java b/screening/sourceTest/java/BiozentrumMatLabApiTest.java
new file mode 100644
index 00000000000..be1c7b9b75f
--- /dev/null
+++ b/screening/sourceTest/java/BiozentrumMatLabApiTest.java
@@ -0,0 +1,222 @@
+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Integration tests of MatLab API on Biozentrum server.
+ *
+ * @author Tomasz Pylak
+ */
+public class BiozentrumMatLabApiTest
+{
+ public static void main(String[] args)
+ {
+ if (args.length == 0)
+ {
+ System.err
+ .println("Specify the password and optionally the plate identifier, mount point and dataset type pattern!");
+ }
+ String passwd = "xxx";
+ String chosenPlate = "/GROUP_DEHIO/KB03-2O-130";
+ String mountPoint = null;
+ String datasetTypePattern = "HCS_ANALYSIS_CELL_CLASSIFICATIONS_MAT";
+ if (args.length > 1)
+ {
+ chosenPlate = args[1];
+ }
+ if (args.length > 2)
+ {
+ mountPoint = args[2];
+ }
+
+ // OpenBISScreeningML.login("tpylak", passwd, "https://infectx.biozentrum.unibas.ch");
+ // Object[][] metadata = OpenBISScreeningML.getImagesMetadata("/RESEARCH_IT/TEST2");
+ // System.out.println(Arrays.toString(metadata[0]));
+ // Object[][][] loadImages =
+ // OpenBISScreeningML.loadImages("/INFECTX/DZ56-1K", 1, 1, 1, new String[]
+ // { "CY5", "DAPI" });
+ // System.out.println(loadImages[0][0][0]);
+
+ // OpenBISScreeningML.login("cisd", passwd, "http://bc2-openbis01.bc2.unibas.ch:8443");
+ OpenBISScreeningML.login("admin", passwd, "https://sprint-openbis.ethz.ch:8446");
+
+ // OpenBISScreeningML.login("admin", passwd, "http://127.0.0.1:8888");
+ // Object[][] metadata = OpenBISScreeningML.getImagesMetadata("/TEST/PLATE1");
+ // System.out.println(Arrays.toString(metadata[0]));
+ // Object[][][] loadImages =
+ Object[][][] images = OpenBISScreeningML.loadImages("/TEST/PLATE1", 1, 1, 1, new String[]
+ { "DAPI" });
+
+ // String experiment = "/TEST/TEST-USER/MY-ASSAY";
+
+ // testListChannels(experiment);
+ // testLoadFeatures(chosenPlate, experiment);
+ // testImagesMetadata(chosenPlate);
+ // testLoadImage(chosenPlate);
+ // testWellProperties(chosenPlate);
+ // testLoadDataset(chosenPlate, mountPoint, datasetTypePattern);
+
+ // testUploadDataset(chosenPlate);
+ // testUploadDataset("/RESEARCH_IT/TEST2");
+ // testUploadDataset("/DEMO/VL0206A-FV1801");
+ // testUploadDataset("/TEST/PLATE1");
+
+ // integration server test
+ // OpenBISScreeningML.login("admin", passwd, "https://127.0.0.1:8443");
+ // String datasetType = "HCS_MODULESETTINGS_OBJECTCLASSIFICATION_MAT";
+ // String datasetType = "HCS_ANALYSIS_WELL_CLASSIFICATION_SUMMARIES";
+ // String datasetType = "HCS_ANALYSIS_CELL_CLASSIFICATIONS_MAT";
+ // String datasetType = "UNKNOWN";
+ // testUploadDataset("/GROUP_RESEARCHIT/DZ01-1A", datasetType);
+
+ // PlateMetadata[] plateMetadataList = OpenBISScreeningML.getPlateMetadataList(new String[]
+ // { "/TEST/PLATE1", "/TEST/PLATE2", "/TEST/PLATE1.96WELLS", "/TEST/PLATE1.SUFFIX", });
+ sleepLong();
+ OpenBISScreeningML.logout();
+ }
+
+ private static void sleepLong()
+ {
+ try
+ {
+ Thread.sleep(1000000);
+ } catch (InterruptedException ex)
+ {
+ ex.printStackTrace();
+ }
+ }
+
+ private static void testListChannels(String experiment)
+ {
+ Object[][] channels = OpenBISScreeningML.listChannels(experiment);
+ print2DArray(channels);
+ }
+
+ private static void testUploadDataset(String chosenPlate, String datasetType)
+ {
+ String file = "/Users/tpylak/main/tmp/bioz-test/PlateSummary.csv";
+ String datasetCode;
+ Object[][] dataSetProperties = new Object[][]
+ { new Object[] {} };
+
+ datasetCode =
+ (String) OpenBISScreeningML.uploadDataSet(chosenPlate, file, datasetType,
+ dataSetProperties);
+ System.out.println("Uploaded any dataset " + datasetCode);
+ }
+
+ private static void testLoadFeatures(String chosenPlate, String experiment)
+ {
+ Object[][] features = OpenBISScreeningML.listFeatures(experiment, null);
+ print2DArray(features);
+
+ Object[][][] featureMatrix =
+ OpenBISScreeningML.getFeatureMatrixForPlate(chosenPlate, null, null);
+ System.out.println("per plate features -------------------------------");
+ print3DArray(featureMatrix);
+
+ String[] featureNames = new String[]
+ { "OOF" };
+ featureMatrix =
+ OpenBISScreeningML.getFeatureMatrixForPlate(chosenPlate, null, featureNames);
+ System.out.println("one per plate feature -------------------------------");
+ print3DArray(featureMatrix);
+
+ featureMatrix = OpenBISScreeningML.getFeatureMatrix(experiment, "149420", featureNames);
+ System.out.println("one per gene feature -------------------------------");
+ print3DArray(featureMatrix);
+ }
+
+ private static void print3DArray(Object[][][] array)
+ {
+ System.out.println("{");
+ for (int i = 0; i < array.length; i++)
+ {
+ print2DArray(array[i]);
+ }
+ System.out.println("}");
+ }
+
+ private static void print2DArray(Object[][] array)
+ {
+ System.out.println("[");
+ for (int i = 0; i < array.length; i++)
+ {
+ printAsRow(array[i]);
+ }
+ System.out.println("]");
+ }
+
+ private static void printAsRow(Object[] objects)
+ {
+ StringBuffer sb = new StringBuffer();
+ sb.append("[");
+ for (int i = 0; i < objects.length; i++)
+ {
+ sb.append(objects[i]);
+ sb.append(",");
+ }
+ sb.append("]");
+ System.out.println(sb.toString());
+ }
+
+ private static void testImagesMetadata(String chosenPlate)
+ {
+ Object[][] meta = OpenBISScreeningML.getImagesMetadata(chosenPlate);
+ System.out.println(String.format("Number of tiles: %s. Plate geometry %sx%s", meta[0][2],
+ meta[0][5], meta[0][6]));
+ }
+
+ private static void testLoadDataset(String chosenPlate, String mountPoint,
+ String datasetTypePattern)
+ {
+ // Loads dataset with classification results.
+ // OpenBIS store diretcory is mounted locally in "/mount/openbis/store", so no data are
+ // copied and just a path to the appropriate location
+ // is returned.
+ Object[][] datasets =
+ OpenBISScreeningML.loadDataSets(chosenPlate, datasetTypePattern, mountPoint);
+ System.out.println("Path to the first downloaded dataset: " + datasets[0][1]);
+ }
+
+ private static void testLoadImage(String chosenPlate)
+ {
+ // fetch 3ed tile of well (1,1). The second call to fetch this image will return it from the
+ // local cache.
+ // The last optional parameter can specify a list of channels to load (otherwise all
+ // channels are loaded).
+ long start = System.currentTimeMillis();
+ Object[][][] images = OpenBISScreeningML.loadImages(chosenPlate, 1, 1, 3, new String[]
+ { "CY3" });
+ System.out.println("Image path: " + images[0][0][0]);
+ System.out.println("Took: " + (System.currentTimeMillis() - start));
+ }
+
+ private static void testWellProperties(String chosenPlate)
+ {
+ // properties of well (2,4)
+ Object[][] props = OpenBISScreeningML.getWellProperties(chosenPlate, 2, 4);
+ for (int i = 0; i < props.length; i++)
+ {
+ System.out.println(String.format("Property %s = %s ", props[i][0], props[i][1]));
+ }
+
+ // save description of the well (2,4)
+ props = new Object[][]
+ {
+ { "DESCRIPTION", "hello example" } };
+ OpenBISScreeningML.updateWellProperties(chosenPlate, 2, 4, props);
+ }
+}
diff --git a/screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/ImagingQueryDAOTest.java b/screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/ImagingQueryDAOTest.java
index 6ed8e649a48..068607e7007 100644
--- a/screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/ImagingQueryDAOTest.java
+++ b/screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/ImagingQueryDAOTest.java
@@ -32,6 +32,7 @@ import org.testng.annotations.Test;
import ch.systemsx.cisd.base.image.IImageTransformerFactory;
import ch.systemsx.cisd.bds.hcs.Location;
+import ch.systemsx.cisd.common.test.AssertionUtil;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ChannelColorRGB;
import ch.systemsx.cisd.openbis.generic.shared.basic.CodeNormalizer;
import ch.systemsx.cisd.openbis.plugin.screening.client.api.v1.ExampleImageTransformerFactory;
@@ -45,6 +46,7 @@ import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgIm
import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgImageDatasetDTO;
import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgImageEnrichedDTO;
import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgImageTransformationDTO;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgImageZoomLevelDTO;
import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgSpotDTO;
/**
@@ -424,6 +426,11 @@ public class ImagingQueryDAOTest extends AbstractDBTest
new ImgImageDatasetDTO(permId, fieldsHeight, fieldsWidth, containerIdOrNull, false,
libraryName, libraryReader);
final long datasetId = dao.addImageDataset(dataset);
+ String physicalDatasetPermIdPrefix = permId + "666";
+ dao.addImageZoomLevel(new ImgImageZoomLevelDTO(physicalDatasetPermIdPrefix + "-123", true,
+ datasetId));
+ dao.addImageZoomLevel(new ImgImageZoomLevelDTO(physicalDatasetPermIdPrefix + "-666", false,
+ datasetId));
final ImgImageDatasetDTO loadedDataset = dao.tryGetImageDatasetByPermId(permId);
assertNotNull(loadedDataset);
@@ -440,6 +447,15 @@ public class ImagingQueryDAOTest extends AbstractDBTest
assertEquals(1, datasets.size());
assertEquals(loadedDataset, datasets.get(0));
+ // test listImageZoomLevels
+ List<ImgImageZoomLevelDTO> zoomLevels = dao.listImageZoomLevels(permId);
+ assertEquals(2, zoomLevels.size());
+ for (ImgImageZoomLevelDTO zoomLevel : zoomLevels)
+ {
+ AssertionUtil.assertContains(physicalDatasetPermIdPrefix,
+ zoomLevel.getPhysicalDatasetPermId());
+ }
+
return datasetId;
}
diff --git a/screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/featurevector/FeatureVectorUploaderTest.java b/screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/featurevector/FeatureVectorUploaderTest.java
index 4cb76b3ce14..749eb8c40f4 100644
--- a/screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/featurevector/FeatureVectorUploaderTest.java
+++ b/screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/featurevector/FeatureVectorUploaderTest.java
@@ -68,7 +68,7 @@ public class FeatureVectorUploaderTest extends AbstractDBTest
{
HCSContainerDatasetInfo info = new HCSContainerDatasetInfo();
info.setExperimentPermId(EXP_PERM_ID);
- info.setContainerPermId(CONTAINER_PERM_ID);
+ info.setContainerSamplePermId(CONTAINER_PERM_ID);
info.setDatasetPermId(DS_PERM_ID);
FeatureVectorUploader uploader = new FeatureVectorUploader(dao, info);
ArrayList<CanonicalFeatureVector> fvecs = new ArrayList<CanonicalFeatureVector>();
diff --git a/screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/jython/SimpleImageDataSetRegistratorTest.java b/screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/jython/SimpleImageDataSetRegistratorTest.java
index fa2fcf351bb..0ffc20c951f 100644
--- a/screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/jython/SimpleImageDataSetRegistratorTest.java
+++ b/screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/jython/SimpleImageDataSetRegistratorTest.java
@@ -33,21 +33,21 @@ import ch.systemsx.cisd.etlserver.registrator.DataSetRegistrationDetails;
import ch.systemsx.cisd.etlserver.registrator.IDataSetRegistrationDetailsFactory;
import ch.systemsx.cisd.imagereaders.IImageReader;
import ch.systemsx.cisd.imagereaders.ImageID;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.ImageDataSetInformation;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ChannelColor;
-import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageDataSetInformation;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageFileInfo;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.SimpleImageContainerDataConfig;
import ch.systemsx.cisd.openbis.dss.etl.jython.SimpleImageDataSetRegistrator.IImageReaderFactory;
/**
- *
- *
* @author Franz-Josef Elmer
*/
-@Friend(toClasses=SimpleImageDataSetRegistrator.class)
+@Friend(toClasses = SimpleImageDataSetRegistrator.class)
public class SimpleImageDataSetRegistratorTest extends AbstractFileSystemTestCase
{
- private static final File TEST_DATA_FOLDER = new File("resource/test-data/SimpleImageDataSetRegistratorTest");
+ private static final File TEST_DATA_FOLDER = new File(
+ "resource/test-data/SimpleImageDataSetRegistratorTest");
+
private static final class MyConfigData extends SimpleImageContainerDataConfig
{
String channelCodes = "";
@@ -58,7 +58,7 @@ public class SimpleImageDataSetRegistratorTest extends AbstractFileSystemTestCas
channelCodes += channelCode;
return ChannelColor.createFromIndex(channelCodes.length());
}
-
+
}
private Mockery context;
@@ -125,24 +125,27 @@ public class SimpleImageDataSetRegistratorTest extends AbstractFileSystemTestCas
factory, readerFactory);
assertSame(details, actualDetails);
- List<ImageFileInfo> images = actualDetails.getDataSetInformation().getImages();
- assertEquals("ImageFileInfo [well=null, tile=[x=1,y=1], channel=CHANNEL-4, " +
- "path=images/pond.tif, timepoint=1.0, depth=2.0, seriesNumber=1]", images.get(0).toString());
- assertEquals("ImageFileInfo [well=null, tile=[x=1,y=1], channel=CHANNEL-10, " +
- "path=images/pond.tif, timepoint=2.0, depth=3.0, seriesNumber=2]", images.get(1).toString());
- assertEquals("ImageFileInfo [well=null, tile=[x=1,y=1], channel=CHANNEL-4, " +
- "path=images/pond.tif, timepoint=8.0, depth=9.0, seriesNumber=3]", images.get(2).toString());
+ List<ImageFileInfo> images =
+ actualDetails.getDataSetInformation().getImageDataSetStructure().getImages();
+ assertEquals("ImageFileInfo [well=null, tile=[x=1,y=1], channel=CHANNEL-4, "
+ + "path=images/pond.tif, timepoint=1.0, depth=2.0, seriesNumber=1]", images.get(0)
+ .toString());
+ assertEquals("ImageFileInfo [well=null, tile=[x=1,y=1], channel=CHANNEL-10, "
+ + "path=images/pond.tif, timepoint=2.0, depth=3.0, seriesNumber=2]", images.get(1)
+ .toString());
+ assertEquals("ImageFileInfo [well=null, tile=[x=1,y=1], channel=CHANNEL-4, "
+ + "path=images/pond.tif, timepoint=8.0, depth=9.0, seriesNumber=3]", images.get(2)
+ .toString());
assertEquals(3, images.size());
assertEquals("CHANNEL-4CHANNEL-10", configData.channelCodes);
context.assertIsSatisfied();
}
-
+
private File copyExampleFile(String fileName)
{
- File destinationFile = new File(imageFolder,
- fileName);
+ File destinationFile = new File(imageFolder, fileName);
FileUtilities.copyFileTo(new File(TEST_DATA_FOLDER, fileName), destinationFile, false);
return destinationFile;
}
-
+
}
--
GitLab