From 074ba94358ca00581aad840d9ec9d8879f5f73e4 Mon Sep 17 00:00:00 2001
From: tpylak <tpylak>
Date: Tue, 15 Mar 2011 12:04:26 +0000
Subject: [PATCH] LMS-2069, LMS-2085 ibrain dropboxes
SVN: 20326
---
.../data-ibrain2/scripts/commonDropbox.py | 75 +++++++++++++------
.../scripts/commonImageDropbox.py | 33 +++++---
.../scripts/hcs_image_overview.py | 53 +++++++++++++
.../data-ibrain2/scripts/hcs_image_raw.py | 59 ++++++++-------
.../scripts/hcs_image_segmentation.py | 10 +--
5 files changed, 161 insertions(+), 69 deletions(-)
diff --git a/integration-tests/templates/data-ibrain2/scripts/commonDropbox.py b/integration-tests/templates/data-ibrain2/scripts/commonDropbox.py
index 57886ecb685..a06c849e7b9 100755
--- a/integration-tests/templates/data-ibrain2/scripts/commonDropbox.py
+++ b/integration-tests/templates/data-ibrain2/scripts/commonDropbox.py
@@ -1,6 +1,7 @@
#! /usr/bin/env python
import os
+from java.io import File
class AbstractPropertiesParser:
_propertiesDict = None
@@ -40,21 +41,18 @@ class AbstractPropertiesParser:
# Returns:
# an iterator which yields (propertyName, propertyValue) pairs
def getPropertiesIter(self):
- for key, value in self._propertiesDict.iteritems():
- yield (key, value)
-
+ return [ (key, value) for key, value in self._propertiesDict.iteritems() ]
+
# All dataset properties.
# Returns:
# an iterator which yields (propertyCode, propertyValue) pairs
def getDatasetPropertiesIter(self):
- for key, value in self._propertiesDict.iteritems():
- if key.startswith(self.DATASET_PROPERTY_PREFIX):
- yield (key, value)
+ return [ (key, value) for key, value in self._propertiesDict.iteritems() if key.startswith(self.DATASET_PROPERTY_PREFIX) ]
class AbstractMetadataParser(AbstractPropertiesParser):
METADATA_FILE="metadata.properties"
- IBRAIN2_DATASET_ID_PROPERTY = "brain2.dataset.id"
+ IBRAIN2_DATASET_ID_PROPERTY = "ibrain2.dataset.id"
DATASET_PROPERTY_PREFIX = "ibrain2."
DATASET_TYPE_PROPERTY = "dataset.type"
@@ -78,9 +76,10 @@ class AcquiredDatasetMetadataParser(AbstractMetadataParser):
# Returns:
# an iterator which yields (propertyCode, propertyValue) pairs
def getDatasetPropertiesIter(self):
- for propertyCode, value in AbstractPropertiesParser.getDatasetPropertiesIter(self):
- yield (propertyCode, value)
- yield (self.INSTRUMENT_PROPERTY, self.get(self.INSTRUMENT_PROPERTY))
+ properties = AbstractPropertiesParser.getDatasetPropertiesIter(self)
+ properties = [ (key, value) for (key, value) in properties if key != "ibrain2.assay.id" ]
+ properties.append((self.INSTRUMENT_PROPERTY, self.get(self.INSTRUMENT_PROPERTY)))
+ return properties
def getPlateCode(self):
return self.get(self.PLATE_CODE_PRPOPERTY)
@@ -111,7 +110,6 @@ class AssayParser(AbstractPropertiesParser):
def _findAssayFile(self, incoming):
for file in os.listdir(incoming):
- print file
if file.startswith(self.ASSAY_FILE_PREFIX):
return file
raise Exception("Assay file not found!")
@@ -119,9 +117,9 @@ class AssayParser(AbstractPropertiesParser):
def __init__(self, incoming):
AbstractPropertiesParser.__init__(self, incoming, self._findAssayFile(incoming))
-class IBRAIN2Utils:
+class RegistrationConfirmationUtils:
""" path to the registration confirmation directory relative to the incoming dataset """
- CONFIRMATION_DIRECTORY = "../../registration-status"
+ CONFIRMATION_DIRECTORY = "registration-status"
STATUS_PROPERTY = "storage_provider.storage.status"
OK = "STORAGE_SUCCESS"
@@ -132,31 +130,66 @@ class IBRAIN2Utils:
IBRAIN2_STATUS_FILE_PREFIX = "ibrain2_dataset_id_"
IBRAIN2_STATUS_FILE_SUFFIX = ".properties"
- def _getStatusFileName(self, ibrain2DatasetId, incoming):
- return incoming + "/" + self.CONFIRMATION_DIRECTORY + "/" + self.IBRAIN2_STATUS_FILE_PREFIX + ibrain2DatasetId + self.IBRAIN2_STATUS_FILE_SUFFIX
+ def _getDestinationDir(self, incoming):
+ return File(incoming).getParentFile().getParentFile().getParent() + "/" + self.CONFIRMATION_DIRECTORY
+
+ def _getConfirmationFileName(self, ibrain2DatasetId):
+ return self.IBRAIN2_STATUS_FILE_PREFIX + ibrain2DatasetId + self.IBRAIN2_STATUS_FILE_SUFFIX
+
+ def _getStatusFilePath(self, ibrain2DatasetId, incoming):
+ return self._getDestinationDir(incoming) + "/" + self._getConfirmationFileName(ibrain2DatasetId)
def _prop(self, name, value):
return name + " = " + value + "\n"
def _writeConfirmationFile(self, ibrain2DatasetId, fileContent, incoming):
- confirmationFile = self._getStatusFileName(ibrain2DatasetId, incoming)
- file = open(confirmationFile, "w")
+ confirmationFile = self._getStatusFilePath(ibrain2DatasetId, incoming)
+ self._writeFile(confirmationFile, fileContent)
+
+ def _writeFile(self, file, fileContent):
+ file = open(file, "w")
file.write(fileContent)
file.close()
-
+
def createSuccessStatus(self, ibrain2DatasetId, openbisDatasetId, incoming):
fileContent = self._prop(self.STATUS_PROPERTY, self.OK)
fileContent += self._prop(AbstractMetadataParser.IBRAIN2_DATASET_ID_PROPERTY, ibrain2DatasetId)
fileContent += self._prop(self.OPENBIS_DATASET_ID_PROPERTY, openbisDatasetId)
self._writeConfirmationFile(ibrain2DatasetId, fileContent, incoming)
-
+
def createFailureStatus(self, ibrain2DatasetId, errorMessage, incoming):
fileContent = self._prop(self.STATUS_PROPERTY, self.ERROR)
fileContent += self._prop(AbstractMetadataParser.IBRAIN2_DATASET_ID_PROPERTY, ibrain2DatasetId)
fileContent += self._prop(self.ERROR_MSG_PROPERTY, errorMessage)
self._writeConfirmationFile(ibrain2DatasetId, fileContent, incoming)
-
-
+
+# --------------
+
+"""
+Returns:
+ (plateSpace, plateCode) tuple for the plate connected with the specified dataset
+ or (error-message, None) if the dataset does not exist or is not connected to the plate.
+"""
+def tryGetConnectedPlate(state, openbisDatasetId):
+ openbis = state.getOpenBisService()
+ dataset = openbis.tryGetDataSet(openbisDatasetId)
+ if dataset != None:
+ plate = dataset.getSample()
+ if plate != None:
+ return (plate.getSpace().getCode(), plate.getCode())
+ else:
+ return ("No plate is connected to the dataset: "+openbisDatasetId+".", None)
+ else:
+ return ("Dataset does not exist or is not accessible: "+openbisDatasetId+". "+
+ "Maybe the dataset has not been registered yet. Try again later.", None)
+
+def createSuccessStatus(iBrain2DatasetId, dataset, incomingPath):
+ datasetCode = dataset.getDataSetCode()
+ RegistrationConfirmationUtils().createSuccessStatus(iBrain2DatasetId, datasetCode, incomingPath)
+
+def createFailureStatus(iBrain2DatasetId, throwable, incoming):
+ RegistrationConfirmationUtils().createFailureStatus(iBrain2DatasetId, throwable.getMessage(), incoming.getPath())
+
# -------------- TODO: remove tests
TEST_DIR = "/Users/tpylak/main/src/screening-demo/biozentrum/dropboxes/ibrain2-dropboxes-test"
diff --git a/integration-tests/templates/data-ibrain2/scripts/commonImageDropbox.py b/integration-tests/templates/data-ibrain2/scripts/commonImageDropbox.py
index 60372dc0da4..c65b119c3e6 100755
--- a/integration-tests/templates/data-ibrain2/scripts/commonImageDropbox.py
+++ b/integration-tests/templates/data-ibrain2/scripts/commonImageDropbox.py
@@ -5,26 +5,31 @@ from ch.systemsx.cisd.openbis.dss.etl.dto.api.v1 import *
from ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto import Geometry
class IBrain2ImageDataSetConfig(SimpleImageDataConfig):
- THUMBANAIL_SIZE = 200
-
+ THUMBANAIL_SIZE = 200
+
def extractImageMetadata(self, imagePath):
- image_tokens = ImageMetadata()
-
- basename = os.path.splitext(imagePath)[0]
- #
+ basename = self.getFileBasename(imagePath)
token_dict = {}
for token in basename.split("_"):
token_dict[token[:1]] = token[1:]
-
+
+ image_tokens = ImageMetadata()
image_tokens.well = token_dict["w"]
- fieldText = token_dict["s"]
+ image_tokens.tileNumber = self.fieldAsInt(token_dict["s"], basename)
+ image_tokens.channelCode = self.extractChannelCode(token_dict, basename)
+ return image_tokens
+
+ def extractChannelCode(self, token_dict, basename):
+ return token_dict["c"]
+
+ def getFileBasename(self, filePath):
+ return os.path.splitext(filePath)[0]
+
+ def fieldAsInt(self, fieldText, basename):
try:
- image_tokens.tileNumber = int(fieldText)
+ return int(fieldText)
except ValueError:
raise Exception("Cannot parse field number from '" + fieldText + "' in '" + basename + "' file name.")
-
- image_tokens.channelCode = basename.split("_")[-1] + " ("+ token_dict["c"] + ")"
- return image_tokens
def geom(self, row, col):
return Geometry.createFromRowColDimensions(row, col)
@@ -48,4 +53,8 @@ class IBrain2ImageDataSetConfig(SimpleImageDataConfig):
else:
return self.geom(maxTileNumber, 1)
+class IBrain2SegmentationImageDataSetConfig(IBrain2ImageDataSetConfig):
+ def extractChannelCode(self, token_dict, basename):
+ return basename.split("_")[-1] + " ("+ token_dict["c"] + ")"
+
\ No newline at end of file
diff --git a/integration-tests/templates/data-ibrain2/scripts/hcs_image_overview.py b/integration-tests/templates/data-ibrain2/scripts/hcs_image_overview.py
index e69de29bb2d..0162c6413ee 100755
--- a/integration-tests/templates/data-ibrain2/scripts/hcs_image_overview.py
+++ b/integration-tests/templates/data-ibrain2/scripts/hcs_image_overview.py
@@ -0,0 +1,53 @@
+#! /usr/bin/env python
+
+import commonImageDropbox
+import commonDropbox
+
+reload(commonImageDropbox)
+reload(commonDropbox)
+
+
+def setPropertiesAndRegister(imageDataset, iBrain2DatasetId, metadataParser, incoming, factory):
+ imageRegistrationDetails = factory.createImageRegistrationDetails(imageDataset, incoming)
+ for propertyCode, value in metadataParser.getDatasetPropertiesIter():
+ imageRegistrationDetails.setPropertyValue(propertyCode, value)
+
+ tr = service.transaction(incoming, factory)
+ dataset = tr.createNewDataSet(imageRegistrationDetails)
+ imageDataSetFolder = tr.moveFile(incoming.getPath(), dataset)
+ tr.commit()
+ commonDropbox.createSuccessStatus(iBrain2DatasetId, dataset, incoming.getPath())
+
+# -------
+
+# Global variable where we set the iBrain2 id of the dataset at the beginning,
+# so that the rollback can use it as well.
+iBrain2DatasetId = None
+
+def register(incomingPath):
+ metadataParser = commonDropbox.DerivedDatasetMetadataParser(incomingPath)
+ iBrain2DatasetId = metadataParser.getIBrain2DatasetId()
+ openbisDatasetParent = metadataParser.getParentDatasetPermId()
+
+ (spaceOrError, plate) = commonDropbox.tryGetConnectedPlate(state, openbisDatasetParent)
+ if plate == None:
+ commonDropbox.RegistrationConfirmationUtils().createFailureStatus(iBrain2DatasetId, spaceOrError, incomingPath)
+ return
+
+ imageDataset = commonImageDropbox.IBrain2ImageDataSetConfig()
+ imageDataset.setOverviewImageDatasetType()
+ imageDataset.setPlate(spaceOrError, plate)
+ imageDataset.setFileFormatType("JPG")
+ imageDataset.setRecognizedImageExtensions(["jpg", "jpeg", "png", "gif"])
+ imageDataset.setStoreChannelsOnExperimentLevel(False)
+ imageDataset.setGenerateThumbnails(True)
+ imageDataset.setMaxThumbnailWidthAndHeight(imageDataset.THUMBANAIL_SIZE)
+
+ setPropertiesAndRegister(imageDataset, iBrain2DatasetId, metadataParser, incoming, factory)
+
+def rollback_transaction(service, transaction, algorithmRunner, throwable):
+ commonDropbox.createFailureStatus(iBrain2DatasetId, throwable, incoming)
+
+if incoming.isDirectory():
+ register(incoming.getPath())
+
diff --git a/integration-tests/templates/data-ibrain2/scripts/hcs_image_raw.py b/integration-tests/templates/data-ibrain2/scripts/hcs_image_raw.py
index b5446e31576..4ed722e4f4e 100755
--- a/integration-tests/templates/data-ibrain2/scripts/hcs_image_raw.py
+++ b/integration-tests/templates/data-ibrain2/scripts/hcs_image_raw.py
@@ -1,7 +1,10 @@
#! /usr/bin/env python
-from commonImageDropbox import IBrain2ImageDataSetConfig
-from commonDropbox import AcquiredDatasetMetadataParser
+import commonImageDropbox
+import commonDropbox
+
+reload(commonImageDropbox)
+reload(commonDropbox)
""" Plate geometry which will be used. Other possible value: 96_WELLS_8X12 """
PLATE_GEOMETRY = "384_WELLS_16X24"
@@ -14,8 +17,8 @@ PLATE_GEOMETRY_PROPERTY_CODE = "$PLATE_GEOMETRY"
def createPlateWithExperimentIfNeeded(transaction, assayParser, plate, space):
project = assayParser.get(assayParser.EXPERIMENTER_PROPERTY)
experiment = assayParser.get(assayParser.ASSAY_ID_PROPERTY)
- experimentDesc = assayParser.get(assayParser.ASSAY_DESC_PROPERTY)
- experimentType = assayParser.get(assayParser.ASSAY_TYPE_PROPERTY)
+ experimentDesc = assayParser.get(assayParser.ASSAY_DESC_PROPERTY)
+ experimentType = assayParser.get(assayParser.ASSAY_TYPE_PROPERTY)
sampleIdentifier = "/"+space+"/"+plate
plate = transaction.getSample(sampleIdentifier)
@@ -24,48 +27,46 @@ def createPlateWithExperimentIfNeeded(transaction, assayParser, plate, space):
experiment = transaction.getExperiment(expIdentifier)
if experiment == None:
experiment = transaction.createNewExperiment(expIdentifier, SIRNA_EXP_TYPE)
- openbisExpDesc = experimentDesc + "\ntype: "+experimentType
+ openbisExpDesc = experimentDesc + " (type: "+experimentType + ")"
experiment.setPropertyValue("DESCRIPTION", openbisExpDesc)
plate = transaction.createNewSample(sampleIdentifier, PLATE_TYPE_CODE)
plate.setPropertyValue(PLATE_GEOMETRY_PROPERTY_CODE, PLATE_GEOMETRY)
plate.setExperiment(experiment)
+ return plate
-"""
-TODO:
--
-"""
-if incoming.isDirectory():
- imageDataset = IBrain2ImageDataSetConfig()
- imageDataset.setRawImageDatasetType()
- metadataParser = AcquiredDatasetMetadataParser(incoming)
- assayParser = AssayParser(incoming)
- plate = metadataParser.getPlateCode()
- space = assayParser.get(assayParser.LAB_LEADER_PROPERTY)
+iBrain2DatasetId = None
+
+if incoming.isDirectory():
+ incomingPath = incoming.getPath()
+ metadataParser = commonDropbox.AcquiredDatasetMetadataParser(incomingPath)
+ iBrain2DatasetId = metadataParser.getIBrain2DatasetId()
+ assayParser = commonDropbox.AssayParser(incomingPath)
- imageDataset.setPlate(space, plate)
+ imageDataset = commonImageDropbox.IBrain2ImageDataSetConfig()
+ imageDataset.setRawImageDatasetType()
imageDataset.setFileFormatType("TIFF")
+ imageDataset.setRecognizedImageExtensions(["tif", "tiff"])
+ imageDataset.setStoreChannelsOnExperimentLevel(False)
imageDataset.setGenerateThumbnails(True)
imageDataset.setMaxThumbnailWidthAndHeight(imageDataset.THUMBANAIL_SIZE)
- imageDataset.setRecognizedImageExtensions(["tif, tiff"])
- imageDataset.setStoreChannelsOnExperimentLevel(False)
imageRegistrationDetails = factory.createImageRegistrationDetails(imageDataset, incoming)
- for propertyCode, value in metadataParser.getPropertiesIter():
+ for propertyCode, value in metadataParser.getDatasetPropertiesIter():
imageRegistrationDetails.setPropertyValue(propertyCode, value)
tr = service.transaction(incoming, factory)
- createPlateWithExperimentIfNeeded(tr, assayParser, plate, space)
+ plate = metadataParser.getPlateCode()
+ space = assayParser.get(assayParser.LAB_LEADER_PROPERTY)
+ plate = createPlateWithExperimentIfNeeded(tr, assayParser, plate, space)
+
dataset = tr.createNewDataSet(imageRegistrationDetails)
- imageDataSetFolder = tr.moveFile(incoming.getPath(), dataset)
- imageDatasetCode = dataset.getDataSetCode()
- IBRAIN2Utils().createSuccessStatus(metadataParser.getIBrain2DatasetId(), imageDatasetCode, incoming)
- print "Registered dataset:", imageDatasetCode
+ dataset.setSample(plate)
+ imageDataSetFolder = tr.moveFile(incomingPath, dataset)
+ tr.commit()
+ commonDropbox.createSuccessStatus(iBrain2DatasetId, dataset, incomingPath)
-# TODO: test this !!!
def rollback_transaction(service, transaction, algorithmRunner, throwable):
- incoming = service.incomingDataSetFile
- iBrain2DatasetId = AcquiredDatasetMetadataParser(incoming).getIBrain2DatasetId()
- IBRAIN2Utils().createFailureStatus(iBrain2DatasetId, throwable.getMessage(), incoming)
+ commonDropbox.createFailureStatus(iBrain2DatasetId, throwable, incoming)
\ No newline at end of file
diff --git a/integration-tests/templates/data-ibrain2/scripts/hcs_image_segmentation.py b/integration-tests/templates/data-ibrain2/scripts/hcs_image_segmentation.py
index 409e1a3331d..c61504efbf6 100755
--- a/integration-tests/templates/data-ibrain2/scripts/hcs_image_segmentation.py
+++ b/integration-tests/templates/data-ibrain2/scripts/hcs_image_segmentation.py
@@ -1,13 +1,9 @@
#! /usr/bin/env python
# This is a dropbox for importing HCS segmentation image datasets
-import IBrain2ImageDataSetConfig from common-image-dropbox
-
-"""
-TODO:
-- check if parent exists and exit otherwise (ask Eva)
--
-"""
+from commonImageDropbox import IBrain2ImageDataSetConfig
+from ch.systemsx.cisd.openbis.dss.etl.dto.api.v1 import *
+
if incoming.isDirectory():
imageDataset = IBrain2ImageDataSetConfig()
imageDataset.setSegmentationImageDatasetType()
--
GitLab