LMS-2069 iBrain2 dropboxes: HCS_ANALYSIS_WELL_FEATURES dropbox

SVN: 20349

integration-tests/templates/data-ibrain2/scripts/commonDropbox.py

@@ -10,6 +10,7 @@ class AbstractPropertiesParser:
 		path = os.path.join(incoming, fileName)
 		self._propertiesDict = self._parseMetadata(path)
 	
+	# Returns: name of the file
 	def _findFile(self, incoming, prefix):
 		for file in os.listdir(incoming):
 			if file.startswith(prefix):
@@ -184,30 +185,34 @@ class RegistrationConfirmationUtils:
 
 # --------------
 
-def setImageDatasetPropertiesAndRegister(imageDataset, iBrain2DatasetId, metadataParser, incoming, service, factory):
-    imageRegistrationDetails = factory.createImageRegistrationDetails(imageDataset, incoming)
-    for propertyCode, value in metadataParser.getDatasetPropertiesIter():
-        imageRegistrationDetails.setPropertyValue(propertyCode, value)
-
-    tr = service.transaction(incoming, factory)
-    dataset = tr.createNewDataSet(imageRegistrationDetails)
-    dataset.setParentDatasets([metadataParser.getParentDatasetPermId()])
-    imageDataSetFolder = tr.moveFile(incoming.getPath(), dataset)
-    if tr.commit():
+def setImageDatasetPropertiesAndRegister(imageDataset, metadataParser, incoming, service, factory):
+	iBrain2DatasetId = metadataParser.getIBrain2DatasetId()
+   	imageRegistrationDetails = factory.createImageRegistrationDetails(imageDataset, incoming)
+   	for propertyCode, value in metadataParser.getDatasetPropertiesIter():
+   		imageRegistrationDetails.setPropertyValue(propertyCode, value)
+
+	tr = service.transaction(incoming, factory)
+	dataset = tr.createNewDataSet(imageRegistrationDetails)
+	dataset.setParentDatasets([metadataParser.getParentDatasetPermId()])
+	imageDataSetFolder = tr.moveFile(incoming.getPath(), dataset)
+	if tr.commit():
 		createSuccessStatus(iBrain2DatasetId, dataset, incoming.getPath())
 
 def registerDerivedBlackBoxDataset(state, service, factory, incoming, metadataParser, datasetType, fileFormatType):
+    transaction = service.transaction(incoming, factory)
+    dataset = transaction.createNewDataSet()
+    dataset.setDataSetType(datasetType)
+    dataset.setFileFormatType(fileFormatType)
+    registerDerivedDataset(state, transaction, dataset, incoming, metadataParser)
+    
+def registerDerivedDataset(state, transaction, dataset, incoming, metadataParser):
     iBrain2DatasetId = metadataParser.getIBrain2DatasetId()
     openbisDatasetParent = metadataParser.getParentDatasetPermId()
 
     (space, plate) = tryGetConnectedPlate(state, openbisDatasetParent, iBrain2DatasetId, incoming.getPath())
     if plate == None:
         return
-    transaction = service.transaction(incoming, factory)
     
-    dataset = transaction.createNewDataSet()
-    dataset.setDataSetType(datasetType)
-    dataset.setFileFormatType(fileFormatType)
     dataset.setSample(transaction.getSample('/'+space+'/'+plate))
     dataset.setMeasuredData(False)
     for propertyCode, value in metadataParser.getDatasetPropertiesIter():
@@ -218,6 +223,12 @@ def registerDerivedBlackBoxDataset(state, service, factory, incoming, metadataPa
     if transaction.commit():
         createSuccessStatus(iBrain2DatasetId, dataset, incoming.getPath())
 
+def findCSVFile(dir):
+	for file in os.listdir(dir):
+		if file.endswith(".csv"):
+			return dir + "/" + file
+	raise Exception("No CSV file has been found in "+dir)
+
 """
 Returns:
    (plateSpace, plateCode) tuple for the plate connected with the specified dataset

integration-tests/templates/data-ibrain2/scripts/hcs_analysis_well_features.py

 #! /usr/bin/env python
 # This is an example Jython dropbox for importing feature vectors coming from analysis of image datasets
  
-import os
 from ch.systemsx.cisd.openbis.dss.etl.dto.api.v1 import *
+import commonImageDropbox
+import commonDropbox
 
+reload(commonImageDropbox)
+reload(commonDropbox)
+
+# Global variable where we set the iBrain2 id of the dataset at the beginning, 
+# so that the rollback can use it as well.
+iBrain2DatasetId = None
+    
 SEPARATOR = ","
  
-# Specific code which defines the feature vector values for the dataset..
+# Specific code which defines the feature vector values for the dataset.
 # Usually you will parse the content of the incoming file or directory to get the values.
-# Here all the values are hard-coded for simplicity,
-# but the example shows which calls you need to perform in your parser.
 # Parameters
-#     incoming: java.io.File which points to the incoming dataset
-def defineFeatures(featuresBuilder, incoming):
-    file = open(incoming.getPath())
+#     incomingCsvPath: path which points to the incoming CSV file
+def defineFeatures(incomingCsvFile):
+    featuresBuilder = factory.createFeaturesBuilder()
+    file = open(incomingCsvFile)
     for header in file:
         headerTokens = header.split(SEPARATOR)
         featureCode = headerTokens[0]
@@ -27,40 +34,24 @@ def defineFeatures(featuresBuilder, incoming):
                 value = rowTokens[column].strip()
                 well = rowLabel + str(column)
                 featureValues.addValue(well, value)
- 
-# Returns the code of the plate to which the dataset should be connected.
-# Parameters
-#     incoming: java.io.File which points to the incoming dataset
-def extractPlateCode(incoming):
-    return os.path.splitext(incoming.getName())[0]
- 
-def extractSpaceCode(incoming):
-    return "TEST"
- 
-# ----------------------------               
-# --- boilerplate code which register one dataset with image analysis results on the well level
-# --- Nothing has to be modified if your case is not complicated.
-# ----------------------------               
- 
-featuresBuilder = factory.createFeaturesBuilder()
-defineFeatures(featuresBuilder, incoming)
- 
-analysisRegistrationDetails = factory.createFeatureVectorRegistrationDetails(featuresBuilder, incoming)
-tr = service.transaction(incoming, factory)
-analysisDataset = tr.createNewDataSet(analysisRegistrationDetails)
- 
+    return featuresBuilder
+
+def register(incomingPath):
+    metadataParser = commonDropbox.DerivedDatasetMetadataParser(incomingPath)
+    global iBrain2DatasetId
+    iBrain2DatasetId = metadataParser.getIBrain2DatasetId()
+
+    transaction = service.transaction(incoming, factory)
+    incomingCsvFile = commonDropbox.findCSVFile(incomingPath)
+    featuresBuilder = defineFeatures(incomingCsvFile)
+    analysisRegistrationDetails = factory.createFeatureVectorRegistrationDetails(featuresBuilder, incoming)
+    dataset = transaction.createNewDataSet(analysisRegistrationDetails)
+    dataset.setDataSetType('HCS_ANALYSIS_WELL_FEATURES')
+    dataset.setFileFormatType('CSV')
+    commonDropbox.registerDerivedDataset(state, transaction, dataset, incoming, metadataParser)
+ 
+def rollback_transaction(service, transaction, algorithmRunner, throwable):
+    commonDropbox.createFailureStatus(iBrain2DatasetId, throwable, incoming)
+            
+if incoming.isDirectory():
+    register(incoming.getPath())
\ No newline at end of file
-# set plate to which the dataset should be connected
-sampleIdentifier = "/"+extractSpaceCode(incoming)+"/"+extractPlateCode(incoming)
-plate = tr.getSample(sampleIdentifier)
-analysisDataset.setSample(plate)
- 
-# store the original file in the dataset.
-tr.moveFile(incoming.getPath(), analysisDataset)
- 
-# ----------------------------               
-# --- optional: other standard operations on analysisDataset can be performed (see IDataSet interface)
-# ----------------------------               
- 
-analysisDataset.setFileFormatType("CSV")
-analysisDataset.setDataSetType("HCS_ANALYSIS_WELL_FEATURES")
-#analysisDataset.setParentDatasets(["20110302085840150-90"])
\ No newline at end of file

integration-tests/templates/data-ibrain2/scripts/hcs_analysis_well_quality.py

@@ -19,16 +19,7 @@ def register(incomingPath):
     if plate == None:
         return
     
-    imageDataset = commonImageDropbox.IBrain2ImageDataSetConfig()
-    imageDataset.setOverviewImageDatasetType()
-    imageDataset.setPlate(space, plate)
-    imageDataset.setFileFormatType("JPG")
-    imageDataset.setRecognizedImageExtensions(["jpg", "jpeg", "png", "gif"])    
-    imageDataset.setStoreChannelsOnExperimentLevel(False)
-    imageDataset.setGenerateThumbnails(True)
-    imageDataset.setMaxThumbnailWidthAndHeight(imageDataset.THUMBANAIL_SIZE)
-
-    commonDropbox.setPropertiesAndRegister(imageDataset, iBrain2DatasetId, metadataParser, incoming, service, factory)
+    #TODO: finish me !!!!
 
 def rollback_transaction(service, transaction, algorithmRunner, throwable):
     commonDropbox.createFailureStatus(iBrain2DatasetId, throwable, incoming)

integration-tests/templates/data-ibrain2/scripts/hcs_image_overview.py

@@ -29,7 +29,7 @@ def register(incomingPath):
     imageDataset.setGenerateThumbnails(True)
     imageDataset.setMaxThumbnailWidthAndHeight(imageDataset.THUMBANAIL_SIZE)
 
-    commonDropbox.setImageDatasetPropertiesAndRegister(imageDataset, iBrain2DatasetId, metadataParser, incoming, service, factory)
+    commonDropbox.setImageDatasetPropertiesAndRegister(imageDataset, metadataParser, incoming, service, factory)
 
 def rollback_transaction(service, transaction, algorithmRunner, throwable):
     commonDropbox.createFailureStatus(iBrain2DatasetId, throwable, incoming)

integration-tests/templates/data-ibrain2/scripts/hcs_image_segmentation.py

@@ -33,7 +33,7 @@ def register(incomingPath):
     imageDataset.setStoreChannelsOnExperimentLevel(False)
     imageDataset.setOriginalDataStorageFormat(OriginalDataStorageFormat.HDF5)
   
-    commonDropbox.setImageDatasetPropertiesAndRegister(imageDataset, iBrain2DatasetId, metadataParser, incoming, service, factory)
+    commonDropbox.setImageDatasetPropertiesAndRegister(imageDataset, metadataParser, incoming, service, factory)
 
 def rollback_transaction(service, transaction, algorithmRunner, throwable):
     commonDropbox.createFailureStatus(iBrain2DatasetId, throwable, incoming)