LMS-1992 template dropbox: add documentation

SVN: 19688

screening/etc/data-set-handler.py

@@ -12,36 +12,194 @@ from ch.systemsx.cisd.openbis.dss.etl.dto.api.v1 import *
 from ch.systemsx.cisd.openbis.dss.etl.custom.geexplorer import GEExplorerImageAnalysisResultParser
 from java.io import File
 
+# ------------
+# Generic utility
+# ------------
+
+""" 
+Finds first occurence of the patter from the right.
+Throws exception if the pattern cannot be found.
+"""
+def rfind(text, pattern):
+    ix = text.rfind(pattern)
+    ensurePatternFound(ix, text, pattern)
+    return ix
+
+""" 
+Finds first occurence of the patter from the left. 
+Throws exception if the pattern cannot be found.
+"""
+def find(text, pattern):
+    ix = text.find(pattern)
+    ensurePatternFound(ix, text, pattern)
+    return ix
+
+def ensurePatternFound(ix, file, pattern):
+    if ix == -1:
+        raise Exception("Cannot find '" + pattern + "' pattern in file name '" + file + "'")    
+
+""" Returns: name of the file without the extension """
+def extract_file_basename(filename):
+    lastDot = filename.rfind(".")
+    if lastDot != -1:
+        return filename[0:lastDot]
+    else:
+        return filename
+
+""" Returns: extension of the file """
+def get_file_ext(file):
+    return os.path.splitext(file)[1][1:].lower()
+
+""" Returns: java.io.File - first file with the specified extension or None if no file matches """
+def find_file_by_ext(incoming_file, expected_ext):
+    if not incoming_file.isDirectory():
+        return None
+    incoming_path = incoming_file.getPath()
+    for file in os.listdir(incoming_path):
+        ext = get_file_ext(file)
+        if ext.upper() == expected_ext.upper():
+            return File(incoming_path, file)
+    return None
+
+""" Returns: java.io.File - subdirectory which contains the specified marker in the name """
+def find_dir(incoming_file, dir_name_marker):
+    if not incoming_file.isDirectory():
+        return None
+    incoming_path = incoming_file.getPath()
+    for file in os.listdir(incoming_path):
+        if dir_name_marker.upper() in file.upper():
+            return File(incoming_path, file)
+    return None
+
+def get_random_string():
+    return str(int(random.random()*1000000000))
+
+""" 
+Creates a temporary directory two levels above the specified incoming file.
+The name of the directory will contain the specified label and a random text. 
+Returns:
+    java.io.File - path to the temporary directory
+"""
+def get_tmp_dir(incoming, label):
+    dropbox_parent_dir = incoming.getParentFile().getParent()
+    tmp_dir = File(dropbox_parent_dir, "tmp")
+    if not os.path.exists(tmp_dir.getPath()):
+        os.mkdir(tmp_dir.getPath())
+    tmp_labeled_dir = File(tmp_dir, label + ".tmp." + get_random_string())
+    os.mkdir(tmp_labeled_dir.getPath())
+    return tmp_labeled_dir
+
+# ------------
+# Generic dataset registration
+# ------------
+
+""" Basic attributes of a dataset connected to a sample and optionally to one parent dataset """
+class Dataset:
+    # code of the dataset type and file format for the new dataset
+    dataset_type = ""
+    file_format = ""
+    # attributes of the connected sample
+    sample_code = ""
+    sample_space = ""
+    # attributes of the parent dataset - optional
+    parent_dataset_code = ""
+    # boolean which marks, if a dataset is measured or derived from measured data
+    is_measured = True
+
+"""
+Sets registartion details using the specified Dataset.
+Parameters:
+   dataset - basic information about the dataset (see Dataset type)
+   registration_details - DataSetRegistrationDetails which will be filled (see javadoc)
+"""
+def set_dataset_details(dataset, registration_details):
+    data_set_info = registration_details.getDataSetInformation()
+    data_set_info.setSpaceCode(dataset.sample_space)
+    data_set_info.setSampleCode(dataset.sample_code)
+    if (dataset.parent_dataset_code):
+        data_set_info.setParentDataSetCodes([dataset.parent_dataset_code])
+    registration_details.setFileFormatType(FileFormatType(dataset.file_format));
+    registration_details.setDataSetType(DataSetType(dataset.dataset_type));
+    registration_details.setMeasuredData(dataset.is_measured);
+
+# ------------
+# Image dataset registration
+# ------------
+
+"""
+Extends basic information about dataset with image dataset specific data.
+"""
+class ImageDataset(Dataset): 
+    # location of the tile on the well matrix
+    tile_rows_number = -1
+    tile_columns_number = -1
+    # list of ImageFileInfo objects. Describes each image in the dataset.
+    images = None
+    # list of Channel objects. Describes all channels of the dataset.
+    channels = None
+
+"""
+Auxiliary function to extract all channel codes used by specified images.
+The channel label will be equal to channel code.
+Parameters:
+    images - list of ImageFileInfo
+Returns: 
+    list of Channel
+"""
+def get_available_channels(images):
+    channel_codes = {}
+    for image in images:
+        channel_codes[image.getChannelCode()] = 1
+    channels = []
+    for channelCode in channel_codes.keys():
+        channels.append(Channel(channelCode, channelCode))
+    return channels
+
+"""
+Converts ImageDataset to DataSetRegistrationDetails
+Parameters:
+    image_dataset - ImageDataset
+Returns: 
+    DataSetRegistrationDetails
+"""
+def create_image_dataset_details(image_dataset):
+    registration_details = factory.createImageRegistrationDetails()
+    
+    set_dataset_details(image_dataset, registration_details)
+
+    data_set_info = registration_details.getDataSetInformation()
+    data_set_info.setTileGeometry(image_dataset.tile_rows_number, image_dataset.tile_columns_number)
+    data_set_info.setImages(image_dataset.images)
+    data_set_info.setChannels(image_dataset.channels)
+    return registration_details
+    
 # ------------
 # Dropbox specific image dataset registration. You may want to modify this part.
 # ------------
 
 """ type of the new image dataset """
 IMAGE_DATASET_TYPE = "HCS_IMAGE"
-""" file format of files in a new image dataset """
+""" file format code of files in a new image dataset """
 IMAGE_FILE_FORMAT = "TIFF"
+
 """ type of the new analysis dataset """
 ANALYSIS_DATASET_TYPE = "HCS_IMAGE_ANALYSIS_DATA"
-""" file format of the analysis dataset file """
+""" file format of the analysis dataset """
 ANALYSIS_FILE_FORMAT = "CSV"
 
+""" type of the new image overlay dataset """
 OVERLAY_IMAGE_DATASET_TYPE = "HCS_IMAGE_SEGMENTATION_OVERLAY"
+""" file format of the image overlay dataset """
 OVERLAY_IMAGE_FILE_FORMAT = "PNG"
 
 """ space where the plate for which the dataset has been acquired exist """
 PLATE_SPACE = "DEMO"
 
+""" only files with these extensions will be recognized as images """
 RECOGNIZED_IMAGES_EXTENSIONS = ["tiff", "tif", "png", "gif", "jpg", "jpeg"]
 
 
 # ---------
-""" sample type code of the plate, needed if a new sample is registered automatically """
-PLATE_TYPE_CODE = "PLATE"
-""" project and experiment where new plates will be registered """
-DEFAULT_PROJECT_CODE = "TEST"
-DEFAULT_EXPERIMENT_CODE = "SANOFI"
-PLATE_GEOMETRY_PROPERTY_CODE = "$PLATE_GEOMETRY"
-PLATE_GEOMETRY = "384_WELLS_16X24"
 
 """ extracts code of the sample from the directory name """
 def extract_sample_code(incoming_name):
@@ -52,19 +210,26 @@ def extract_sample_code(incoming_name):
         code = file_basename
     return code
 
+""" 
+For a given tile number and tiles geometry returns a (x,y) tuple which describes where the tile
+is located on the well.
+"""
 def get_tile_coords(tile_num, tile_geometry):
     columns = tile_geometry[1]
     row = ((tile_num - 1) / columns) + 1
     col = ((tile_num - 1) % columns) + 1
     return (row, col)
 
+""" Returns: integer - maximal tile number """
 def get_max_tile_number(image_tokens_list):
     max_tile = 0
     for image_tokens in image_tokens_list:
         max_tile = max(max_tile, image_tokens.tile)
     return max_tile
 
-""" returns (rows, columns) """
+""" 
+Returns:  (rows, columns) tuple describing the matrix of tiles (aka fields or sides) in the well  
+"""
 def get_tile_geometry(image_tokens_list):
     max_tile = get_max_tile_number(image_tokens_list)
     if max_tile % 4 == 0 and max_tile != 4:
@@ -76,15 +241,22 @@ def get_tile_geometry(image_tokens_list):
     else:
         return (max_tile, 1)
 
+""" Auxiliary structure to store tokens of the image file name.  """
 class ImageTokens:
+    # channel code
     channel = None
+    # tile number
     tile = -1
+    # path to the image
     path = ""
+    # well code, e.g. A1
     well = ""
 
 """
 Creates ImageTokens for a given path to an image
 Example file name: A - 1(fld 1 wv Cy5 - Cy5).tif
+Returns:
+    ImageTokens
 """
 def create_image_tokens(path):
     image_tokens = ImageTokens()
@@ -107,7 +279,9 @@ def create_image_tokens(path):
     
 """ 
 Creates ImageFileInfo for a given path to an image
-Example file name: A - 1(fld 1 wv Cy5 - Cy5).tif
+Example of the accepted file name: A - 1(fld 1 wv Cy5 - Cy5).tif
+Returns:
+   ImageFileInfo 
 """
 def create_image_info(image_tokens, tile_geometry):
     fieldNum = image_tokens.tile - 1
@@ -118,6 +292,11 @@ def create_image_info(image_tokens, tile_geometry):
     img.setWell(image_tokens.well)
     return img
 
+"""
+Tokenizes file names of all images in the directory.
+Returns: 
+  list of ImageTokens
+"""
 def parse_image_tokens(dir):
     image_tokens_list = []
     dir_path = dir.getPath()
@@ -133,7 +312,13 @@ def parse_image_tokens(dir):
             pass # extension not recognized    
     return image_tokens_list
 
-    
+"""
+Parameters:
+- image_tokens_list - list of ImageTokens for each image
+- tile_geometry - (rows, columns) tuple describing the matrix of tiles (aka fields or sides) in the well  
+Returns: 
+  list of ImageFileInfo
+"""    
 def create_image_infos(image_tokens_list, tile_geometry):
     images = []
     for image_tokens in image_tokens_list:
@@ -141,108 +326,15 @@ def create_image_infos(image_tokens_list, tile_geometry):
         images.append(image)    
     return images
 
-# ------------
-# Generic utility
-# ------------
-
-def rfind(text, pattern):
-    ix = text.rfind(pattern)
-    ensurePatternFound(ix, text, pattern)
-    return ix
-
-def find(text, pattern):
-    ix = text.find(pattern)
-    ensurePatternFound(ix, text, pattern)
-    return ix
-
-def ensurePatternFound(ix, file, pattern):
-    if ix == -1:
-        raise Exception("Cannot find '" + pattern + "' pattern in file name '" + file + "'")    
-
-def extract_file_basename(filename):
-    lastDot = filename.rfind(".")
-    if lastDot != -1:
-        return filename[0:lastDot]
-    else:
-        return filename
-
-def get_file_ext(file):
-    return os.path.splitext(file)[1][1:].lower()
-
-def find_file_by_ext(incoming_file, expected_ext):
-    if not incoming_file.isDirectory():
-        return None
-    incoming_path = incoming_file.getPath()
-    for file in os.listdir(incoming_path):
-        ext = get_file_ext(file)
-        if ext.upper() == expected_ext.upper():
-            return File(incoming_path, file)
-    return None
-
-def find_dir(incoming_file, dir_name_marker):
-    if not incoming_file.isDirectory():
-        return None
-    incoming_path = incoming_file.getPath()
-    for file in os.listdir(incoming_path):
-        if dir_name_marker.upper() in file.upper():
-            return File(incoming_path, file)
-    return None
-
-def get_random_string():
-    return str(int(random.random()*1000000000))
-
-def get_tmp_dir(incoming, label):
-    dropbox_parent_dir = incoming.getParentFile().getParent()
-    tmp_dir = File(dropbox_parent_dir, "tmp")
-    if not os.path.exists(tmp_dir.getPath()):
-        os.mkdir(tmp_dir.getPath())
-    tmp_labeled_dir = File(tmp_dir, label + ".tmp." + get_random_string())
-    os.mkdir(tmp_labeled_dir.getPath())
-    return tmp_labeled_dir
-
-# ------------
-# Generic dataset registration
-# ------------
-
-class Dataset:
-    sample_code = ""
-    sample_space = ""
-    dataset_type = ""
-    file_format = ""
-    is_measured = True
-    parent_dataset_code = ""
-
-def set_dataset_details(dataset, registration_details):
-    data_set_info = registration_details.getDataSetInformation()
-    data_set_info.setSpaceCode(dataset.sample_space)
-    data_set_info.setSampleCode(dataset.sample_code)
-    if (dataset.parent_dataset_code):
-        data_set_info.setParentDataSetCodes([dataset.parent_dataset_code])
-    registration_details.setFileFormatType(FileFormatType(dataset.file_format));
-    registration_details.setDataSetType(DataSetType(dataset.dataset_type));
-    registration_details.setMeasuredData(dataset.is_measured);
-    return registration_details
-
-# ------------
-# Image dataset registration
-# ------------
-
-class ImageDataset: # extends Dataset
-    tile_num = -1
-    tile_rows_number = -1
-    tile_columns_number = -1
-    images = None
-    channels = None
-
-def get_available_channels(images):
-    channel_codes = {}
-    for image in images:
-        channel_codes[image.getChannelCode()] = 1
-    channels = []
-    for channelCode in channel_codes.keys():
-        channels.append(Channel(channelCode, channelCode))
-    return channels
+# ---------------------
 
+"""
+Extracts all images from the incoming directory.
+Parameters:
+    incoming - java.io.File, folder with images
+Returns:
+    ImageDataset
+"""
 def create_image_dataset(incoming):
     dataset = ImageDataset()
     
@@ -263,19 +355,15 @@ def create_image_dataset(incoming):
 
     return dataset
 
-def create_image_dataset_details(image_dataset):
-    registration_details = factory.createImageRegistrationDetails()
-    
-    set_dataset_details(image_dataset, registration_details)
-
-    data_set_info = registration_details.getDataSetInformation()
-    data_set_info.setTileGeometry(image_dataset.tile_rows_number, image_dataset.tile_columns_number)
-    data_set_info.setImages(image_dataset.images)
-    data_set_info.setChannels(image_dataset.channels)
-    return registration_details
-    
-# ---------------------
-
+"""
+Extracts all overlay images from the overlays_dir directory.
+Parameters:
+    overlays_dir - java.io.File, folder with 
+    image_dataset - ImageDataset, image dataset to which the overlay dataset belongs
+    img_dataset_code - string, code of the  image dataset to which the overlay dataset belongs
+Returns:
+    ImageDataset
+"""
 def create_overlay_dataset(overlays_dir, image_dataset, img_dataset_code):
     dataset = ImageDataset()
     
@@ -297,18 +385,34 @@ def create_overlay_dataset(overlays_dir, image_dataset, img_dataset_code):
 
     return dataset
 
+"""
+Creates registration details of the image overlays dataset.
+Parameters:
+    overlays_dir - java.io.File, folder with 
+    image_dataset - ImageDataset, image dataset to which the overlay dataset belongs
+    img_dataset_code - string, code of the  image dataset to which the overlay dataset belongs
+Returns:
+    DataSetRegistrationDetails
+"""
 def create_overlay_dataset_details(overlays_dir, image_dataset, img_dataset_code):
     overlay_dataset = create_overlay_dataset(overlays_dir, image_dataset, img_dataset_code)
     overlay_dataset_details = create_image_dataset_details(overlay_dataset)
 
     data_set_info = overlay_dataset_details.getDataSetInformation()
     config = ImageStorageConfiguraton.createDefault()
+    # channels will be connected to the dataset
     config.setStoreChannelsOnExperimentLevel(False)
     data_set_info.setImageStorageConfiguraton(config)
     return overlay_dataset_details
 
 # ---------------------
 
+"""
+Creates the analysis dataset description. 
+The dataset will be connected to the specified sample and parent dataset.
+Returns:
+    Dataset
+"""
 def create_analysis_dataset(sample_space, sample_code, parent_dataset_code):
     dataset = Dataset()
     
@@ -320,12 +424,18 @@ def create_analysis_dataset(sample_space, sample_code, parent_dataset_code):
     dataset.parent_dataset_code = parent_dataset_code
     return dataset
 
+"""
+Creates registration details of the analysis dataset.
+Returns:
+    DataSetRegistrationDetails
+"""
 def create_analysis_dataset_details(sample_space, sample_code, parent_dataset_code):
     analysis_registration_details = factory.createRegistrationDetails()
     analysis_dataset = create_analysis_dataset(sample_space, sample_code, parent_dataset_code)
     set_dataset_details(analysis_dataset, analysis_registration_details)
     return analysis_registration_details
 
+""" registers sample if it does not exist already """
 def register_sample_if_necessary(space_code, project_code, experiment_code, sample_code):   
     openbis = state.getOpenBisService()
     sampleIdentifier = SampleIdentifier.create(space_code, sample_code)