BDLSRFortessaDropbox.py

"""
@author: Aaron Ponti
"""
import re
import os
import logging
import xml.etree.ElementTree as xml
from datetime import datetime

class Processor:
    """The Processor class performs all steps required for registering datasets
    from the assigned dropbox folder."""

    # A transaction object passed by openBIS
    transaction = None

    # The incoming folder to process (a java.io.File object)
    incoming = ""

    # Constructor
    def __init__(self, transaction, logFile):

        self.transaction = transaction
        self.incoming = transaction.getIncoming()

        # Set up logging
        logging.basicConfig(filename=logFile, level=logging.DEBUG)

    def createExperiment(self, expId, expName,
                         expType="LSR_FORTESSA_EXPERIMENT"):
        """Create an experiment with given Experiment ID extended with the addition
        of a string composed from current date and time.

        @param expID, the experiment ID
        @param expName, the experiment name
        @param expType, the experiment type that must already exist; optional,
        default is "LSR_FORTESSA_EXPERIMENT"
        """

        # Make sure to keep the code length within the limits imposed by
        # openBIS for codes
        if len(expId) > 41:
            expId = expId[0:41]

        # Create univocal ID
        expId = expId + "_" + self.getCustomTimeStamp()

        # Create the experiment
        logging.info("Register experiment %s" % expId)
        exp = self.transaction.createNewExperiment(expId, expType)
        if not exp:
            msg = "Could not create experiment " + expId + "!"
            logging.error(msg)
            raise Exception(msg)
        else:
            logging.info("Created experiment with ID " + expId + ".")

        # Store the name
        exp.setPropertyValue("LSR_FORTESSA_EXPERIMENT_NAME", expName)

        return exp

    def createSampleWithGenCode(self, spaceCode,
                                sampleType="LSR_FORTESSA_PLATE"):
        """Create a sample with automatically generated code.

        @param spaceCode, the code of the space
        @param sampleType, the sample type that must already exist
        @return sample An ISample
        """

        # Make sure there are not slashes in the spaceCode
        spaceCode = spaceCode.replace("/", "")

        # Create the sample
        sample = self.transaction.createNewSampleWithGeneratedCode(spaceCode, sampleType)
        if not sample:
            msg = "Could not create sample with generated code"
            logging.error(msg)
            raise Exception(msg)

        return sample

    def formatExpDateForPostgreSQL(self, expDate):
        """Format the experiment date to be compatible with postgreSQL's
        'timestamp' data type

        @param Date stored in the FCS file, in the form 01-JAN-2013
        @return Date in the form 2013-01-01
        """

        monthMapper = {'JAN': '01', 'FEB': '02', 'MAR': '03', 'APR': '04',
                       'MAY': '05', 'JUN': '06', 'JUL': '07', 'AUG': '08',
                       'SEP': '09', 'OCT': '10', 'NOV': '11', 'DEC': '12'}

        # Separate the date into day, month, and year
        (day, month, year) = expDate.split("-")

        # Try mapping the month to digits (e.g. "06"). If the mapping does
        # not work, return "NOT_FOUND"
        month = monthMapper.get(month, "NOT_FOUND")

        # Build the date in the correct format. If the month was not found,
        # return 01-01-1970
        if (month == "NOT_FOUND"):
            logging.info("Invalid experiment date %s found. " \
                         "Reverting to 1970/01/01." % expDate)
            return "1970-01-01"
        else:
            return (year + "-" + month + "-" + day)

    def getCustomTimeStamp(self):
        """Create an univocal time stamp based on the current date and time
        (works around incomplete API of Jython 2.5).
        """

        t = datetime.now()
        return (t.strftime("%y%d%m%H%M%S") + unicode(t)[20:])

    def getSubFolders(self):
        """Returns a list of subfolders of the passed incoming directory.

        @return list of subfolders (String)
        """

        incomingStr = self.incoming.getAbsolutePath()
        return [name for name in os.listdir(incomingStr)
                if os.path.isdir(os.path.join(incomingStr, name))]

    def processExperiment(self, experimentNode,
                          openBISExpType="LSR_FORTESSA_EXPERIMENT"):
        """Register an IExperimentUpdatable based on the Experiment XML node.

        @param experimentNode An XML node corresponding to an Experiment
        @param openBISExpType The experiment type
        @return IExperimentUpdatable experiment
        """

        # Get the openBIS identifier
        openBISIdentifier = experimentNode.attrib.get("openBISIdentifier")

        # Get the experiment name
        expName = experimentNode.attrib.get("name")

        # Get the experiment date and reformat it to be compatible
        # with postgreSQL
        expDate = self.formatExpDateForPostgreSQL(experimentNode.attrib.get("date"))

        # Get the description
        description = experimentNode.attrib.get("description")

        # Get the acquisition hardware
        acqHardware = experimentNode.attrib.get("acq_hardware")

        # Get the acquisition software
        acqSoftware = experimentNode.attrib.get("acq_software")

        # Get the owner name
        owner = experimentNode.attrib.get("owner_name")

        # Create the experiment (with corrected ID if needed: see above)
        openBISExperiment = self.createExperiment(openBISIdentifier,
                                                  expName, openBISExpType)
        if not openBISExperiment:
            msg = "Could not create experiment " + openBISIdentifier
            logging.error(msg)
            raise Exception(msg)

        # Set the date
        openBISExperiment.setPropertyValue("LSR_FORTESSA_EXPERIMENT_DATE",
                                           expDate)
        # Set the description
        openBISExperiment.setPropertyValue("LSR_FORTESSA_EXPERIMENT_DESCRIPTION",
                                           description)

        # Set the acquisition hardware
        openBISExperiment.setPropertyValue("LSR_FORTESSA_EXPERIMENT_ACQ_HARDWARE",
                                           acqHardware)

        # Set the acquisition software
        openBISExperiment.setPropertyValue("LSR_FORTESSA_EXPERIMENT_ACQ_SOFTWARE",
                                           acqSoftware)

        # Set the experiment owner
        openBISExperiment.setPropertyValue("LSR_FORTESSA_EXPERIMENT_OWNER",
                                           owner)

        # Return the openBIS Experiment object
        return openBISExperiment

    def processFCSFile(self, fcsFileNode, openBISTube, openBISExperiment):
        """Register the FCS File using the parsed properties file

        @param fcsFileNode An XML node corresponding to an FCS file (dataset)
        @param openBISTube  An ISample object representing a Tube or Well
        @param openBISExperiment An ISample object representing an Experiment
        """

        # Dataset type
        datasetType = "LSR_FORTESSA_FCSFILE"

        # Create a new dataset
        dataset = self.transaction.createNewDataSet()
        if not dataset:
            msg = "Could not get or create dataset"
            logging.error(msg)
            raise Exception(msg)

        # Set the dataset type
        dataset.setDataSetType(datasetType)

        # Assign the dataset to the sample
        dataset.setSample(openBISTube)

        # Assign the dataset to the experiment
        dataset.setExperiment(openBISExperiment)

        # Set the file type
        dataset.setFileFormatType("FCS")

        # Assign the file to the dataset (we will use the absolute path)
        fileName = fcsFileNode.attrib.get("relativeFileName")
        fileName = os.path.join(self.incoming.getAbsolutePath(), fileName)

        # Log
        logging.info("PROCESSFCSFILE: Registering file: " + fileName)

        # Move the file
        self.transaction.moveFile(fileName, dataset)


    def processTray(self, trayNode, openBISExperiment):
        """Register a Tray (Plate) based on the Tray XML node
        and an IExperimentUpdatable object

        @param trayNode An XML node corresponding to a Tray (Plate)
        @param openBISExperiment An IExperimentUpdatable object
        @param openBISSampleType sample type (default "LSR_FORTESSA_PLATE")
        @return ISample sample, or null
        """

        # openBIS sample type
        openBISSampleType = "LSR_FORTESSA_PLATE"

        # Get the identifier of the space all relevant attributes
        openBISSpaceIdentifier = \
            trayNode.attrib.get("openBISSpaceIdentifier")

        # Get the tray name
        name = trayNode.attrib.get("name")

        # Get the tray geometry
        trayGeometry = trayNode.attrib.get("trayGeometry")

        # Create the sample. The Plate is configured in openBIS to
        # auto-generate its own identifier.
        openBISTray = self.createSampleWithGenCode(openBISSpaceIdentifier,
                                                   openBISSampleType)
        if not openBISTray:
            msg = "Could not create plate sample."
            logging.error(msg)
            raise Exception(msg)

        # Set the experiment for the sample
        openBISTray.setExperiment(openBISExperiment)

        # Set the plate name
        openBISTray.setPropertyValue("LSR_FORTESSA_PLATE_NAME", name)

        # Set the tray geometry
        openBISTray.setPropertyValue("LSR_FORTESSA_PLATE_GEOMETRY", trayGeometry)

        # Return the openBIS ISample object
        return openBISTray


    def processTubeOrWell(self, tubeNode, openBISContainerSample,
                          specimenName, openBISExperiment):
        """Register a Tube or Well (as a child of a Specimen) based on the Tube or
        Well XML node and an ISample object.

        The associated fcs file is attached as a IDataset

        @param tubeNode An XML node corresponding to a Tube or Well
        @param openBISContainerSample  An ISample object that will contain
        the Tube or Well
        @param specimenName Name of the specimen associated to the Tube or Well
        @param openBISExperiment The IExperiment to which the Tube belongs
        @param openBISSpecimenType (default "LSR_FORTESSA_TUBE"), the
        sample type. One of LSR_FORTESSA_TUBE and LSR_FORTESSA_WELL.
        @return ISample sample, or null
        """

        # Get the name
        name = tubeNode.attrib.get("name")

        # openBIS type
        if tubeNode.tag == "Tube":
            openBISSpecimenType = "LSR_FORTESSA_TUBE"
        elif tubeNode.tag == "Well":
            openBISSpecimenType = "LSR_FORTESSA_WELL"
        else:
            msg = "Unknown tube type" + tubeNode.tag
            logging.error(msg)
            raise Exception(msg)

        # Build the openBIS Identifier
        openBISSpaceIdentifier = \
            tubeNode.attrib.get("openBISSpaceIdentifier")

        # Create the sample. The Tube/Well is configured in openBIS to
        # auto-generate its own identifier.
        openBISTube = self.createSampleWithGenCode(openBISSpaceIdentifier,
                                                   openBISSpecimenType)
        if not openBISTube:
            msg = "Could not create sample with auto-generated identifier"
            logging.error(msg)
            raise Exception(msg)

        # Set the experiment to which it belongs
        openBISTube.setExperiment(openBISExperiment)

        # Set the Specimen name as a property
        openBISTube.setPropertyValue("LSR_FORTESSA_SPECIMEN", specimenName)

        # Set the name
        if openBISSpecimenType == "LSR_FORTESSA_TUBE":
            openBISTube.setPropertyValue("LSR_FORTESSA_TUBE_NAME", name)
        elif openBISSpecimenType == "LSR_FORTESSA_WELL":
            openBISTube.setPropertyValue("LSR_FORTESSA_WELL_NAME", name)
        else:
            msg = "Unknown value for openBISSpecimenType."
            logging.error(msg)
            raise Exception(msg)

        # Set the TubeSet container
        openBISTube.setContainer(openBISContainerSample)

        # Return the openBIS ISample
        return openBISTube

    def processTubeSet(self, experimentNode, openBISExperiment):
        """Register a TubeSet (virtual tube container).

        @param experimentNode An XML node corresponding to an Experiment
        @param openBISExperiment An IExperimentUpdatable object
        @param openBISSampleType  The TubeSet sample type
        @return ISample sample, or null
        """

        # Sample type
        openBISSampleType = "LSR_FORTESSA_TUBESET"

        # Get the identifier of the space all relevant attributes
        openBISSpaceIdentifier = \
            experimentNode.attrib.get("openBISSpaceIdentifier")

        # Create the sample. The Tubeset is configured in openBIS to
        # auto-generate its own identifier.
        openBISTubeSet = self.createSampleWithGenCode(openBISSpaceIdentifier,
                                                      openBISSampleType)
        if not openBISTubeSet:
            msg = "Could not get or create TubeSet"
            logging.error(msg)
            raise Exception(msg)

        # Set the experiment for the sample
        openBISTubeSet.setExperiment(openBISExperiment)

        logging.info("PROCESS_TUBESET: Created new TubeSet " \
                     "with identifier %s, sample type %s" \
                     % (openBISTubeSet.getSampleIdentifier(),
                        openBISSampleType))

        # Return the openBIS ISample object
        return openBISTubeSet

    def register(self, tree):
        """Register the Experiment using the parsed properties file.

        @param tree ElementTree parsed from the properties XML file
        """

        # Get the root node (obitXML)
        root = tree.getroot()

        # Create a virtual TubeSet
        openBISTubeSet = None

        # Iterate over the children (Experiments)
        for experimentNode in root:

            # The tag of the immediate children of the root experimentNode
            # must be Experiment
            if experimentNode.tag != "Experiment":
                msg = "Expected Experiment node, found " + experimentNode.tag
                logging.error(msg)
                raise Exception(msg)

            # Process an Experiment XML node and get/create an IExperimentUpdatable
            openBISExperiment = self.processExperiment(experimentNode,
                                                       "LSR_FORTESSA_EXPERIMENT")

            # Process children of the Experiment
            for childNode in experimentNode:

                # The child of an Experiment can be a Tray or a Specimen
                nodeType = childNode.tag

                if nodeType == "Specimen":

                    # A specimen is a direct child of an experiment if there
                    # is no plate, and the FCS files are therefore associated
                    # to tubes. In this case, we create a virtual TubeSet
                    # sample container (one for all Tubes in the experiment).
                    if openBISTubeSet is None:
                        openBISTubeSet = self.processTubeSet(experimentNode,
                                                             openBISExperiment)

                    # The only information we need from the Specimen is its
                    # name to associate to the Tubes as property
                    specimenNameProperty = childNode.attrib.get("name")

                    # Now iterate over the children of the Specimen
                    for tubeNode in childNode:

                        # The child of a Specimen is a Tube
                        if tubeNode.tag != "Tube":
                            msg = "Expected Tube node!"
                            logging.error(msg)
                            raise Exception(msg)

                        # Process the tube node and get the openBIS object
                        openBISTube = self.processTubeOrWell(tubeNode,
                                                       openBISTubeSet,
                                                       specimenNameProperty,
                                                       openBISExperiment)

                        # Now process the FCS file
                        for fcsNode in tubeNode:

                            # The child of a Tube is an FCSFile
                            if fcsNode.tag != "FCSFile":
                                msg = "Expected FSC File node!"
                                logging.error(msg)
                                raise Exception(msg)

                            # Process the FCS file node
                            self.processFCSFile(fcsNode, openBISTube,
                                                openBISExperiment)

                elif nodeType == "Tray":

                    # Process the tray node and get the openBIS object
                    openBISTray = self.processTray(childNode,
                                                   openBISExperiment)

                    # Now iterate over the children of the Tray
                    for specimenNode in childNode:

                        # The child of a Tray is a Specimen
                        if specimenNode.tag != "Specimen":
                            msg = "Expected Specimen node!"
                            logging.error(msg)
                            raise Exception(msg)

                        # The only information we need from the Specimen is its
                        # name to associate to the Wells as property
                        specimenNameProperty = specimenNode.attrib.get("name")

                        for wellNode in specimenNode:

                            # The child of a Specimen is a Tube
                            if wellNode.tag != "Well":
                                msg = "Expected Well node!"
                                logging.error(msg)
                                raise Exception(msg)

                            # Process the tube node and get the openBIS object
                            openBISWell = self.processTubeOrWell(wellNode,
                                                           openBISTray,
                                                           specimenNameProperty,
                                                           openBISExperiment)

                            # Now process the FCS file
                            for fcsNode in wellNode:

                                # The child of a Tube is an FCSFile
                                if fcsNode.tag != "FCSFile":
                                    msg = "Expected FSC File node!"
                                    logging.error(msg)
                                    raise Exception(msg)

                                # Process the FCS file node
                                self.processFCSFile(fcsNode, openBISWell,
                                                    openBISExperiment)

                else:

                    msg = "The Node must be either a Specimen or a Tray"
                    logging.error(msg)
                    raise Exception(msg)

        # Log that we are finished with the registration
        logging.info("REGISTER: Registration completed")


    def run(self):
        """Run the registration."""

        # Make sure that incoming is a folder
        if not self.incoming.isDirectory():
            msg = "Incoming MUST be a folder!"
            logging.error(msg)
            raise Exception(msg)

        # Log
        logging.info("Incoming folder: " + self.incoming.getAbsolutePath())

        # There must be just one subfolder: the user subfolder
        subFolders = self.getSubFolders()
        if len(subFolders) != 1:
            msg = "Expected user subfolder!"
            logging.error(msg)
            raise Exception(msg)

        # Set the user folder
        userFolder = os.path.join(self.incoming.getAbsolutePath(),
                                  subFolders[0])

        # In the user subfolder we must find the data_structure.ois file
        dataFileName = os.path.join(userFolder, "data_structure.ois")
        if not os.path.exists(dataFileName):
            msg = "File data_structure.ois not found!"
            logging.error(msg)
            raise Exception(msg)

        # Now read the data structure file and store all the pointers to
        # the properties files. The paths are stored relative to self.incoming,
        # so we can easily build the full file paths.
        propertiesFileList = []
        f = open(dataFileName)
        try:
            for line in f:
                line = re.sub('[\r\n]', '', line)
                propertiesFile = os.path.join(self.incoming.getAbsolutePath(),
                                            line)
                propertiesFileList.append(propertiesFile)
        finally:
            f.close()

        # Process (and ultimately register) all experiments
        for propertiesFile in propertiesFileList:

            # Log
            logging.info("* * * Processing: " + propertiesFile)

            # Read the properties file into an ElementTree
            tree = xml.parse(propertiesFile)

            # Now register the experiment
            self.register(tree)


def process(transaction):
    """Dropbox entry point.

    @param transaction, the transaction object
    """

    # Get path to containing folder
    # __file__ does not work (reliably) in Jython
    dbPath = "../core-plugins/microscopy/1/dss/drop-boxes/BDLSRFortessaDropbox"

    # Path to the logs subfolder
    logPath = os.path.join(dbPath, "logs")

    # Make sure the logs subforder exist
    if not os.path.exists(logPath):
        os.makedirs(logPath)

    # Path for the log file
    logFile = os.path.join(logPath, "registration_log.txt")

    # Create a Processor
    processor = Processor(transaction, logFile)

    # Run
    processor.run()