pybis.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
pybis.py

Work with openBIS from Python.

"""

import os
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

import time
from datetime import datetime
import json
import re
from urllib.parse import urlparse
import zlib
from collections import namedtuple


import pandas as pd
from pandas import DataFrame, Series

import threading
from threading import Thread
from queue import Queue
DROPBOX_PLUGIN = "jupyter-uploader-api"


def _definitions(what):
    entities = {
        "Sample": {
            "attrs": "space experiment parents children container components tags".split(),
            "ids2type": {
                'parentIds': { 'permId': { '@type': 'as.dto.sample.id.SamplePermId' } },
                'childIds':  { 'permId': { '@type': 'as.dto.sample.id.SamplePermId' } },
                'componentIds': { 'permId': {'@type': 'as.dto.sample.id.SamplePermId' } },
            },
            "identifier": "sampleId",
            "up_method": "updateSamples",
            "cre_method": "createSamples",
            "cre_type": "as.dto.sample.create.SampleCreation",
            "multi": "parents children components tags".split(),
        },
        "Experiment": {
            "attrs": "project tags".split(),
            "multi": "tags".split(),
            "identifier": "experimentId",
            "up_method": "updateExperimentss",
            "cre_method": "createExperimentss",
        },
        "DataSet": {
            "autoGeneratedCode" : True,
            "attrs": "experiment sample parents children container components tags".split(),
            
            "ids2type": {
                'parentIds': { 'permId': { '@type': 'as.dto.dataset.id.DataSetPermId' } },
                'childIds':  { 'permId': { '@type': 'as.dto.dataset.id.DataSetPermId' } }, 
            },
            "multi": [],
            "identifier": "dataSetId",
            "up_method": "updateDataSets",
            "cre_method": "createDataSets",
        },
        "attr2ids": {
            "sample"     : "sampleId",
            "experiment" : "experimentId",
            "space"      : "spaceId",
            "container"  : "containerId",
            "component"  : "componentId",
            "components" : "componentIds",
            "parents"    : "parentIds",
            "children"   : "childIds",
            "project"    : "projectId",
            "tags"       : "tagIds",
        },
        "ids2type": {
            'spaceId': { 'permId': { '@type': 'as.dto.space.id.SpacePermId' } },
            'projectId': { 'permId': { '@type': 'as.dto.project.id.ProjectPermId' } },
            'experimentId': { 'permId': { '@type': 'as.dto.experiment.id.ExperimentPermId' } },
            'tagIds': { 'code': { '@type': 'as.dto.tag.id.TagCode' } },
        },
    }
    return entities[what]


search_for_type = {
    "space":      "as.dto.space.search.SpaceSearchCriteria",
    "project":    "as.dto.project.search.ProjectSearchCriteria",
    "experiment": "as.dto.experiment.search.ExperimentSearchCriteria",
    "code":       "as.dto.common.search.CodeSearchCriteria",
    "sample_type":"as.dto.sample.search.SampleTypeSearchCriteria",
}

fetch_option = {
    "space":        { "@type": "as.dto.space.fetchoptions.SpaceFetchOptions" },
    "project":      { "@type": "as.dto.project.fetchoptions.ProjectFetchOptions" },
    "experiment":   { "@type": "as.dto.experiment.fetchoptions.ExperimentFetchOptions" },
    "sample":       { "@type": "as.dto.sample.fetchoptions.SampleFetchOptions" },
    "dataset":      { "@type": "as.dto.dataset.fetchoptions.DataSetFetchOptions" },
    "physicalData": { "@type": "as.dto.dataset.fetchoptions.PhysicalDataFetchOptions" },
    "linkedData":   { "@type": "as.dto.dataset.fetchoptions.LinkedDataFetchOptions" },


    "properties":   { "@type": "as.dto.property.fetchoptions.PropertyFetchOptions" },
    "tags":         { "@type": "as.dto.tag.fetchoptions.TagFetchOptions" },

    "registrator":  { "@type": "as.dto.person.fetchoptions.PersonFetchOptions" },
    "modifier":     { "@type": "as.dto.person.fetchoptions.PersonFetchOptions" },
    "leader":       { "@type": "as.dto.person.fetchoptions.PersonFetchOptions" },

    "attachments":  { "@type": "as.dto.attachment.fetchoptions.AttachmentFetchOptions" },
    "history":      { "@type": "as.dto.history.fetchoptions.HistoryEntryFetchOptions" },
    "dataStore":    { "@type": "as.dto.datastore.fetchoptions.DataStoreFetchOptions" },
}


def parse_jackson(input_json):
    """openBIS uses a library called «jackson» to automatically generate the JSON RPC output.
       Objects that are found the first time are added an attribute «@id».
       Any further findings only carry this reference id.
       This function is used to dereference the output.
    """
    interesting=['tags', 'registrator', 'modifier', 'type', 'parents', 
        'children', 'containers', 'properties', 'experiment', 'sample',
        'project', 'space', 'propertyType' 
    ]
    found = {} 
    def build_cache(graph):
        if isinstance(graph, list):
            for item in graph:
                build_cache(item)
        elif isinstance(graph, dict) and len(graph) > 0:
            for key, value in graph.items():
                if key in interesting:
                    if isinstance(value, dict):
                        if '@id' in value:
                            found[value['@id']] = value
                        build_cache(value)
                    elif isinstance(value, list):
                        for item in value:
                            if isinstance(item, dict):
                                if '@id' in item:
                                    found[item['@id']] = item
                                build_cache(item)
                elif isinstance(value, dict):
                    build_cache(value)
                elif isinstance(value, list):
                    build_cache(value)
                    
    def deref_graph(graph):            
        if isinstance(graph, list):
            for i, list_item in enumerate(graph):
                if isinstance(list_item, int):
                    graph[i] = found[list_item]
                else:
                    deref_graph(list_item)
        elif isinstance(graph, dict) and len(graph) > 0:
            for key, value in graph.items():
                if key in interesting:
                    if isinstance(value, dict):
                        deref_graph(value)
                    elif isinstance(value, int):
                        graph[key] = found[value]
                    elif isinstance(value, list):
                        for i, list_item in enumerate(value):
                            if isinstance(list_item, int):
                                value[i] = found[list_item]
                elif isinstance(value, dict):
                    deref_graph(value)
                elif isinstance(value, list):
                    deref_graph(value)

    build_cache(input_json)
    deref_graph(found)
    deref_graph(input_json)

def check_datatype(type_name, value):
    if type_name == 'INTEGER':
        return isinstance(value, int)
    if type_name == 'BOOLEAN':
        return isinstance(value, bool)
    if type_name == 'VARCHAR':
        return isinstance(value, str)
    return True

def _determine_ident_type(ident):
    # assume we got a sample identifier e.g. /TEST/TEST-SAMPLE
    match = re.match('/', ident)
    if match:
        return "identifier"
    else:
        return "permId"

def search_request_for_identifier(ident, entity_type):