pybis.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
pybis.py

Work with openBIS from Python.

"""

import os
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

import time
from datetime import datetime
import json
import re
from urllib.parse import urlparse
import zlib
import base64
from collections import namedtuple


import pandas as pd
from pandas import DataFrame, Series

import threading
from threading import Thread
from queue import Queue
DROPBOX_PLUGIN = "jupyter-uploader-api"


def _definitions(entity):
    entities = {
        "Sample": {
            "attrs": "code permId identifier type parents children attachments space experiment container components tags ".split(),
            "ids2type": {
                'parentIds': { 'permId': { '@type': 'as.dto.sample.id.SamplePermId' } },
                'childIds':  { 'permId': { '@type': 'as.dto.sample.id.SamplePermId' } },
                'componentIds': { 'permId': {'@type': 'as.dto.sample.id.SamplePermId' } },
            },
            "identifier": "sampleId",
            "cre_type": "as.dto.sample.create.SampleCreation",
            "multi": "parents children components tags".split(),
        },
        "Experiment": {
            "attrs": "code permId identifier type space project tags attachments".split(),
            "multi": "tags".split(),
            "identifier": "experimentId",
        },
        "Project": {
            "attrs": "code description permId identifier space attachments".split(),
            "multi": "tags".split(),
            "identifier": "projectId",
        },
        "DataSet": {
            "autoGeneratedCode" : True,
            "attrs": "experiment sample parents children container components tags".split(),
            
            "ids2type": {
                'parentIds': { 'permId': { '@type': 'as.dto.dataset.id.DataSetPermId' } },
                'childIds':  { 'permId': { '@type': 'as.dto.dataset.id.DataSetPermId' } }, 
            },
            "multi": [],
            "identifier": "dataSetId",
        },
        "attr2ids": {
            "sample"     : "sampleId",
            "experiment" : "experimentId",
            "space"      : "spaceId",
            "project"    : "projectId",
            "container"  : "containerId",
            "component"  : "componentId",
            "components" : "componentIds",
            "parents"    : "parentIds",
            "children"   : "childIds",
            "tags"       : "tagIds",
        },
        "ids2type": {
            'spaceId': { 'permId': { '@type': 'as.dto.space.id.SpacePermId' } },
            'projectId': { 'permId': { '@type': 'as.dto.project.id.ProjectPermId' } },
            'experimentId': { 'permId': { '@type': 'as.dto.experiment.id.ExperimentPermId' } },
            'tagIds': { 'code': { '@type': 'as.dto.tag.id.TagCode' } },
        },
    }
    return entities[entity]


search_criteria = {
    "space":      "as.dto.space.search.SpaceSearchCriteria",
    "project":    "as.dto.project.search.ProjectSearchCriteria",
    "experiment": "as.dto.experiment.search.ExperimentSearchCriteria",
    "sample":     "as.dto.sample.search.SampleSearchCriteria",
    "dataset":    "as.dto.dataset.search.DataSetSearchCriteria",
    "code":       "as.dto.common.search.CodeSearchCriteria",
    "sample_type":"as.dto.sample.search.SampleTypeSearchCriteria",
}

fetch_option = {
    "space":        { "@type": "as.dto.space.fetchoptions.SpaceFetchOptions" },
    "project":      { "@type": "as.dto.project.fetchoptions.ProjectFetchOptions" },
    "experiment":   { "@type": "as.dto.experiment.fetchoptions.ExperimentFetchOptions" },
    "sample":       { "@type": "as.dto.sample.fetchoptions.SampleFetchOptions" },
    "samples":       { "@type": "as.dto.sample.fetchoptions.SampleFetchOptions" },
    "dataSets":    {
        "@type": "as.dto.dataset.fetchoptions.DataSetFetchOptions",
        "properties": { "@type": "as.dto.property.fetchoptions.PropertyFetchOptions" },
        "type": { "@type": "as.dto.dataset.fetchoptions.DataSetTypeFetchOptions" },
    },
    "physicalData": { "@type": "as.dto.dataset.fetchoptions.PhysicalDataFetchOptions" },
    "linkedData":   { "@type": "as.dto.dataset.fetchoptions.LinkedDataFetchOptions" },


    "properties":   { "@type": "as.dto.property.fetchoptions.PropertyFetchOptions" },
    "propertyAssignments" : {
        "@type" : "as.dto.property.fetchoptions.PropertyAssignmentFetchOptions",
        "propertyType": {
            "@type": "as.dto.property.fetchoptions.PropertyTypeFetchOptions"
        }
    },
    "tags":         { "@type": "as.dto.tag.fetchoptions.TagFetchOptions" },

    "registrator":  { "@type": "as.dto.person.fetchoptions.PersonFetchOptions" },
    "modifier":     { "@type": "as.dto.person.fetchoptions.PersonFetchOptions" },
    "leader":       { "@type": "as.dto.person.fetchoptions.PersonFetchOptions" },

    "attachments":  { "@type": "as.dto.attachment.fetchoptions.AttachmentFetchOptions" },
    "attachmentsWithContent": {
        "@type": "as.dto.attachment.fetchoptions.AttachmentFetchOptions",
        "content": {
            "@type": "as.dto.common.fetchoptions.EmptyFetchOptions"
        },
    },
    "history": { "@type": "as.dto.history.fetchoptions.HistoryEntryFetchOptions" },
    "dataStore": { "@type": "as.dto.datastore.fetchoptions.DataStoreFetchOptions" },
}


def parse_jackson(input_json):
    """openBIS uses a library called «jackson» to automatically generate the JSON RPC output.
       Objects that are found the first time are added an attribute «@id».
       Any further findings only carry this reference id.
       This function is used to dereference the output.
    """
    interesting=['tags', 'registrator', 'modifier', 'type', 'parents', 
        'children', 'containers', 'properties', 'experiment', 'sample',
        'project', 'space', 'propertyType' 
    ]
    found = {} 
    def build_cache(graph):
        if isinstance(graph, list):
            for item in graph:
                build_cache(item)
        elif isinstance(graph, dict) and len(graph) > 0:
            for key, value in graph.items():
                if key in interesting:
                    if isinstance(value, dict):
                        if '@id' in value:
                            found[value['@id']] = value
                        build_cache(value)
                    elif isinstance(value, list):
                        for item in value:
                            if isinstance(item, dict):
                                if '@id' in item:
                                    found[item['@id']] = item
                                build_cache(item)
                elif isinstance(value, dict):
                    build_cache(value)
                elif isinstance(value, list):
                    build_cache(value)
                    
    def deref_graph(graph):            
        if isinstance(graph, list):
            for i, list_item in enumerate(graph):
                if isinstance(list_item, int):
                    graph[i] = found[list_item]
                else:
                    deref_graph(list_item)
        elif isinstance(graph, dict) and len(graph) > 0:
            for key, value in graph.items():
                if key in interesting:
                    if isinstance(value, dict):
                        deref_graph(value)
                    elif isinstance(value, int):
                        graph[key] = found[value]
                    elif isinstance(value, list):
                        for i, list_item in enumerate(value):
                            if isinstance(list_item, int):
                                if list_item in found:
                                    value[i] = found[list_item]
                                else:
                                    value[i] = list_item
                elif isinstance(value, dict):
                    deref_graph(value)
                elif isinstance(value, list):
                    deref_graph(value)

    build_cache(input_json)