pybis.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
pybis.py

Work with openBIS from Python.

"""

import os
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

import time
import json
import re
from urllib.parse import urlparse
import zlib
import base64
from collections import namedtuple
from pybis.utils import parse_jackson, check_datatype, split_identifier, format_timestamp, is_identifier, is_permid
from pybis.property import PropertyHolder, PropertyAssignments
from pybis.masterdata import Vocabulary


import pandas as pd
from pandas import DataFrame, Series

import threading
from threading import Thread
from queue import Queue
DROPBOX_PLUGIN = "jupyter-uploader-api"


def _definitions(entity):
    entities = {
        "Space": {
            "attrs_new": "code description".split(),
            "attrs_up": "description".split(),
            "attrs": "code permId description registrator registrationDate modificationDate".split(), "identifier": "spaceId",
        },
        "Project": {
            "attrs_new": "code description space attachments".split(),
            "attrs_up": "description space attachments".split(),
            "attrs": "code description permId identifier space leader registrator registrationDate modifier modificationDate attachments".split(),
            "multi": "".split(),
            "identifier": "projectId",
        },
        "Experiment": {
            "attrs_new": "code type project tags attachments".split(),
            "attrs_up": "project tags attachments".split(),
            "attrs": "code permId identifier type project tags attachments".split(),
            "multi": "tags attachments".split(),
            "identifier": "experimentId",
        },
        "Sample": {
            "attrs_new": "code type space project experiment tags attachments".split(),
            "attrs_up": "space project experiment tags attachments".split(),
            "attrs": "code permId identifier type space project experiment tags attachments".split(),
            "ids2type": {
                'parentIds': { 'permId': { '@type': 'as.dto.sample.id.SamplePermId' } },
                'childIds':  { 'permId': { '@type': 'as.dto.sample.id.SamplePermId' } },
                'componentIds': { 'permId': {'@type': 'as.dto.sample.id.SamplePermId' } },
            },
            "identifier": "sampleId",
            "cre_type": "as.dto.sample.create.SampleCreation",
            "multi": "parents children components tags".split(),
        },
        "DataSet": {
            "attrs_new": "type experiment sample parents children container components tags".split(),
            "attrs_up": "experiment sample parents children container components tags".split(),
            "attrs": "code permId type experiment sample parents children container components tags".split(),
            
            "ids2type": {
                'parentIds':     { 'permId': { '@type': 'as.dto.dataset.id.DataSetPermId' } },
                'childIds':      { 'permId': { '@type': 'as.dto.dataset.id.DataSetPermId' } }, 
                'componentIds':  { 'permId': { '@type': 'as.dto.dataset.id.DataSetPermId' } }, 
                'containerIds':  { 'permId': { '@type': 'as.dto.dataset.id.DataSetPermId' } }, 
            },
            "multi": "".split(),
            "identifier": "dataSetId",
        },
        "Material": {
            "attrs_new": "code description type creation tags".split(),
            "attrs" : "code description type creation registrator tags".split()
        },
        "Tag": {
            "attrs_new": "code description experiments samples dataSets materials".split(),
            "attrs": "code description experiments samples dataSets materials registrationDate".split(),
        },
        "attr2ids": {
            "space"      : "spaceId",
            "project"    : "projectId",
            "sample"     : "sampleId",
            "samples"    : "sampleIds",
            "dataSet"    : "dataSetId",
            "dataSets"   : "dataSetIds",
            "experiment" : "experimentId",
            "experiments": "experimentIds",
            "material"   : "materialId",
            "materials"  : "materialIds",
            "container"  : "containerId",
            "component"  : "componentId",
            "components" : "componentIds",
            "parents"    : "parentIds",
            "children"   : "childIds",
            "tags"       : "tagIds",
        },
        "ids2type": {
            'spaceId': { 'permId': { '@type': 'as.dto.space.id.SpacePermId' } },
            'projectId': { 'permId': { '@type': 'as.dto.project.id.ProjectPermId' } },
            'experimentId': { 'permId': { '@type': 'as.dto.experiment.id.ExperimentPermId' } },
            'tagIds': { 'code': { '@type': 'as.dto.tag.id.TagCode' } },
        },
    }
    return entities[entity]


search_criteria = {
    "space":      "as.dto.space.search.SpaceSearchCriteria",
    "project":    "as.dto.project.search.ProjectSearchCriteria",
    "experiment": "as.dto.experiment.search.ExperimentSearchCriteria",
    "sample":     "as.dto.sample.search.SampleSearchCriteria",
    "dataset":    "as.dto.dataset.search.DataSetSearchCriteria",
    "code":       "as.dto.common.search.CodeSearchCriteria",
    "sample_type":"as.dto.sample.search.SampleTypeSearchCriteria",
}

fetch_option = {
    "space":        { "@type": "as.dto.space.fetchoptions.SpaceFetchOptions" },
    "project":      { "@type": "as.dto.project.fetchoptions.ProjectFetchOptions" },
    "experiment":   { "@type": "as.dto.experiment.fetchoptions.ExperimentFetchOptions" },
    "sample":       { "@type": "as.dto.sample.fetchoptions.SampleFetchOptions" },
    "samples":       { "@type": "as.dto.sample.fetchoptions.SampleFetchOptions" },
    "dataSets":    {
        "@type": "as.dto.dataset.fetchoptions.DataSetFetchOptions",
        "properties": { "@type": "as.dto.property.fetchoptions.PropertyFetchOptions" },
        "type": { "@type": "as.dto.dataset.fetchoptions.DataSetTypeFetchOptions" },
    },
    "physicalData": { "@type": "as.dto.dataset.fetchoptions.PhysicalDataFetchOptions" },
    "linkedData":   { "@type": "as.dto.dataset.fetchoptions.LinkedDataFetchOptions" },


    "properties":   { "@type": "as.dto.property.fetchoptions.PropertyFetchOptions" },
    "propertyAssignments" : {
        "@type" : "as.dto.property.fetchoptions.PropertyAssignmentFetchOptions",
        "propertyType": {
            "@type": "as.dto.property.fetchoptions.PropertyTypeFetchOptions"
        }
    },
    "tags":         { "@type": "as.dto.tag.fetchoptions.TagFetchOptions" },

    "registrator":  { "@type": "as.dto.person.fetchoptions.PersonFetchOptions" },
    "modifier":     { "@type": "as.dto.person.fetchoptions.PersonFetchOptions" },
    "leader":       { "@type": "as.dto.person.fetchoptions.PersonFetchOptions" },

    "attachments":  { "@type": "as.dto.attachment.fetchoptions.AttachmentFetchOptions" },
    "attachmentsWithContent": {
        "@type": "as.dto.attachment.fetchoptions.AttachmentFetchOptions",
        "content": {
            "@type": "as.dto.common.fetchoptions.EmptyFetchOptions"
        },
    },
    "history": { "@type": "as.dto.history.fetchoptions.HistoryEntryFetchOptions" },
    "dataStore": { "@type": "as.dto.datastore.fetchoptions.DataStoreFetchOptions" },
}

def search_request_for_identifier(ident, entity):
    search_request = {}

    if is_identifier(ident):
        search_request = {
            "identifier": ident.upper(),
            "@type": "as.dto.{}.id.{}Identifier".format(entity.lower(), entity.capitalize())
        }
    else:
        search_request = {
            "permId": ident,
            "@type": "as.dto.{}.id.{}PermId".format(entity.lower(), entity.capitalize())
        }
    return search_request

def extract_code(obj):
    if not isinstance(obj, dict):
        return str(obj)
    return obj['code']

def extract_deletion(obj):
    del_objs = []
    for deleted_object in obj['deletedObjects']:
        del_objs.append({
            "reason": obj['reason'],
            "permId": deleted_object["id"]["permId"],
            "type": deleted_object["id"]["@type"]
        })
    return del_objs

def extract_identifier(ident):