diff --git a/jupyter-openbis-extension/connection.py b/jupyter-openbis-extension/connection.py new file mode 100644 index 0000000000000000000000000000000000000000..4999b5c04a28fd9a5d3c2ea818f997d1e54d7b6e --- /dev/null +++ b/jupyter-openbis-extension/connection.py @@ -0,0 +1,168 @@ +import os +from pybis import Openbis +from notebook.base.handlers import IPythonHandler + +openbis_connections = {} + +def register_connection(connection_info): + + conn = OpenBISConnection( + name = connection_info.get('name'), + url = connection_info.get('url'), + verify_certificates = connection_info.get('verify_certificates', False), + username = connection_info.get('username'), + password = connection_info.get('password'), + status = 'not connected', + ) + openbis_connections[conn.name] = conn + return conn + + + +class OpenBISConnection: + """register an openBIS connection + """ + + def __init__(self, **kwargs): + for needed_key in ['name', 'url']: + if needed_key not in kwargs: + raise KeyError("{} is missing".format(needed_key)) + + for key in kwargs: + setattr(self, key, kwargs[key]) + + openbis = Openbis( + url = self.url, + verify_certificates = self.verify_certificates + ) + self.openbis = openbis + self.status = "not connected" + + def is_session_active(self): + return self.openbis.is_session_active() + + def check_status(self): + if self.openbis.is_session_active(): + self.status = "connected" + else: + self.status = "not connected" + + def login(self, username=None, password=None): + if username is None: + username=self.username + if password is None: + password=self.password + self.openbis.login( + username = username, + password = password + ) + # store username and password in memory + self.username = username + self.password = password + self.status = 'connected' + + def get_info(self): + return { + 'name' : self.name, + 'url' : self.url, + 'status' : self.status, + 'username': self.username, + 'password': self.password, + } + +class OpenBISConnections(IPythonHandler): + + def post(self): + """create a new connection + + :return: a new connection object + """ + data = self.get_json_body() + conn = register_connection(data) + if conn.username and conn.password: + try: + conn.login() + except Exception: + pass + self.get() + return + + def get(self): + """returns all available openBIS connections + """ + + connections= [] + for conn in openbis_connections.values(): + conn.check_status() + connections.append(conn.get_info()) + + self.write({ + 'status' : 200, + 'connections': connections, + 'cwd' : os.getcwd() + }) + return + + +class OpenBISConnectionHandler(IPythonHandler): + """Handle the requests to /openbis/conn + """ + + def put(self, connection_name): + """reconnect to a current connection + :return: an updated connection object + """ + data = self.get_json_body() + + try: + conn = openbis_connections[connection_name] + except KeyError: + self.set_status(404) + self.write({ + "reason" : 'No such connection: {}'.format(data) + }) + return + + try: + conn.login(data.get('username'), data.get('password')) + except ConnectionError: + self.set_status(500) + self.write({ + "reason": "Could not establish connection to {}".format(connection_name) + }) + return + except ValueError: + self.set_status(401) + self.write({ + "reason": "Incorrect username or password for {}".format(connection_name) + }) + return + + self.write({ + 'status' : 200, + 'connection': conn.get_info(), + 'cwd' : os.getcwd() + }) + + def get(self, connection_name): + """returns information about a connection name + """ + + try: + conn = openbis_connections[connection_name] + except KeyError: + self.set_status(404) + self.write({ + "reason" : 'No such connection: {}'.format(connection_name) + }) + return + + conn.check_status() + + self.write({ + 'status' : 200, + 'connection': conn.get_info(), + 'cwd' : os.getcwd() + }) + return + diff --git a/jupyter-openbis-extension/dataset.py b/jupyter-openbis-extension/dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..d105181ad9c3ece30e790ccebb87b50011d94995 --- /dev/null +++ b/jupyter-openbis-extension/dataset.py @@ -0,0 +1,280 @@ +import os +from urllib.parse import unquote +from notebook.base.handlers import IPythonHandler +from .connection import openbis_connections + +class DataSetDownloadHandler(IPythonHandler): + """Handle the requests for /openbis/dataset/connection/permId""" + + + def download_data(self, conn, permId, downloadPath=None): + if not conn.is_session_active(): + try: + conn.login() + except Exception as exc: + self.set_status(500) + self.write({ + "reason" : 'connection to {} could not be established: {}'.format(conn.name, exc) + }) + return + + try: + dataset = conn.openbis.get_dataset(permId) + except Exception as exc: + self.set_status(404) + self.write({ + "reason" : 'No such dataSet found: {}'.format(permId) + }) + return + + # dataset was found, download the data to the disk + try: + destination = dataset.download(destination=downloadPath) + except Exception as exc: + self.set_status(500) + self.write({ + "reason": 'Data for DataSet {} could not be downloaded: {}'.format(permId, exc) + }) + return + + # return success message + path = os.path.join(downloadPath, dataset.permId) + self.write({ + 'url' : conn.url, + 'permId' : dataset.permId, + 'path' : path, + 'dataStore' : dataset.dataStore, + 'location' : dataset.physicalData.location, + 'size' : dataset.physicalData.size, + 'statusText': 'Data for DataSet {} was successfully downloaded to: {}.'.format(dataset.permId, path) + }) + + def get(self, **params): + """Handle a request to /openbis/dataset/connection_name/permId + download the data and return a message + """ + + try: + conn = openbis_connections[params['connection_name']] + except KeyError: + self.set_status(404) + self.write({ + "reason":'connection {} was not found'.format(params['connection_name']) + }) + return + + results = self.download_data(conn=conn, permId=params['permId'], downloadPath=params['downloadPath']) + + +class DataSetTypesHandler(IPythonHandler): + def get(self, **params): + """Handle a request to /openbis/datasetTypes/connection_name + """ + + try: + conn = openbis_connections[params['connection_name']] + except KeyError: + self.set_status(404) + self.write({ + "reason":'connection {} was not found'.format(params['connection_name']) + }) + return + + try: + dataset_types = conn.openbis.get_dataset_types() + dts = dataset_types.df.to_dict(orient='records') + + # get property assignments for every dataset-type + # and add it to the dataset collection + for dt in dts: + dataset_type = conn.openbis.get_dataset_type(dt['code']) + pa = dataset_type.get_propertyAssignments(including_vocabulary=True) + pa_dicts = pa.to_dict(orient='records') + for pa_dict in pa_dicts: + if pa_dict['dataType'] == 'CONTROLLEDVOCABULARY': + terms = conn.openbis.get_terms(pa_dict['vocabulary']['code']) + pa_dict['terms'] = terms.df[['code','label','description','official','ordinal']].to_dict(orient='records') + + dt['propertyAssignments'] = pa_dicts + + self.write({ + "dataSetTypes": dts + }) + return + + except Exception as e: + print(e) + self.set_status(500) + self.write({ + "reason":'Could not fetch dataset-types: {}'.format(e) + }) + return + + +class DataSetUploadHandler(IPythonHandler): + """Handle the POST requests for /openbis/dataset/connection_name""" + + def upload_data(self, conn, data): + if not conn.is_session_active(): + try: + conn.login() + except Exception as e: + print(e) + self.set_status(500) + self.write({ + "reason": 'connection to {} could not be established: {}'.format(conn.name, e) + }) + return + + errors = [] + + sample = None + experiment = None + + if (data.get('sampleIdentifier')): + try: + sample = conn.openbis.get_sample(data.get('sampleIdentifier')) + except Exception as e: + print(e) + errors.append( + {"sampleIdentifier" : 'No such sample: {}'.format(data.get('sampleIdentifier')) } + ) + else: + errors.append( + {"sampleIdentifier": "please provide a sample identifier"} + ) + print("--------------_HERE_-----------1") + + if (data.get('experimentIdentifier')): + try: + experiment = conn.openbis.get_experiment(data.get('experimentIdentifier')) + except Exception as e: + print(e) + errors.append( + {"experimentIdentifier" : 'No such experiment: {}'.format(data.get('experimentIdentifier')) } + ) + print("--------------_HERE_-----------2") + + + filenames = [] + for filename in data.get('files'): + filename = unquote(filename) + if os.path.isfile(filename): + filenames.append(filename) + else: + errors.append({ + "file": "File not found: {}".format(filename) + }) + print("--------------_HERE_-----------3") + + try: + dataset = conn.openbis.new_dataset( + type = data.get('type'), + sample = sample, + experiment = experiment, + files = filenames, + ) + except Exception as e: + print(e) + errors.append({ + "create": 'Error while creating the dataset: {}'.format(e) + }) + print("--------------_HERE_-----------4") + + # try to set the properties + if data.get('props'): + props = data.get('props') + for prop, value in props.items(): + try: + setattr(dataset.props, prop.lower(), value) + except Exception as e: + errors.append({ + "prop."+prop : str(e) + }) + + # check if any mandatory property is missing + for prop_name, prop in dataset.props._property_names.items(): + if prop['mandatory']: + if getattr(dataset.props, prop_name) is None or getattr(dataset.props, prop_name) == "": + errors.append({ + "prop."+prop_name : "is mandatory" + }) + print("--------------_HERE_-----------5") + + # write errors back if already occured + if errors: + self.set_status(500) + self.write({ "errors": errors }) + return + print("--------------_HERE_-----------6") + + try: + dataset.save() + except Exception as e: + errors.append({ + "save": 'Error while saving the dataset: {}'.format(e) + }) + print("--------------_HERE_-----------7") + + # write errors back if they occured + if errors: + self.set_status(500) + self.write({ "errors": errors }) + else: + # ...or return a success message + self.write({ + 'status': 200, + 'statusText': 'Jupyter Notebook was successfully uploaded to: {} with permId: {}'.format(conn.name, dataset.permId) + }) + print("--------------_HERE_-----------8") + + def post(self, **params): + """Handle a request to /openbis/dataset/connection_name/permId + download the data and return a message + """ + + try: + conn = openbis_connections[params['connection_name']] + except KeyError: + self.write({ + "reason": 'connection {} was not found'.format(params['connection_name']) + }) + return + + data = self.get_json_body() + print("Received DATA") + print(data) + self.upload_data(conn=conn,data=data) + + +class FileListHandler(IPythonHandler): + + def get(self, **params): + """ + Returns the file list of the current working directory + + :param params: + :return: dictionary of files, key is the fully qualified name, + value is the relative name (for display) + """ + + cwd = os.getcwd() + files = {} + for (dirpath, dirnames, filenames) in os.walk(cwd): + if filenames: + for filename in filenames: + # ignore hidden files + if filename.startswith('.'): + continue + # ignore hidden folders + if os.path.relpath(dirpath) != '.' \ + and os.path.relpath(dirpath).startswith('.'): + continue + fqn = os.path.join(dirpath, filename) + files[fqn] = os.path.relpath(fqn, cwd) + + self.set_status(200) + self.write({ + "files": files + }) + diff --git a/jupyter-openbis-extension/sample.py b/jupyter-openbis-extension/sample.py new file mode 100644 index 0000000000000000000000000000000000000000..e892c8f909f1eae715cfa7178f8c00bb86769e6d --- /dev/null +++ b/jupyter-openbis-extension/sample.py @@ -0,0 +1,54 @@ +from notebook.base.handlers import IPythonHandler +import numpy as np +from .connection import openbis_connections + +class SampleHandler(IPythonHandler): + """Handle the requests for /openbis/sample/connection/permId""" + + def get_datasets(self, conn, permId): + if not conn.is_session_active(): + try: + conn.login() + except Exception as exc: + self.write({ + "reason" : 'connection to {} could not be established: {}'.format(conn.name, exc) + }) + + sample = None + try: + sample = conn.openbis.get_sample(permId) + except Exception as exc: + self.set_status(404) + self.write({ + "reason" : 'No such sample: {}'.format(permId) + }) + if sample is None: + return + + datasets = sample.get_datasets().df + datasets.replace({np.nan:None}, inplace=True) # replace NaN with None, otherwise we cannot convert it correctly + return datasets.to_dict(orient='records') # is too stupid to handle NaN + + + def get(self, **params): + """Handle a request to /openbis/sample/connection_name/permId + download the data and return a message + """ + try: + conn = openbis_connections[params['connection_name']] + except KeyError: + self.write({ + "reason" : 'connection {} was not found'.format( + params['connection_name'] + ) + }) + return + + datasets = self.get_datasets(conn, params['permId']) + if datasets is not None: + self.set_status(200) + self.write({ + "dataSets": datasets + }) + + diff --git a/jupyter-openbis-extension/server.py b/jupyter-openbis-extension/server.py index f44078da738bfa485e4db7f51921a51775f821b3..df56bbccab95445f01c1d6440d663e23d49fe93b 100644 --- a/jupyter-openbis-extension/server.py +++ b/jupyter-openbis-extension/server.py @@ -1,12 +1,10 @@ from notebook.utils import url_path_join -from notebook.base.handlers import IPythonHandler -from pybis import Openbis -import numpy as np import os -from urllib.parse import unquote import yaml -openbis_connections = {} +from .connection import OpenBISConnections, OpenBISConnectionHandler, register_connection +from .dataset import DataSetTypesHandler, DataSetDownloadHandler, DataSetUploadHandler, FileListHandler +from .sample import SampleHandler def _jupyter_server_extension_paths(): @@ -140,483 +138,4 @@ def load_jupyter_server_extension(nb_server_app): )] ) - print("pybis loaded: {}".format(Openbis)) - -def register_connection(connection_info): - - conn = OpenBISConnection( - name = connection_info.get('name'), - url = connection_info.get('url'), - verify_certificates = connection_info.get('verify_certificates', False), - username = connection_info.get('username'), - password = connection_info.get('password'), - status = 'not connected', - ) - openbis_connections[conn.name] = conn - return conn - - -class OpenBISConnection: - """register an openBIS connection - """ - - def __init__(self, **kwargs): - for needed_key in ['name', 'url']: - if needed_key not in kwargs: - raise KeyError("{} is missing".format(needed_key)) - - for key in kwargs: - setattr(self, key, kwargs[key]) - - openbis = Openbis( - url = self.url, - verify_certificates = self.verify_certificates - ) - self.openbis = openbis - self.status = "not connected" - - def is_session_active(self): - return self.openbis.is_session_active() - - def check_status(self): - if self.openbis.is_session_active(): - self.status = "connected" - else: - self.status = "not connected" - - def login(self, username=None, password=None): - if username is None: - username=self.username - if password is None: - password=self.password - self.openbis.login( - username = username, - password = password - ) - # store username and password in memory - self.username = username - self.password = password - self.status = 'connected' - - def get_info(self): - return { - 'name' : self.name, - 'url' : self.url, - 'status' : self.status, - 'username': self.username, - 'password': self.password, - } - -class OpenBISConnections(IPythonHandler): - - def post(self): - """create a new connection - - :return: a new connection object - """ - data = self.get_json_body() - conn = register_connection(data) - if conn.username and conn.password: - try: - conn.login() - except Exception: - pass - self.get() - return - - def get(self): - """returns all available openBIS connections - """ - - connections= [] - for conn in openbis_connections.values(): - conn.check_status() - connections.append(conn.get_info()) - - self.write({ - 'status' : 200, - 'connections': connections, - 'cwd' : os.getcwd() - }) - return - - -class OpenBISConnectionHandler(IPythonHandler): - """Handle the requests to /openbis/conn - """ - - def put(self, connection_name): - """reconnect to a current connection - :return: an updated connection object - """ - data = self.get_json_body() - - try: - conn = openbis_connections[connection_name] - except KeyError: - self.set_status(404) - self.write({ - "reason" : 'No such connection: {}'.format(data) - }) - return - - try: - conn.login(data.get('username'), data.get('password')) - except ConnectionError: - self.set_status(500) - self.write({ - "reason": "Could not establish connection to {}".format(connection_name) - }) - return - except ValueError: - self.set_status(401) - self.write({ - "reason": "Incorrect username or password for {}".format(connection_name) - }) - return - - self.write({ - 'status' : 200, - 'connection': conn.get_info(), - 'cwd' : os.getcwd() - }) - - def get(self, connection_name): - """returns information about a connection name - """ - - try: - conn = openbis_connections[connection_name] - except KeyError: - self.set_status(404) - self.write({ - "reason" : 'No such connection: {}'.format(data) - }) - return - - conn.check_status() - - self.write({ - 'status' : 200, - 'connection': conn.get_info(), - 'cwd' : os.getcwd() - }) - return - - -class SampleHandler(IPythonHandler): - """Handle the requests for /openbis/sample/connection/permId""" - - def get_datasets(self, conn, permId): - if not conn.is_session_active(): - try: - conn.login() - except Exception as exc: - self.write({ - "reason" : 'connection to {} could not be established: {}'.format(conn.name, exc) - }) - - sample = None - try: - sample = conn.openbis.get_sample(permId) - except Exception as exc: - self.set_status(404) - self.write({ - "reason" : 'No such sample: {}'.format(permId) - }) - if sample is None: - return - - datasets = sample.get_datasets().df - datasets.replace({np.nan:None}, inplace=True) # replace NaN with None, otherwise we cannot convert it correctly - return datasets.to_dict(orient='records') # is too stupid to handle NaN - - - def get(self, **params): - """Handle a request to /openbis/sample/connection_name/permId - download the data and return a message - """ - try: - conn = openbis_connections[params['connection_name']] - except KeyError: - self.write({ - "reason" : 'connection {} was not found'.format( - params['connection_name'] - ) - }) - return - - datasets = self.get_datasets(conn, params['permId']) - if datasets is not None: - self.set_status(200) - self.write({ - "dataSets": datasets - }) - - -class FileListHandler(IPythonHandler): - - def get(self, **params): - """ - Returns the file list of the current working directory - - :param params: - :return: dictionary of files, key is the fully qualified name, - value is the relative name (for display) - """ - - cwd = os.getcwd() - files = {} - for (dirpath, dirnames, filenames) in os.walk(cwd): - if filenames: - for filename in filenames: - # ignore hidden files - if filename.startswith('.'): - continue - # ignore hidden folders - if os.path.relpath(dirpath) != '.' \ - and os.path.relpath(dirpath).startswith('.'): - continue - fqn = os.path.join(dirpath, filename) - files[fqn] = os.path.relpath(fqn, cwd) - - self.set_status(200) - self.write({ - "files": files - }) - - -class DataSetDownloadHandler(IPythonHandler): - """Handle the requests for /openbis/dataset/connection/permId""" - - - def download_data(self, conn, permId, downloadPath=None): - if not conn.is_session_active(): - try: - conn.login() - except Exception as exc: - self.set_status(500) - self.write({ - "reason" : 'connection to {} could not be established: {}'.format(conn.name, exc) - }) - return - - try: - dataset = conn.openbis.get_dataset(permId) - except Exception as exc: - self.set_status(404) - self.write({ - "reason" : 'No such dataSet found: {}'.format(permId) - }) - return - - # dataset was found, download the data to the disk - try: - destination = dataset.download(destination=downloadPath) - except Exception as exc: - self.set_status(500) - self.write({ - "reason": 'Data for DataSet {} could not be downloaded: {}'.format(permId, exc) - }) - return - - # return success message - path = os.path.join(downloadPath, dataset.permId) - self.write({ - 'url' : conn.url, - 'permId' : dataset.permId, - 'path' : path, - 'dataStore' : dataset.dataStore, - 'location' : dataset.physicalData.location, - 'size' : dataset.physicalData.size, - 'statusText': 'Data for DataSet {} was successfully downloaded to: {}.'.format(dataset.permId, path) - }) - - def get(self, **params): - """Handle a request to /openbis/dataset/connection_name/permId - download the data and return a message - """ - - try: - conn = openbis_connections[params['connection_name']] - except KeyError: - self.set_status(404) - self.write({ - "reason":'connection {} was not found'.format(params['connection_name']) - }) - return - - results = self.download_data(conn=conn, permId=params['permId'], downloadPath=params['downloadPath']) - - -class DataSetTypesHandler(IPythonHandler): - def get(self, **params): - """Handle a request to /openbis/datasetTypes/connection_name - """ - - try: - conn = openbis_connections[params['connection_name']] - except KeyError: - self.set_status(404) - self.write({ - "reason":'connection {} was not found'.format(params['connection_name']) - }) - return - - try: - dataset_types = conn.openbis.get_dataset_types() - dts = dataset_types.df.to_dict(orient='records') - - # get property assignments for every dataset-type - # and add it to the dataset collection - for dt in dts: - dataset_type = conn.openbis.get_dataset_type(dt['code']) - pa = dataset_type.get_propertyAssignments(including_vocabulary=True) - pa_dicts = pa.to_dict(orient='records') - for pa_dict in pa_dicts: - if pa_dict['dataType'] == 'CONTROLLEDVOCABULARY': - terms = conn.openbis.get_terms(pa_dict['vocabulary']['code']) - pa_dict['terms'] = terms.df[['code','label','description','official','ordinal']].to_dict(orient='records') - - dt['propertyAssignments'] = pa_dicts - - self.write({ - "dataSetTypes": dts - }) - return - - except Exception as e: - print(e) - self.set_status(500) - self.write({ - "reason":'Could not fetch dataset-types: {}'.format(e) - }) - return - - -class DataSetUploadHandler(IPythonHandler): - """Handle the POST requests for /openbis/dataset/connection_name""" - - def upload_data(self, conn, data): - if not conn.is_session_active(): - try: - conn.login() - except Exception as e: - print(e) - self.set_status(500) - self.write({ - "reason": 'connection to {} could not be established: {}'.format(conn.name, e) - }) - return - - errors = [] - - sample = None - experiment = None - - if (data.get('sampleIdentifier')): - try: - sample = conn.openbis.get_sample(data.get('sampleIdentifier')) - except Exception as e: - print(e) - errors.append( - {"sampleIdentifier" : 'No such sample: {}'.format(data.get('sampleIdentifier')) } - ) - else: - errors.append( - {"sampleIdentifier": "please provide a sample identifier"} - ) - print("--------------_HERE_-----------1") - - if (data.get('experimentIdentifier')): - try: - experiment = conn.openbis.get_experiment(data.get('experimentIdentifier')) - except Exception as e: - print(e) - errors.append( - {"experimentIdentifier" : 'No such experiment: {}'.format(data.get('experimentIdentifier')) } - ) - print("--------------_HERE_-----------2") - - filenames = [] - for filename in data.get('files'): - filename = unquote(filename) - if os.path.isfile(filename): - filenames.append(filename) - else: - errors.append({ - "file": "File not found: {}".format(filename) - }) - print("--------------_HERE_-----------3") - - try: - dataset = conn.openbis.new_dataset( - type = data.get('type'), - sample = sample, - experiment = experiment, - files = filenames, - ) - except Exception as e: - print(e) - errors.append({ - "create": 'Error while creating the dataset: {}'.format(e) - }) - print("--------------_HERE_-----------4") - - # try to set the properties - if (data.get('props')): - props = data.get('props') - for prop, value in props.items(): - try: - setattr(dataset.props, prop.lower(), value) - except Exception as e: - errors.append({ - "prop."+prop : str(e) - }) - print("--------------_HERE_-----------5") - - # write errors back if already occured - if errors: - self.set_status(500) - self.write({ "errors": errors }) - return - print("--------------_HERE_-----------6") - - try: - dataset.save() - except Exception as e: - errors.append({ - "save": 'Error while saving the dataset: {}'.format(e) - }) - print("--------------_HERE_-----------7") - - # write errors back if they occured - if errors: - self.set_status(500) - self.write({ "errors": errors }) - else: - # ...or return a success message - self.write({ - 'status': 200, - 'statusText': 'Jupyter Notebook was successfully uploaded to: {} with permId: {}'.format(conn.name, dataset.permId) - }) - print("--------------_HERE_-----------8") - - def post(self, **params): - """Handle a request to /openbis/dataset/connection_name/permId - download the data and return a message - """ - - try: - conn = openbis_connections[params['connection_name']] - except KeyError: - self.write({ - "reason": 'connection {} was not found'.format(params['connection_name']) - }) - return - - data = self.get_json_body() - print("Received DATA") - print(data) - self.upload_data(conn=conn,data=data) diff --git a/setup.py b/setup.py index affc1047c6a441146e3b686a8681a044b6d3f6bc..63ce3a6c30860bc456a6a5166f9c1d40d18ab419 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,3 @@ -import os import sys if sys.version_info < (3,3): @@ -25,6 +24,7 @@ setup( 'jupyter-nbextensions-configurator', 'jupyter', 'pybis>=1.8.1', + 'numpy', ], python_requires=">=3.3", classifiers=[