from notebook.utils import url_path_join
from notebook.base.handlers import IPythonHandler
from pybis import Openbis
import numpy as np
import os
from urllib.parse import unquote
import yaml
openbis_connections = {}
def _jupyter_server_extension_paths():
return [{'module': 'jupyter-openbis-extension.server'}]
def _load_configuration(paths, filename='openbis-connections.yaml'):
if paths is None:
paths = []
home = os.path.expanduser("~")
paths.append(os.path.join(home, '.jupyter'))
# look in all config file paths of jupyter
# for openbis connection files and load them
connections = []
for path in paths:
abs_filename = os.path.join(path, filename)
if os.path.isfile(abs_filename):
with open(abs_filename, 'r') as stream:
try:
config = yaml.safe_load(stream)
for connection in config['connections']:
connections.append(connection)
except yaml.YAMLexception as e:
print(e)
return None
return connections
def load_jupyter_server_extension(nb_server_app):
"""Call when the extension is loaded.
:param nb_server_app: Handle to the Notebook webserver instance.
"""
# load the configuration file
# and register the openBIS connections.
# If username and password is available, try to connect to the server
connections = _load_configuration(
paths = nb_server_app.config_file_paths,
filename = 'openbis-connections.yaml'
)
for connection_info in connections:
conn = register_connection(connection_info)
print("Registered: {}".format(conn.url))
if conn.username and conn.password:
try:
conn.login()
print("Successfully connected to: {}".format(conn.url))
except ValueError:
print("Incorrect username or password for: {}".format(conn.url))
except Exception:
print("Cannot establish connection to: {}".format(conn.url))
# Add URL handlers to our web_app
# see Tornado documentation: https://www.tornadoweb.org
web_app = nb_server_app.web_app
host_pattern = '.*$'
base_url = web_app.settings['base_url']
# get the file list
web_app.add_handlers(
host_pattern,
[(url_path_join( base_url, '/general/filelist'),
FileListHandler
)]
)
# DataSet download
web_app.add_handlers(
host_pattern,
[(url_path_join(
base_url,
'/openbis/dataset/(?P<connection_name>.*)?/(?P<permId>.*)?/(?P<downloadPath>.*)'),
DataSetDownloadHandler
)]
)
# DataSet upload
web_app.add_handlers( host_pattern, [(
url_path_join( base_url, '/openbis/dataset/(?P<connection_name>.*)' ),
DataSetUploadHandler
)]
)
# DataSet-Types
web_app.add_handlers(
host_pattern,
[(
url_path_join( base_url, '/openbis/datasetTypes/(?P<connection_name>.*)'
),
DataSetTypesHandler
)]
)
# DataSets for Sample identifier/permId
web_app.add_handlers(
host_pattern,
[(
url_path_join(
base_url,
'/openbis/sample/(?P<connection_name>.*)?/(?P<permId>.*)'
),
SampleHandler
)]
)
# OpenBIS connections
web_app.add_handlers(
host_pattern,
[(
url_path_join(
base_url,
'/openbis/conns'
),
OpenBISConnections
)]
)
# Modify / reconnect to a connection
web_app.add_handlers(
host_pattern,
[(
url_path_join(
base_url,
'/openbis/conn/(?P<connection_name>.*)'
),
OpenBISConnectionHandler
)]
)
print("pybis loaded: {}".format(Openbis))
def register_connection(connection_info):
conn = OpenBISConnection(
name = connection_info.get('name'),
url = connection_info.get('url'),
verify_certificates = connection_info.get('verify_certificates', False),
username = connection_info.get('username'),
password = connection_info.get('password'),
status = 'not connected',
)
openbis_connections[conn.name] = conn
return conn
class OpenBISConnection:
"""register an openBIS connection
"""
def __init__(self, **kwargs):
for needed_key in ['name', 'url']:
if needed_key not in kwargs:
raise KeyError("{} is missing".format(needed_key))
for key in kwargs:
setattr(self, key, kwargs[key])
openbis = Openbis(
url = self.url,
verify_certificates = self.verify_certificates
)
self.openbis = openbis
self.status = "not connected"
def is_session_active(self):
return self.openbis.is_session_active()
def check_status(self):
if self.openbis.is_session_active():
self.status = "connected"
else:
self.status = "not connected"
def login(self, username=None, password=None):
if username is None:
username=self.username
if password is None:
password=self.password
self.openbis.login(
username = username,
password = password
)
# store username and password in memory
self.username = username
self.password = password
self.status = 'connected'
def get_info(self):
return {
'name' : self.name,
'url' : self.url,
'status' : self.status,
'username': self.username,
'password': self.password,
}
class OpenBISConnections(IPythonHandler):
def post(self):
"""create a new connection
:return: a new connection object
"""
data = self.get_json_body()
conn = register_connection(data)
if conn.username and conn.password:
try:
conn.login()
except Exception:
pass
self.get()
return
def get(self):
"""returns all available openBIS connections
"""
connections= []
for conn in openbis_connections.values():
conn.check_status()
connections.append(conn.get_info())
self.write({
'status' : 200,
'connections': connections,
'cwd' : os.getcwd()
})
return
class OpenBISConnectionHandler(IPythonHandler):
"""Handle the requests to /openbis/conn
"""
def put(self, connection_name):
"""reconnect to a current connection
:return: an updated connection object
"""
data = self.get_json_body()
try:
conn = openbis_connections[connection_name]
except KeyError:
self.set_status(404)
self.write({
"reason" : 'No such connection: {}'.format(data)
})
return
try:
conn.login(data.get('username'), data.get('password'))
except ConnectionError:
self.set_status(500)
self.write({
"reason": "Could not establish connection to {}".format(connection_name)
})
return
except ValueError:
self.set_status(401)
self.write({
"reason": "Incorrect username or password for {}".format(connection_name)
})
return
self.write({
'status' : 200,
'connection': conn.get_info(),
'cwd' : os.getcwd()
})
def get(self, connection_name):
"""returns information about a connection name
"""
try:
conn = openbis_connections[connection_name]
except KeyError:
self.set_status(404)
self.write({
"reason" : 'No such connection: {}'.format(data)
})
return
conn.check_status()
self.write({
'status' : 200,
'connection': conn.get_info(),
'cwd' : os.getcwd()
})
return
class SampleHandler(IPythonHandler):
"""Handle the requests for /openbis/sample/connection/permId"""
def get_datasets(self, conn, permId):
if not conn.is_session_active():
try:
conn.login()
except Exception as exc:
self.write({
"reason" : 'connection to {} could not be established: {}'.format(conn.name, exc)
})
sample = None
try:
sample = conn.openbis.get_sample(permId)
except Exception as exc:
self.set_status(404)
self.write({
"reason" : 'No such sample: {}'.format(permId)
})
if sample is None:
return
datasets = sample.get_datasets().df
datasets.replace({np.nan:None}, inplace=True) # replace NaN with None, otherwise we cannot convert it correctly
return datasets.to_dict(orient='records') # is too stupid to handle NaN
def get(self, **params):
"""Handle a request to /openbis/sample/connection_name/permId
download the data and return a message
"""
try:
conn = openbis_connections[params['connection_name']]
except KeyError:
self.write({
"reason" : 'connection {} was not found'.format(
params['connection_name']
)
})
return
datasets = self.get_datasets(conn, params['permId'])
if datasets is not None:
self.set_status(200)
self.write({
"dataSets": datasets
})
class FileListHandler(IPythonHandler):
def get(self, **params):
"""
Returns the file list of the current working directory
:param params:
:return: dictionary of files, key is the fully qualified name,
value is the relative name (for display)
"""
cwd = os.getcwd()
files = {}
for (dirpath, dirnames, filenames) in os.walk(cwd):
if filenames:
for filename in filenames:
# ignore hidden files
if filename.startswith('.'):
continue
# ignore hidden folders
if os.path.relpath(dirpath) != '.' \
and os.path.relpath(dirpath).startswith('.'):
continue
fqn = os.path.join(dirpath, filename)
files[fqn] = os.path.relpath(fqn, cwd)
self.set_status(200)
self.write({
"files": files
})
class DataSetDownloadHandler(IPythonHandler):
"""Handle the requests for /openbis/dataset/connection/permId"""
def download_data(self, conn, permId, downloadPath=None):
if not conn.is_session_active():
try:
conn.login()
except Exception as exc:
self.set_status(500)
self.write({
"reason" : 'connection to {} could not be established: {}'.format(conn.name, exc)
})
return
try:
dataset = conn.openbis.get_dataset(permId)
except Exception as exc:
self.set_status(404)
self.write({
"reason" : 'No such dataSet found: {}'.format(permId)
})
return
# dataset was found, download the data to the disk
try:
destination = dataset.download(destination=downloadPath)
except Exception as exc:
self.set_status(500)
self.write({
"reason": 'Data for DataSet {} could not be downloaded: {}'.format(permId, exc)
})
return
# return success message
path = os.path.join(downloadPath, dataset.permId)
self.write({
'url' : conn.url,
'permId' : dataset.permId,
'path' : path,
'dataStore' : dataset.dataStore,
'location' : dataset.physicalData.location,
'size' : dataset.physicalData.size,
'statusText': 'Data for DataSet {} was successfully downloaded to: {}.'.format(dataset.permId, path)
})
def get(self, **params):
"""Handle a request to /openbis/dataset/connection_name/permId
download the data and return a message
"""
try:
conn = openbis_connections[params['connection_name']]
except KeyError:
self.set_status(404)
self.write({
"reason":'connection {} was not found'.format(params['connection_name'])
})
return
results = self.download_data(conn=conn, permId=params['permId'], downloadPath=params['downloadPath'])
class DataSetTypesHandler(IPythonHandler):
def get(self, **params):
"""Handle a request to /openbis/datasetTypes/connection_name
"""
try:
conn = openbis_connections[params['connection_name']]
except KeyError:
self.set_status(404)
self.write({
"reason":'connection {} was not found'.format(params['connection_name'])
})
return
try:
dataset_types = conn.openbis.get_dataset_types()
dts = dataset_types.df.to_dict(orient='records')
# get property assignments for every dataset-type
# and add it to the dataset collection
for dt in dts:
dataset_type = conn.openbis.get_dataset_type(dt['code'])
pa = dataset_type.get_propertyAssignments(including_vocabulary=True)
pa_dicts = pa.to_dict(orient='records')
for pa_dict in pa_dicts:
if pa_dict['dataType'] == 'CONTROLLEDVOCABULARY':
terms = conn.openbis.get_terms(pa_dict['vocabulary']['code'])
pa_dict['terms'] = terms.df[['code','label','description','official','ordinal']].to_dict(orient='records')
dt['propertyAssignments'] = pa_dicts
self.write({
"dataSetTypes": dts
})
return
except Exception as e:
print(e)
self.set_status(500)
self.write({
"reason":'Could not fetch dataset-types: {}'.format(e)
})
return
class DataSetUploadHandler(IPythonHandler):
"""Handle the POST requests for /openbis/dataset/connection_name"""
def upload_data(self, conn, data):
if not conn.is_session_active():
try:
conn.login()
except Exception as e:
print(e)
self.set_status(500)
self.write({
"reason": 'connection to {} could not be established: {}'.format(conn.name, e)
})
return
errors = []
sample = None
experiment = None
if (data.get('sampleIdentifier')):
try:
sample = conn.openbis.get_sample(data.get('sampleIdentifier'))
except Exception as e:
print(e)
errors.append(
{"sampleIdentifier" : 'No such sample: {}'.format(data.get('sampleIdentifier')) }
)
else:
errors.append(
{"sampleIdentifier": "please provide a sample identifier"}
)
print("--------------_HERE_-----------1")
if (data.get('experimentIdentifier')):
try:
experiment = conn.openbis.get_experiment(data.get('experimentIdentifier'))
except Exception as e:
print(e)
errors.append(
{"experimentIdentifier" : 'No such experiment: {}'.format(data.get('experimentIdentifier')) }
)
print("--------------_HERE_-----------2")
filenames = []
for filename in data.get('files'):
filename = unquote(filename)
if os.path.isfile(filename):
filenames.append(filename)
else:
errors.append({
"file": "File not found: {}".format(filename)
})
print("--------------_HERE_-----------3")
try:
dataset = conn.openbis.new_dataset(
type = data.get('type'),
sample = sample,
experiment = experiment,
files = filenames,
)
except Exception as e:
print(e)
errors.append({
"create": 'Error while creating the dataset: {}'.format(e)
})
print("--------------_HERE_-----------4")
# try to set the properties
if (data.get('props')):
props = data.get('props')
for prop, value in props.items():
try:
setattr(dataset.props, prop.lower(), value)
except Exception as e:
errors.append({
"prop."+prop : str(e)
})
print("--------------_HERE_-----------5")
# write errors back if already occured
if errors:
self.set_status(500)
self.write({ "errors": errors })
return
print("--------------_HERE_-----------6")
try:
dataset.save()
except Exception as e:
errors.append({
"save": 'Error while saving the dataset: {}'.format(e)
})
print("--------------_HERE_-----------7")
# write errors back if they occured
if errors:
self.set_status(500)
self.write({ "errors": errors })
else:
# ...or return a success message
self.write({
'status': 200,
'statusText': 'Jupyter Notebook was successfully uploaded to: {} with permId: {}'.format(conn.name, dataset.permId)
})
print("--------------_HERE_-----------8")
def post(self, **params):
"""Handle a request to /openbis/dataset/connection_name/permId
download the data and return a message
"""
try:
conn = openbis_connections[params['connection_name']]
except KeyError:
self.write({
"reason": 'connection {} was not found'.format(params['connection_name'])
})
return
data = self.get_json_body()
print("Received DATA")
print(data)
self.upload_data(conn=conn,data=data)