import os
from urllib.parse import unquote
from notebook.base.handlers import IPythonHandler
from .connection import openbis_connections
class DataSetDownloadHandler(IPythonHandler):
"""Handle the requests for /openbis/dataset/connection/permId"""
def download_data(self, conn, permId, downloadPath=None):
if not conn.is_session_active():
try:
conn.login()
except Exception as exc:
self.set_status(500)
self.write({
"reason": 'connection to {} could not be established: {}'.format(conn.name, exc)
})
return
try:
dataset = conn.openbis.get_dataset(permId)
except Exception as exc:
self.set_status(404)
self.write({
"reason": 'No such dataSet found: {}'.format(permId)
})
return
# dataset was found, download the data to the disk
try:
destination = dataset.download(destination=downloadPath)
except Exception as exc:
self.set_status(500)
self.write({
"reason": 'Data for DataSet {} could not be downloaded: {}'.format(permId, exc)
})
return
# return success message
path = os.path.join(downloadPath, dataset.permId)
self.write({
'url' : conn.url,
'permId' : dataset.permId,
'path' : path,
'dataStore' : dataset.dataStore,
'location' : dataset.physicalData.location,
'size' : dataset.physicalData.size,
'files' : dataset.file_list,
'statusText': 'Data for DataSet {} was successfully downloaded to: {}'.format(dataset.permId, path)
})
def get(self, **params):
"""Handle a request to /openbis/dataset/connection_name/permId
download the data and return a message
"""
try:
conn = openbis_connections[params['connection_name']]
except KeyError:
self.set_status(404)
self.write({
"reason": 'connection {} was not found'.format(params['connection_name'])
})
return
results = self.download_data(conn=conn, permId=params['permId'], downloadPath=params['downloadPath'])
class DataSetTypesHandler(IPythonHandler):
def get(self, **params):
"""Handle a request to /openbis/datasetTypes/connection_name
This meta-metadata is used in the dataset upload dialog (uploadDialog.js)
to check data directly in the UI
Returns all datasetTypes of a given connection
- with all assigned properties
- with some details about the property types
- with the vocabulary, if exists
The result will be cached, as it is a costly operation with many fetches
"""
try:
conn = openbis_connections[params['connection_name']]
except KeyError:
self.set_status(404)
self.write({
"reason": 'connection {} was not found'.format(params['connection_name'])
})
return
if getattr(conn, 'dataset_types', False):
self.write({
"dataSetTypes": conn.dataset_types
})
return
try:
# get all dataset types
ds_type_dicts = []
for dt in conn.openbis.get_dataset_types():
dt_dict = dt.attrs.all()
# get property assignments for every dataset-type
# and add them in the key «propertyAssignments»
pas = dt.get_property_assignments()
pas_columns = ['propertyType', 'mandatory', 'ordinal', 'section']
if not set(pas.df).isdisjoint(pas_columns):
pa_dicts = pas.df[pas_columns].to_dict(orient='records')
dt_dict['propertyAssignments'] = pa_dicts
for pa_dict in pa_dicts:
# add a few more attributes to the property assignments
pt = conn.openbis.get_property_type(pa_dict['propertyType'])
pa_dict['code'] = pt.code
pa_dict['label'] = pt.label
pa_dict['description'] = pt.description
pa_dict['dataType'] = pt.dataType
# add vocabulary, if exists, as key «terms»
if pt.dataType == 'CONTROLLEDVOCABULARY':
terms = conn.openbis.get_terms(pt.vocabulary)
terms_columns = ['code', 'label', 'description', 'official', 'ordinal']
if not set(terms.df).isdisjoint(terms_columns):
terms_dict = terms.df[terms_columns].to_dict(orient='records')
pa_dict['terms'] = terms_dict
ds_type_dicts.append(dt_dict)
self.write({
"dataSetTypes": ds_type_dicts
})
conn.dataset_types = ds_type_dicts
return
except Exception as e:
print(e)
self.set_status(500)
self.write({
"reason":'Could not fetch dataset-types: {}'.format(e)
})
return
class DataSetUploadHandler(IPythonHandler):
"""Handle the POST requests for /openbis/dataset/connection_name"""
def _notebook_dir(self):
notebook_dir = os.getcwd()
if 'SingleUserNotebookApp' in self.config and 'notebook_dir' in self.config.SingleUserNotebookApp:
notebook_dir = self.config.SingleUserNotebookApp.notebook_dir
elif 'notebook_dir' in self.config.NotebookApp:
notebook_dir = self.config.NotebookApp.notebook_dir
return notebook_dir
def upload_data(self, conn, data):
if not conn.is_session_active():
try:
conn.login()
except Exception as e:
print(e)
self.set_status(500)
self.write({
"reason": 'connection to {} could not be established: {}'.format(conn.name, e)
})
return
errors = []
sample = None
experiment = None
if (data.get('entityIdentifier')):
sample = None
experiment = None
try:
sample = conn.openbis.get_sample(data.get('entityIdentifier'))
except Exception as e:
pass
if sample is None:
try:
experiment = conn.openbis.get_experiment(data.get('entityIdentifier'))
except Exception as e:
experiments = conn.openbis.get_experiments(data.get('entityIdentifier'))
if len(experiments) == 1:
experiment = experiments[0]
else:
# TODO: search in all experiments with same code
# (but maybe different identifiers)
pass
if sample is None and experiment is None:
errors.append(
{
"entityIdentifier" : 'No such sample or experiment: {}'
.format(data.get('entityIdentifier'))
}
)
else:
errors.append(
{"entityIdentifier": "please provide a sample or experiment identifier"}
)
parents = []
if data.get('parents'):
parents = data.get('parents')
for parent in parents:
try:
conn.openbis.get_dataset(parent)
except Exception as e:
errors.append({
"parent": "Parent DataSet not found: {}".format(parent)
})
filenames = []
notebook_dir = self._notebook_dir()
for filename in data.get('files'):
filename = unquote(filename)
full_filename_path = os.path.join(notebook_dir, filename)
if os.path.isfile(full_filename_path):
filenames.append(full_filename_path)
else:
errors.append({
"file": "File not found: {}".format(full_filename_path)
})
try:
dataset = conn.openbis.new_dataset(
type = data.get('type'),
sample = sample,
parents = parents,
experiment = experiment,
files = filenames,
)
except Exception as e:
self.set_status(500)
self.write({
"reason": 'Error while creating the dataset: {}'.format(e)
})
# try to set the properties
if data.get('props'):
props = data.get('props')
for prop, value in props.items():
try:
setattr(dataset.props, prop.lower(), value)
except Exception as e:
errors.append({
"prop."+prop : str(e)
})
# check if any mandatory property is missing
for prop_name, prop in dataset.props._property_names.items():
if prop['mandatory']:
if getattr(dataset.props, prop_name) is None or getattr(dataset.props, prop_name) == "":
errors.append({
"prop."+prop_name : "is mandatory"
})
# write errors back if already occured
if errors:
self.set_status(500)
self.write({"errors": errors})
return
try:
dataset.save()
except Exception as e:
errors.append({
"save": 'Error while saving the dataset: {}'.format(e)
})
# write errors back if they occured
if errors:
self.set_status(500)
self.write({"errors": errors})
else:
# ...or return a success message
self.write({
'status': 200,
'statusText': 'DataSet successfully registered: {} with permId: {}'.format(conn.name, dataset.permId)
})
print('DataSet successfully registered: {} with permId: {}'.format(conn.name, dataset.permId))
def post(self, **params):
"""Handle a request to /openbis/dataset/connection_name/permId
download the data and return a message
"""
try:
conn = openbis_connections[params['connection_name']]
except KeyError:
self.write({
"reason": 'connection {} was not found'.format(params['connection_name'])
})
return
data = self.get_json_body()
self.upload_data(conn=conn,data=data)