Newer
Older
if sig_int < 0:
sig_int += 2**32
return "%x"%(sig_int & 0xFFFFFFFF)
files = self.get_file_list(start_folder=start_folder)
df = DataFrame(files)
df['relativePath'] = df['pathInDataSet'].map(createRelativePath)
df['crc32Checksum'] = df['crc32Checksum'].fillna(0.0).astype(int).map(signed_to_unsigned)
return df[['isDirectory', 'pathInDataSet', 'fileSize', 'crc32Checksum']]
def get_file_list(self, recursive=True, start_folder="/"):
"""Lists all files of a given dataset. You can specifiy a start_folder other than "/".
By default, all directories and their containing files are listed recursively. You can
turn off this option by setting recursive=False.
"""
request = {
"method" : "listFilesForDataSet",
"params" : [
self.openbis.token,
"id":"1"
self.data["dataStore"]["downloadUrl"] + '/datastore_server/rmi-dss-api-v1.json',
json.dumps(request),
verify=self.openbis.verify_certificates
)
data = resp.json()
if 'error' in data:
Swen Vermeul
committed
raise ValueError('Error from openBIS: ' + data['error'] )
elif 'result' in data:
return data['result']
Swen Vermeul
committed
raise ValueError('request to openBIS did not return either result nor error')
Swen Vermeul
committed
raise ValueError('internal error while performing post request')
class AttrHolder():
""" General class for both samples and experiments that hold all common attributes, such as:
- space
Swen Vermeul
committed
- experiment (sample)
- samples (experiment)
Swen Vermeul
committed
- parents (sample, dataset)
- children (sample, dataset)
- tags
"""
def __init__(self, openbis_obj, entity, type=None):
self.__dict__['_openbis'] = openbis_obj
self.__dict__['_entity'] = entity
if type is not None:
self.__dict__['_allowed_attrs'] = _definitions(entity)['attrs']
self.__dict__['_identifier'] = None
self.__dict__['_is_new'] = True
Swen Vermeul
committed
def __call__(self, data):
self.__dict__['_is_new'] = False
Swen Vermeul
committed
for attr in self._allowed_attrs:
if attr in ["code","permId","identifier","type", "container","components"]:
Swen Vermeul
committed
self.__dict__['_'+attr] = data.get(attr, None)
Swen Vermeul
committed
d = data.get(attr, None)
if d is not None:
d = d['permId']
self.__dict__['_'+attr] = d
elif attr in ["sample", "experiment", "project"]:
Swen Vermeul
committed
d = data.get(attr, None)
if d is not None:
d = d['identifier']
self.__dict__['_'+attr] = d
elif attr in ["parents","children","samples"]:
Swen Vermeul
committed
self.__dict__['_'+attr] = []
for item in data[attr]:
if 'identifier' in item:
self.__dict__['_'+attr].append(item['identifier'])
elif 'permId' in item:
self.__dict__['_'+attr].append(item['permId'])
Swen Vermeul
committed
Swen Vermeul
committed
for item in data[attr]:
Swen Vermeul
committed
"code": item['code'],
"@type": "as.dto.tag.id.TagCode"
})
self.__dict__['_tags'] = tags
import copy
self.__dict__['_prev_tags'] = copy.deepcopy(tags)
else:
self.__dict__['_'+attr] = data.get(attr, None)
def _all_attrs(self):
attr2ids = _definitions('attr2ids')
ids2type = _definitions('ids2type')
request = {}
# look at all attributes available for that entity
Swen Vermeul
committed
for attr in self._allowed_attrs:
# these attributes cannot be changed (or not directly)
if attr in ["code", "permId", "identifier", "type", "registrator", "registrationDate", "modifier", "modificationDate"]:
Swen Vermeul
committed
continue
if attr == 'attachments':
# v3 API currently only supports adding attachments
attachments = self.__dict__.get('_new_attachments', None)
if attachments is None:
continue
atts_data = [ attachment.get_data() for attachment in attachments ]
if self._is_new:
request['attachments'] = atts_data
else:
request['attachments'] = {
"actions": [ {
"items": atts_data,
"@type": "as.dto.common.update.ListUpdateActionAdd"
} ],
"@type": "as.dto.attachment.update.AttachmentListUpdateValue"
}
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
elif attr == 'tagss':
if getattr(self,'_prev_tags') is None:
self.__dict__['_prev_tags'] = []
actions = []
for tagId in self._prev_tags:
if tagId not in self._tags:
actions.append({
"items": [ tagId ],
"@type": "as.dto.common.update.ListUpdateActionRemove"
})
for tagId in self._tags:
if tagId not in self._prev_tags:
actions.append({
"items": [ tagId ],
"@type": "as.dto.common.update.ListUpdateActionAdd"
})
request['tagIds'] = {
"@type": "as.dto.common.update.IdListUpdateValue",
"actions": actions
}
if self._is_new:
# handle multivalue attributes (parents, children, tags etc.)
if attr in defs['multi']:
items = self.__dict__.get('_'+attr, [])
if items == None:
items = []
Swen Vermeul
committed
request[attr2ids[attr]] = items
Swen Vermeul
committed
request[attr2ids[attr]] = self.__dict__.get('_'+attr, None)
else:
# handle multivalue attributes (parents, children, tags etc.)
# we only cover the Set mechanism, which means we always update all items in a
# list
if attr in defs['multi']:
items = self.__dict__.get('_'+attr, [])
if items == None:
items = []
Swen Vermeul
committed
request[attr2ids[attr]] = {
"actions": [
{
"items": items,
"@type": "as.dto.common.update.ListUpdateActionSet",
}
],
"@type": "as.dto.common.update.IdListUpdateValue"
}
else:
# handle single attributes (space, experiment, project, container, etc.)
value = self.__dict__.get('_'+attr, {})
if value is None:
pass
else:
isModified=False
if 'isModified' in value:
isModified=True
del value['isModified']
Swen Vermeul
committed
request[attr2ids[attr]] = {
"@type": "as.dto.common.update.FieldUpdateValue",
"isModified": isModified,
"value": value,
}
#if self.__dict__.get('_code', None) is None:
# request['autoGeneratedCode'] = True
#else:
# pass
return request
def __getattr__(self, name):
""" handles all attribute requests dynamically. Values are returned in a sensible way,
for example the identifiers of parents, children and components are returned
as an array of values.
"""
int_name = '_'+name
if int_name in self.__dict__:
return self.__dict__[int_name].get('userId', None)
elif int_name in ['_registrationDate', '_modificationDate']:
return format_timestamp(self.__dict__[int_name])
# if the attribute contains a list,
# return a list of either identifiers, codes or
# permIds (whatever is available first)
elif isinstance(self.__dict__[int_name], list):
values = []
for item in self.__dict__[int_name]:
if "identifier" in item:
values.append(item['identifier'])
elif "code" in item:
values.append(item['code'])
elif "permId" in item:
values.append(item['permId'])
else:
pass
return values
# attribute contains a dictionary: same procedure as above.
elif isinstance(self.__dict__[int_name], dict):
if "identifier" in self.__dict__[int_name]:
return self.__dict__[int_name]['identifier']
elif "code" in self.__dict__[int_name]:
return self.__dict__[int_name]['code']
elif "permId" in self.__dict__[int_name]:
return self.__dict__[int_name]['permId']
else:
return self.__dict__[int_name]
else:
return None
def __setattr__(self, name, value):
if name in ["parents", "children", "components"]:
if not isinstance(value, list):
value = [value]
objs = []
for val in value:
# fetch objects in openBIS, make sure they actually exists
obj = getattr(self._openbis, 'get_'+self._entity.lower())(val)
objs.append(obj)
self.__dict__['_'+name] = {
"@type": "as.dto.common.update.IdListUpdateValue",
"actions": [{
"@type": "as.dto.common.update.ListUpdateActionSet",
"items": [item._permId for item in objs]
}]
}
elif name in ["tags"]:
self.set_tags(value)
elif name in ["sample", "experiment"]:
if isinstance(value, object):
obj = value
else:
# fetch object in openBIS, make sure it actually exists
obj = getattr(self._openbis, "get_"+name)(value)
self.__dict__['_'+name] = obj.data['identifier']
# mark attribute as modified, if it's an existing entity
if self.__dict__['_is_new']:
pass
else:
self.__dict__['_'+name]['isModified'] = True
elif name in ["space"]:
# fetch object in openBIS, make sure it actually exists
obj = getattr(self._openbis, "get_"+name)(value)
self.__dict__['_'+name] = obj.data['permId']
# mark attribute as modified, if it's an existing entity
if self.__dict__['_is_new']:
pass
else:
elif name in ["project"]:
# fetch object in openBIS, make sure it actually exists
obj = getattr(self._openbis, "get_"+name)(value)
self.__dict__['_'+name] = obj.data['identifier']
# mark attribute as modified, if it's an existing entity
if self.__dict__['_is_new']:
pass
else:
self.__dict__['_'+name]['isModified'] = True
elif name in ["identifier"]:
raise KeyError("you can not modify the {}".format(name))
elif name == "code":
if self.__dict__['_type'].data['autoGeneratedCode']:
raise KeyError("for this {}Type you can not set a code".format(self.__dict__['_entity']))
else:
self.__dict__['_code'] = value
else:
raise KeyError("no such attribute: {}".format(name))
def get_type(self):
def get_parents(self):
Swen Vermeul
committed
# e.g. self._openbis.get_samples(withChildren=self.identifier)
return getattr(self._openbis, 'get_'+self._entity.lower()+'s')(withChildren=self.identifier)
def get_children(self):
Swen Vermeul
committed
# e.g. self._openbis.get_samples(withParents=self.identifier)
return getattr(self._openbis, 'get_'+self._entity.lower()+'s')(withParents=self.identifier)
@property
def tags(self):
if getattr(self, '_tags') is not None:
return [ x['code'] for x in self._tags ]
def set_tags(self, tags):
if getattr(self, '_tags') is None:
self.__dict__['_tags'] = []
tagIds = _create_tagIds(tags)
# remove tags that are not in the new tags list
for tagId in self.__dict__['_tags']:
if tagId not in tagIds:
self.__dict__['_tags'].remove(tagId)
# add all new tags that are not in the list yet
for tagId in tagIds:
if tagId not in self.__dict__['_tags']:
self.__dict__['_tags'].append(tagId)
def add_tags(self, tags):
if getattr(self, '_tags') is None:
self.__dict__['_tags'] = []
# add the new tags to the _tags and _new_tags list,
# if not listed yet
tagIds = _create_tagIds(tags)
for tagId in tagIds:
if not tagId in self.__dict__['_tags']:
self.__dict__['_tags'].append(tagId)
def del_tags(self, tags):
if getattr(self, '_tags') is None:
self.__dict__['_tags'] = []
# remove the tags from the _tags and _del_tags list,
# if listed there
tagIds = _create_tagIds(tags)
for tagId in tagIds:
if tagId in self.__dict__['_tags']:
self.__dict__['_tags'].remove(tagId)
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
def get_attachments(self):
if getattr(self, '_attachments') is None:
return None
else:
return DataFrame(self._attachments)[['fileName','title','description','version']]
def add_attachment(self, filename, title=None, description=None):
att = Attachment(filename=filename, title=title, description=description)
if getattr(self, '_attachments') is None:
self.__dict__['_attachments'] = []
self._attachments.append(att.get_data_short())
if getattr(self, '_new_attachments') is None:
self.__dict__['_new_attachments'] = []
self._new_attachments.append(att)
def download_attachments(self):
method = 'get'+self.entity+'s'
entity = self.entity.lower()
request = {
"method": method,
"params": [ self._openbis.token,
[ self._permId ],
{
"attachments": fetch_option['attachmentsWithContent'],
**fetch_option[entity]
}
]
}
resp = self._openbis._post_request(self._openbis.as_v3, request)
attachments = resp[self.permId]['attachments']
file_list = []
for attachment in attachments:
filename = os.path.join(
self._openbis.hostname,
self.permId,
attachment['fileName']
)
os.makedirs(os.path.dirname(filename), exist_ok=True)
with open(filename, 'wb') as att:
content = base64.b64decode(attachment['content'])
att.write(content)
file_list.append(filename)
return file_list
def _repr_html_(self):
Swen Vermeul
committed
def nvl(val, string=''):
if val is None:
return string
return val
html = """
<table border="1" class="dataframe">
<thead>
<tr style="text-align: right;">
<th>attribute</th>
<th>value</th>
</tr>
</thead>
<tbody>
"""
Swen Vermeul
committed
for attr in self._allowed_attrs:
if attr == 'attachments':
continue
html += "<tr> <td>{}</td> <td>{}</td> </tr>".format(
attr, nvl(getattr(self, attr, ''),'')
)
if getattr(self, '_attachments') is not None:
Swen Vermeul
committed
html += "<tr><td>attachments</td><td>"
html += "<br/>".join(att['fileName'] for att in self._attachments)
html += "</td></tr>"
html += """
</tbody>
</table>
"""
return html
class Sample():
""" A Sample is one of the most commonly used objects in openBIS.
"""
def __init__(self, openbis_obj, type, data=None, **kwargs):
self.__dict__['openbis'] = openbis_obj
self.__dict__['type'] = type
self.__dict__['p'] = PropertyHolder(openbis_obj, type)
self.__dict__['a'] = AttrHolder(openbis_obj, 'Sample', type)
if data is not None:
self._set_data(data)
if kwargs is not None:
for key in kwargs:
setattr(self, key, kwargs[key])
def _set_data(self, data):
# assign the attribute data to self.a by calling it
# (invoking the AttrHolder.__call__ function)
self.a(data)
self.__dict__['data'] = data
# put the properties in the self.p namespace (without checking them)
for key, value in data['properties'].items():
self.p.__dict__[key.lower()] = value
Swen Vermeul
committed
def __dir__(self):
return [
'props', 'get_parents()', 'get_children()',
'get_datasets()', 'get_experiment()',
'space', 'project', 'experiment', 'project', 'tags',
'set_tags()', 'add_tags()', 'del_tags()',
'add_attachment()', 'get_attachments()', 'download_attachments()'
]
Swen Vermeul
committed
@property
def props(self):
return self.__dict__['p']
Swen Vermeul
committed
@property
def type(self):
@type.setter
def type(self, type_name):
sample_type = self.openbis.get_sample_type(type_name)
self.p.__dict__['_type'] = sample_type
self.a.__dict__['_type'] = sample_type
def __getattr__(self, name):
return getattr(self.__dict__['a'], name)
def __setattr__(self, name, value):
if name in ['set_properties', 'set_tags', 'add_tags']:
raise ValueError("These are methods which should not be overwritten")
setattr(self.__dict__['a'], name, value)
def _repr_html_(self):
html = self.a._repr_html_()
return html
def set_properties(self, properties):
self.openbis.update_sample(self.permId, properties=properties)
def save(self):
props = self.p._all_props()
attrs = self.a._all_attrs()
attrs["properties"] = props
if self.identifier is None:
# create a new sample
attrs["@type"] = "as.dto.sample.create.SampleCreation"
attrs["typeId"] = self.__dict__['type'].data['permId']
request = {
"method": "createSamples",
"params": [ self.openbis.token,
[ attrs ]
]
}
resp = self.openbis._post_request(self.openbis.as_v3, request)
new_sample_data = self.openbis.get_sample(resp[0]['permId'], only_data=True)
self._set_data(new_sample_data)
return self
else:
attrs["@type"] = "as.dto.sample.update.SampleUpdate"
attrs["sampleId"] = {
"permId": self.permId,
"@type": "as.dto.sample.id.SamplePermId"
}
request = {
"method": "updateSamples",
"params": [ self.openbis.token,
[ attrs ]
]
}
resp = self.openbis._post_request(self.openbis.as_v3, request)
print('Sample successfully updated')
self.openbis.delete_entity('sample', self.permId, reason)
def get_datasets(self):
return self.openbis.get_datasets(sample=self.permId)
def get_projects(self):
return self.openbis.get_project(withSamples=[self.permId])
Swen Vermeul
committed
def get_experiment(self):
try:
return self.openbis.get_experiment(self._experiment['identifier'])
except Exception:
pass
@property
def experiment(self):
try:
return self.openbis.get_experiment(self._experiment['identifier'])
except Exception:
pass
Swen Vermeul
committed
""" managing openBIS spaces
"""
def __init__(self, openbis_obj, type=None, data=None, **kwargs):
self.__dict__['openbis'] = openbis_obj
self.__dict__['a'] = AttrHolder(openbis_obj, 'Space', type)
if data is not None:
self.a(data)
self.__dict__['data'] = data
if kwargs is not None:
for key in kwargs:
setattr(self, key, kwargs[key])
def __dir__(self):
"""all the available methods and attributes that should be displayed
when using the autocompletion feature (TAB) in Jupyter
"""
return['code','permId', 'description', 'registrator', 'registrationDate',
'modificationDate', 'get_projects()', 'get_samples()', 'delete()']
def __str__(self):
return self.data.get('code', None)
def get_samples(self):
return self.openbis.get_samples(space=self.code)
def get_projects(self):
return self.openbis.get_projects(space=self.code)
def new_project(self, code, description):
return self.openbis.new_project(self.code, code, description)
def delete(self, reason):
self.openbis.delete_entity('Space', self.permId, reason)
class Things():
"""An object that contains a DataFrame object about an entity available in openBIS.
"""
def __init__(self, openbis_obj, entity, df, identifier_name='code'):
self.df = df
self.identifier_name = identifier_name
def _repr_html_(self):
return self.df._repr_html_()
def __getitem__(self, key):
if self.df is not None and len(self.df) > 0:
row = None
if isinstance(key, int):
# get thing by rowid
row = self.df.loc[[key]]
elif isinstance(key, list):
# treat it as a normal dataframe
return self.df[key]
else:
# get thing by code
row = self.df[self.df[self.identifier_name]==key.upper()]
if row is not None:
# invoke the openbis.get_entity() method
return getattr(self.openbis, 'get_'+self.entity)(row[self.identifier_name].values[0])
Swen Vermeul
committed
"""
Swen Vermeul
committed
def __init__(self, openbis_obj, type, data=None, **kwargs):
self.__dict__['openbis'] = openbis_obj
self.__dict__['type'] = type
self.__dict__['p'] = PropertyHolder(openbis_obj, type)
self.__dict__['a'] = AttrHolder(openbis_obj, 'Experiment', type)
Swen Vermeul
committed
if data is not None:
self._set_data(data)
if kwargs is not None:
for key in kwargs:
setattr(self, key, kwargs[key])
def _set_data(self, data):
# assign the attribute data to self.a by calling it
# (invoking the AttrHolder.__call__ function)
self.a(data)
self.__dict__['data'] = data
# put the properties in the self.p namespace (without checking them)
for key, value in data['properties'].items():
self.p.__dict__[key.lower()] = value
def __str__(self):
return self.data['code']
Swen Vermeul
committed
def __dir__(self):
# the list of possible methods/attributes displayed
# when invoking TAB-completition
return [
Swen Vermeul
committed
'project','tags', 'attachments', 'data',
'set_tags()', 'add_tags()', 'del_tags()',
'add_attachment()', 'get_attachments()', 'download_attachments()'
Swen Vermeul
committed
]
@property
def props(self):
return self.__dict__['p']
@property
def type(self):
Swen Vermeul
committed
@type.setter
def type(self, type_name):
experiment_type = self.openbis.get_experiment_type(type_name)
self.p.__dict__['_type'] = experiment_type
self.a.__dict__['_type'] = experiment_type
def __getattr__(self, name):
return getattr(self.__dict__['a'], name)
def __setattr__(self, name, value):
if name in ['set_properties', 'add_tags()', 'del_tags()', 'set_tags()']:
Swen Vermeul
committed
raise ValueError("These are methods which should not be overwritten")
setattr(self.__dict__['a'], name, value)
def _repr_html_(self):
html = self.a._repr_html_()
return html
def set_properties(self, properties):
self.openbis.update_experiment(self.permId, properties=properties)
Swen Vermeul
committed
def save(self):
props = self.p._all_props()
attrs = self.a._all_attrs()
attrs["properties"] = props
Swen Vermeul
committed
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
if self.identifier is None:
# create a new experiment
attrs["@type"] = "as.dto.experiment.create.ExperimentCreation"
attrs["typeId"] = self.__dict__['type'].data['permId']
request = {
"method": "createExperiments",
"params": [ self.openbis.token,
[ attrs ]
]
}
resp = self.openbis._post_request(self.openbis.as_v3, request)
new_experiment_data = self.openbis.get_experiment(resp[0]['permId'], only_data=True)
self._set_data(new_experiment_data)
return self
else:
attrs["@type"] = "as.dto.experiment.update.ExperimentUpdate"
attrs["experimentId"] = {
"permId": self.permId,
"@type": "as.dto.experiment.id.ExperimentPermId"
}
request = {
"method": "updateExperiments",
"params": [ self.openbis.token,
[ attrs ]
]
}
resp = self.openbis._post_request(self.openbis.as_v3, request)
print('Experiment successfully updated')
def delete(self, reason):
self.openbis.delete_entity('experiment', self.permId, reason)
Swen Vermeul
committed
def get_datasets(self):
return self.openbis.get_datasets(experiment=self.permId)
Swen Vermeul
committed
def get_projects(self):
return self.openbis.get_project(experiment=self.permId)
Swen Vermeul
committed
def get_samples(self):
return self.openbis.get_samples(experiment=self.permId)
Swen Vermeul
committed
def __init__(self, filename, title=None, description=None):
if not os.path.exists(filename):
raise ValueError("File not found: {}".format(filename))
self.fileName = filename
self.title = title
self.description = description
Swen Vermeul
committed
def get_data_short(self):
return {
"fileName" : self.fileName,
"title" : self.title,
"description" : self.description,
}
Swen Vermeul
committed
def get_data(self):
with open(self.fileName, 'rb') as att:
content = att.read()
contentb64 = base64.b64encode(content).decode()
return {
"fileName" : self.fileName,
"title" : self.title,
"description" : self.description,
"content" : contentb64,
"@type" : "as.dto.attachment.create.AttachmentCreation",
}
Swen Vermeul
committed
def __init__(self, openbis_obj, data=None, **kwargs):
self.__dict__['openbis'] = openbis_obj
self.__dict__['a'] = AttrHolder(openbis_obj, 'Project')
if data is not None:
self.a(data)
self.__dict__['data'] = data
if kwargs is not None:
for key in kwargs:
setattr(self, key, kwargs[key])
def _modifiable_attrs(self):
return
def __dir__(self):
"""all the available methods and attributes that should be displayed
when using the autocompletion feature (TAB) in Jupyter
"""
return['code','permId', 'identifier', 'description', 'space', 'registrator',
'registrationDate', 'modifier', 'modificationDate', 'add_attachment()',
'get_attachments()', 'download_attachments()',
'get_experiments()', 'get_samples()', 'get_datasets()',
'delete()'
]
def get_samples(self):
return self.openbis.get_samples(project=self.permId)
def get_experiments(self):
return self.openbis.get_experiments(project=self.permId)
def get_datasets(self):
return self.openbis.get_datasets(project=self.permId)
def delete(self, reason):
self.openbis.delete_entity('project', self.permId, reason)
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
def save(self):
attrs = self.a._all_attrs()
if self.identifier is None:
attrs["@type"] = "as.dto.project.create.ProjectCreation"
attrs["typeId"] = self.__dict__['type'].data['permId']
request = {
"method": "createProjects",
"params": [ self.openbis.token, [ attrs ] ]
}
resp = self.openbis._post_request(self.openbis.as_v3, request)
new_sample_data = self.openbis.get_sample(resp[0]['permId'], only_data=True)
self._set_data(new_sample_data)
return self
else:
attrs["@type"] = "as.dto.project.update.ProjectUpdate"
attrs["projectId"] = {
"permId": self.permId,
"@type": "as.dto.project.id.ProjectPermId"
}
request = {
"method": "updateProjects",
"params": [ self.openbis.token,
[ attrs ]
]
}
resp = self.openbis._post_request(self.openbis.as_v3, request)
print('Project successfully updated')