Newer
Older
vkovtun
committed
response=resp,
df_initializer=create_data_frame,
def get_space(self, code, only_data=False, use_cache=True):
"""Returns a Space object for a given identifier."""
code = str(code).upper()
space = (
not only_data
and use_cache
and self._object_cache(entity="space", code=code)
)
if space:
return space
fetchopts = {"@type": "as.dto.space.fetchoptions.SpaceFetchOptions"}
fetchopts[option] = fetch_option[option]
"params": [
self.token,
[{"permId": code, "@type": "as.dto.space.id.SpacePermId"}],
fetchopts,
resp = self._post_request(self.as_v3, request)
raise ValueError("No such space: %s" % code)
for permid in resp:
if only_data:
return resp[permid]
else:
space = Space(self, data=resp[permid])
if self.use_cache:
self._object_cache(entity="space", code=code, value=space)
return space
Chandrasekhar Ramakrishnan
committed
self,
identifier=None,
code=None,
permId=None,
space=None,
project=None,
experiment=None,
collection=None,
type=None,
start_with=None,
count=None,
withParents=None,
withChildren=None,
tags=None,
attrs=None,
props=None,
where=None,
"""Returns a DataFrame of all samples for a given space/project/experiment (or any combination)
Filters
-------
type -- sampleType code or object
space -- space code or object
project -- project code or object
experiment -- experiment code or object (can be a list, too)
tags -- only return samples with the specified tags
where -- key-value pairs of property values to search for
Paging
------
start_with -- default=None
count -- number of samples that should be fetched. default=None.
Include in result list
----------------------
withParents -- the list of parent's permIds in a column 'parents'
withChildren -- the list of children's permIds in a column 'children'
attrs -- list of all desired attributes. Examples:
space, project, experiment: just return their identifier
parents, children, components: return a list of their identifiers
space.code, project.code, experiment.code
registrator.email, registrator.firstName
type.generatedCodePrefix
props -- list of all desired properties. Returns an empty string if
a) property is not present
b) property is not defined for this sampleType
logger = logging.getLogger("get_samples")
logger.setLevel(logging.CRITICAL)
vkovtun
committed
logger.addHandler(logging.StreamHandler(sys.stdout))
if collection is not None:
experiment = collection
crit = _subcriteria_for(identifier, "sample")
sub_criteria += crit["criteria"]
sub_criteria.append(_subcriteria_for(space, "space"))
sub_criteria.append(_subcriteria_for(project, "project"))
sub_criteria.append(_subcriteria_for(experiment, "experiment"))
if withParents:
sub_criteria.append(_subcriteria_for(withParents, "sample", "Parents"))
if withChildren:
sub_criteria.append(_subcriteria_for(withChildren, "sample", "Children"))
if where:
if properties is None:
properties = where
else:
properties = {**where, **properties}
if properties is not None:
for prop in properties:
sub_criteria.append(
_subcriteria_for_properties(prop, properties[prop], entity="sample")
)
if type:
sub_criteria.append(_subcriteria_for_code(type, "sampleType"))
if tags:
sub_criteria.append(_subcriteria_for_tags(tags))
if code:
sub_criteria.append(
_subcriteria_for_code_new(code, "sample", operator="OR")
)
if permId:
sub_criteria.append(
_subcriteria_for_permid_new(permId, "sample", operator="OR")
)
criteria = {
"criteria": sub_criteria,
"@type": "as.dto.sample.search.SampleSearchCriteria",
# build the various fetch options
fetchopts = fetch_option["sample"]
fetchopts["from"] = start_with
fetchopts["count"] = count
options = [
"tags",
"properties",
"attachments",
"space",
"experiment",
"registrator",
"modifier",
]
if self.get_server_information().project_samples_enabled:
for option in options:
fetchopts[option] = fetch_option[option]
if props is not None:
fetchopts["properties"] = fetch_option["properties"]
request = {
"method": "searchSamples",
"params": [
self.token,
criteria,
fetchopts,
],
vkovtun
committed
time1 = now()
logger.debug("get_samples posting request")
Swen Vermeul
committed
resp = self._post_request(self.as_v3, request)
vkovtun
committed
time2 = now()
logger.debug(f"get_samples got response. Delay: {time2 - time1}")
vkovtun
committed
time3 = now()
response = resp["objects"]
logger.debug(f"get_samples got JSON. Delay: {time3 - time2}")
time4 = now()
logger.debug(f"get_samples after result mapping. Delay: {time4 - time3}")
result = self._sample_list_for_response(
response=response,
attrs=attrs,
props=props,
start_with=start_with,
count=count,
totalCount=resp["totalCount"],
parsed=True,
)
vkovtun
committed
time5 = now()
logger.debug(f"get_samples computed final result. Delay: {time5 - time4}")
return result
Swen Vermeul
committed
Swen Vermeul
committed
def _get_fetchopts_for_attrs(self, attrs=None):
if attrs is None:
fetchopts = []
for attr in attrs:
if attr.startswith("space"):
fetchopts.append("space")
if attr.startswith("project"):
fetchopts.append("project")
if attr.startswith("experiment"):
fetchopts.append("experiment")
if attr.startswith("sample"):
fetchopts.append("sample")
if attr.startswith("registrator"):
fetchopts.append("registrator")
if attr.startswith("modifier"):
fetchopts.append("modifier")
return fetchopts
self,
code=None,
permId=None,
type=None,
space=None,
project=None,
start_with=None,
count=None,
tags=None,
is_finished=None,
attrs=None,
props=None,
where=None,
**properties,
Swen Vermeul
committed
"""Returns a DataFrame of all samples for a given space/project (or any combination)
Swen Vermeul
committed
Filters:
--------
space -- a space code or a space object
project -- a project code or a project object
tags -- only experiments with the specified tags
type -- a experimentType code
where -- key-value pairs of property values to search for
Swen Vermeul
committed
Paging:
-------
start_with -- default=None
count -- number of samples that should be fetched. default=None.
Include:
--------
attrs -- list of all desired attributes. Examples:
space, project, experiment: just return their identifier
space.code, project.code, experiment.code
registrator.email, registrator.firstName
type.generatedCodePrefix
props -- list of all desired properties. Returns an empty string if
a) property is not present
b) property is not defined for this sampleType
Swen Vermeul
committed
def extract_attribute(attribute_to_extract):
def return_attribute(obj):
return ""
return obj.get(attribute_to_extract, "")
Swen Vermeul
committed
return return_attribute
def extract_space(obj):
if isinstance(obj, dict):
Swen Vermeul
committed
else:
Swen Vermeul
committed
sub_criteria = []
if space:
sub_criteria.append(
_subcriteria_for(space, "project.space", operator="AND")
)
sub_criteria.append(_subcriteria_for(project, "project", operator="AND"))
sub_criteria.append(_criteria_for_code(code))
sub_criteria.append(_criteria_for_permId(permId))
if type:
sub_criteria.append(_subcriteria_for_code(type, "experimentType"))
if tags:
sub_criteria.append(_subcriteria_for_tags(tags))
if is_finished is not None:
sub_criteria.append(_subcriteria_for_is_finished(is_finished))
if where:
if properties is None:
properties = where
else:
properties = {**where, **properties}
if properties is not None:
sub_criteria.extend(
list(
map(
lambda prop: _subcriteria_for_properties(
prop, properties[prop], entity="experiment"
),
properties,
)
)
)
search_criteria = get_search_type_for_entity("experiment")
search_criteria["criteria"] = sub_criteria
search_criteria["operator"] = "AND"
fetchopts = fetch_option["experiment"]
fetchopts["from"] = start_with
fetchopts["count"] = count
Swen Vermeul
committed
Swen Vermeul
committed
options = self._get_fetchopts_for_attrs(attrs)
for option in ["tags", "properties", "registrator", "modifier"] + options:
request = {
"method": "searchExperiments",
"params": [
self.token,
search_criteria,
fetchopts,
],
}
resp = self._post_request(self.as_v3, request)
Swen Vermeul
committed
vkovtun
committed
def create_data_frame(attrs, props, response):
response = response["objects"]
parse_jackson(response)
vkovtun
committed
default_attrs = [
"identifier",
"permId",
"type",
"registrator",
"registrationDate",
"modifier",
"modificationDate",
]
vkovtun
committed
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
display_attrs = default_attrs + attrs
if props is None:
props = []
else:
if isinstance(props, str):
props = [props]
if len(response) == 0:
for prop in props:
if prop == "*":
continue
display_attrs.append(prop)
experiments = DataFrame(columns=display_attrs)
else:
experiments = DataFrame(response)
experiments["space"] = experiments["project"].map(extract_space)
for attr in attrs:
if "." in attr:
entity, attribute_to_extract = attr.split(".")
experiments[attr] = experiments[entity].map(
extract_attribute(attribute_to_extract)
)
for attr in attrs:
# if no dot supplied, just display the code of the space, project or experiment
if attr in ["project"]:
experiments[attr] = experiments[attr].map(
extract_nested_identifier
)
vkovtun
committed
if attr in ["space"]:
experiments[attr] = experiments[attr].map(extract_code)
experiments["registrationDate"] = experiments["registrationDate"].map(
format_timestamp
)
experiments["modificationDate"] = experiments["modificationDate"].map(
format_timestamp
)
experiments["project"] = experiments["project"].map(extract_code)
experiments["registrator"] = experiments["registrator"].map(
extract_person
)
vkovtun
committed
experiments["modifier"] = experiments["modifier"].map(extract_person)
experiments["identifier"] = experiments["identifier"].map(
extract_identifier
)
experiments["permId"] = experiments["permId"].map(extract_permid)
experiments["type"] = experiments["type"].map(extract_code)
for prop in props:
if prop == "*":
# include all properties in dataFrame.
# expand the dataFrame by adding new columns
columns = []
for i, experiment in enumerate(response):
for prop_name, val in experiment.get(
"properties", {}
).items():
vkovtun
committed
experiments.loc[i, prop_name.upper()] = val
columns.append(prop_name.upper())
display_attrs += set(columns)
continue
else:
# property name is provided
for i, experiment in enumerate(response):
val = experiment.get("properties", {}).get(
prop, ""
) or experiment.get("properties", {}).get(prop.upper(), "")
experiments.loc[i, prop.upper()] = val
display_attrs.append(prop.upper())
return experiments[display_attrs]
Swen Vermeul
committed
entity="experiment",
identifier_name="identifier",
vkovtun
committed
attrs=attrs,
props=props,
response=resp,
df_initializer=create_data_frame,
get_collections = get_experiments # Alias
self,
permId=None,
code=None,
type=None,
withParents=None,
withChildren=None,
start_with=None,
count=None,
kind=None,
status=None,
sample=None,
experiment=None,
collection=None,
project=None,
space=None,
tags=None,
attrs=None,
props=None,
where=None,
"""Returns a DataFrame of all dataSets for a given project/experiment/sample (or any combination)
Filters
-------
permId -- the permId is the unique identifier of a dataSet. A list of permIds can be provided.
code -- actually a synonym for the permId of the dataSet.
project -- a project code or a project object
experiment -- an experiment code or an experiment object
sample -- a sample code/permId or a sample/object
collection -- same as experiment
tags -- only return dataSets with the specified tags
type -- a dataSetType code
where -- key-value pairs of property values to search for
Paging
------
start_with -- default=None
count -- number of dataSets that should be fetched. default=None.
Include in result list
----------------------
withParents -- the list of parent's permIds in a column 'parents'
withChildren -- the list of children's permIds in a column 'children'
attrs -- list of all desired attributes. Examples:
project, experiment, sample: just return their identifier
space.code, project.code, experiment.code
registrator.email, registrator.firstName
type.generatedCodePrefix
props -- list of all desired properties. Returns an empty string if
a) property is not present
b) property is not defined for this dataSetType
"""
if "object" in properties:
sample = properties["object"]
if collection is not None:
experiment = collection
sub_criteria = []
if code or permId:
if code is None:
sub_criteria.append(
_subcriteria_for_code_new(code, "dataSet", operator="OR")
)
if type:
sub_criteria.append(_subcriteria_for_code(type, "dataSetType"))
if withParents:
sub_criteria.append(_subcriteria_for(withParents, "dataSet", "Parents"))
if withChildren:
sub_criteria.append(_subcriteria_for(withChildren, "dataSet", "Children"))
sub_criteria.append(_subcriteria_for(sample, "sample"))
sub_criteria.append(_subcriteria_for(experiment, "experiment"))
sub_criteria.append(_subcriteria_for(project, "experiment.project"))
if space:
sub_criteria.append(_subcriteria_for(space, "experiment.project.space"))
if tags:
sub_criteria.append(_subcriteria_for_tags(tags))
if status:
sub_criteria.append(_subcriteria_for_status(status))
if where:
if properties is None:
properties = where
else:
properties = {**where, **properties}
sub_criteria.extend(
list(
map(
lambda prop: _subcriteria_for_properties(
prop, properties[prop], entity="dataset"
),
properties,
)
)
)
Swen Vermeul
committed
search_criteria = get_search_type_for_entity("dataset")
search_criteria["criteria"] = sub_criteria
search_criteria["operator"] = "AND"
fetchopts = get_fetchoptions("dataSet", including=["type"])
fetchopts["from"] = start_with
fetchopts["count"] = count
Swen Vermeul
committed
for option in [
"tags",
"properties",
"dataStore",
"physicalData",
"linkedData",
"experiment",
"sample",
"registrator",
"modifier",
Swen Vermeul
committed
]:
fetchopts[option] = fetch_option[option]
fetchopts["experiment"]["project"] = fetch_option["project"]
Swen Vermeul
committed
Swen Vermeul
committed
if kind:
kind = kind.upper()
if kind not in ["PHYSICAL_DATA", "CONTAINER", "LINK"]:
"unknown dataSet kind: {}. It should be one of the following: PHYSICAL_DATA, CONTAINER or LINK".format(
kind
)
)
fetchopts["kind"] = kind
raise NotImplementedError("you cannot search for dataSet kinds yet")
request = {
"method": "searchDataSets",
"params": [
self.token,
search_criteria,
fetchopts,
],
}
resp = self._post_request(self.as_v3, request)
Swen Vermeul
committed
parse_jackson(resp)
datasets = []
Swen Vermeul
committed
dataset = DataSet(
type=self.get_dataset_type(obj["type"]["code"]),
data=obj,
Swen Vermeul
committed
datasets.append(dataset)
return self._dataset_list_for_response(
Swen Vermeul
committed
attrs=attrs,
props=props,
start_with=start_with,
count=count,
Swen Vermeul
committed
objects=datasets,
parsed=True,
Swen Vermeul
committed
)
Swen Vermeul
committed
def get_experiment(
self, code, withAttachments=False, only_data=False, use_cache=True
):
"""Returns an experiment object for a given identifier (code)."""
experiment = (
not only_data
and use_cache
and self._object_cache(entity="experiment", code=code)
if experiment:
return experiment
search_request = _type_for_id(code, "experiment")
for option in [
"tags",
"properties",
"attachments",
"project",
"samples",
"registrator",
"modifier",
]:
fetchopts[option] = fetch_option[option]
Swen Vermeul
committed
if withAttachments:
fetchopts["attachments"] = fetch_option["attachmentsWithContent"]
Swen Vermeul
committed
"method": "getExperiments",
"params": [self.token, [search_request], fetchopts],
resp = self._post_request(self.as_v3, request)
if len(resp) == 0:
parse_jackson(resp)
data = resp[code]
if only_data:
return data
experiment = Experiment(
openbis_obj=self,
type=self.get_experiment_type(data["type"]["code"]),
)
if self.use_cache:
identifier = data["identifier"]["identifier"]
self._object_cache(entity="experiment", code=identifier, value=experiment)
return experiment
get_collection = get_experiment # Alias
def new_experiment(self, type, code, project, props=None, **kwargs):
"""Creates a new experiment of a given experiment type."""
openbis_obj=self,
type=self.get_experiment_type(type),
project=project,
data=None,
props=props,
code=code,
new_collection = new_experiment # Alias
def create_external_data_management_system(
self, code, label, address, address_type="FILE_SYSTEM"
):
Chandrasekhar Ramakrishnan
committed
"""Create an external DMS.
:param code: An openBIS code for the external DMS.
:param label: A human-readable label.
:param address: The address for accessing the external DMS. E.g., a URL.
Chandrasekhar Ramakrishnan
committed
:param address_type: One of OPENBIS, URL, or FILE_SYSTEM
Chandrasekhar Ramakrishnan
committed
:return:
"""
request = {
"method": "createExternalDataManagementSystems",
"params": [
self.token,
[
{
"code": code,
"label": label,
"addressType": address_type,
"address": address,
"@type": "as.dto.externaldms.create.ExternalDmsCreation",
}
Chandrasekhar Ramakrishnan
committed
],
}
resp = self._post_request(self.as_v3, request)
return self.get_external_data_management_system(resp[0]["permId"])
def delete_entity(self, entity, id, reason, id_name="permId"):
"""Deletes Spaces, Projects, Experiments, Samples and DataSets"""
type = get_type_for_entity(entity, "delete")
method = get_method_for_entity(entity, "delete")
request = {
"params": [
self.token,
[{id_name: id, "@type": type}],
{"reason": reason, "@type": type},
],
}
self._post_request(self.as_v3, request)
def delete_openbis_entity(self, entity, objectId, reason="No reason given"):
method = get_method_for_entity(entity, "delete")
delete_options = get_type_for_entity(entity, "delete")
delete_options["reason"] = reason
request = {"method": method, "params": [self.token, [objectId], delete_options]}
def get_deletions(self, start_with=None, count=None):
search_criteria = {"@type": "as.dto.deletion.search.DeletionSearchCriteria"}
fetchopts = fetch_option["deletion"]
fetchoptsDeleted = fetch_option["deletedObjects"]
fetchoptsDeleted["from"] = start_with
fetchoptsDeleted["count"] = count
fetchopts["deletedObjects"] = fetchoptsDeleted
request = {
"method": "searchDeletions",
"params": [
self.token,
}
resp = self._post_request(self.as_v3, request)
parse_jackson(objects)
new_objs = []
for value in objects:
del_objs = extract_deletion(value)
if len(del_objs) > 0:
new_objs.append(*del_objs)
return DataFrame(new_objs)
def new_project(self, space, code, description=None, **kwargs):
return Project(
self, None, space=space, code=code, description=description, **kwargs
)
for option in options:
fo[option] = fetch_option[option]
return fo
def get_project(self, projectId, only_data=False, use_cache=True):
"""Returns a Project object for a given identifier, code or permId."""
project = (
not only_data
and use_cache
and self._object_cache(entity="project", code=projectId)
)
if project:
return project
options = ["space", "registrator", "modifier", "attachments"]
if is_identifier(projectId) or is_permid(projectId):
request = self._create_get_request(
"getProjects",
"project",
projectId,
options,
"as.dto.project.fetchoptions.ProjectFetchOptions",
)
resp = self._post_request(self.as_v3, request)
if only_data:
return resp[projectId]
project = Project(openbis_obj=self, type=None, data=resp[projectId])
if self.use_cache:
self._object_cache(entity="project", code=projectId, value=project)
return project
search_criteria = _gen_search_criteria(
{"project": "Project", "operator": "AND", "code": projectId}
)
options, foType="as.dto.project.fetchoptions.ProjectFetchOptions"
)
request = {
"method": "searchProjects",
}
resp = self._post_request(self.as_v3, request)
Swen Vermeul
committed
raise ValueError("No such project: %s" % projectId)
project = Project(openbis_obj=self, type=None, data=resp["objects"][0])
if self.use_cache:
self._object_cache(entity="project", code=projectId, value=project)
return project
self,
space=None,
code=None,
start_with=None,
count=None,
"""Get a list of all available projects (DataFrame object)."""
sub_criteria = []
if space:
sub_criteria.append(_subcriteria_for_code(space, "space"))
if code:
sub_criteria.append(_criteria_for_code(code))
criteria = {
"criteria": sub_criteria,
"@type": "as.dto.project.search.ProjectSearchCriteria",
fetchopts = {"@type": "as.dto.project.fetchoptions.ProjectFetchOptions"}
fetchopts["from"] = start_with
fetchopts["count"] = count
for option in ["registrator", "modifier", "leader"]:
request = {
"method": "searchProjects",
"params": [
self.token,
criteria,
fetchopts,
],
}
resp = self._post_request(self.as_v3, request)
vkovtun
committed
def create_data_frame(attrs, props, response):
attrs = [
"identifier",
"permId",
"leader",
"registrator",
"registrationDate",
"modifier",
"modificationDate",
]
objects = response["objects"]
if len(objects) == 0:
projects = DataFrame(columns=attrs)
else:
parse_jackson(objects)
vkovtun
committed
projects = DataFrame(objects)
vkovtun
committed
projects["registrationDate"] = projects["registrationDate"].map(
format_timestamp
)
projects["modificationDate"] = projects["modificationDate"].map(
format_timestamp
)
projects["leader"] = projects["leader"].map(extract_person)
projects["registrator"] = projects["registrator"].map(extract_person)
projects["modifier"] = projects["modifier"].map(extract_person)
projects["permId"] = projects["permId"].map(extract_permid)
projects["identifier"] = projects["identifier"].map(extract_identifier)
return projects[attrs]
entity="project",
identifier_name="identifier",
vkovtun
committed
response=resp,
df_initializer=create_data_frame,
def _create_get_request(self, method_name, entity, permids, options, foType):
if not isinstance(permids, list):
permids = [permids]
type = "as.dto.{}.id.{}".format(entity.lower(), entity.capitalize())
search_params = []
for permid in permids:
# decide if we got a permId or an identifier
{"identifier": permid, "@type": type + "Identifier"}
else:
search_params.append({"permId": permid, "@type": type + "PermId"})
for option in options:
fo[option] = fetch_option[option]
request = {
"method": method_name,
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
def clear_cache(self, entity=None):
"""Empty the internal object cache
If you do not specify any entity, the complete cache is cleared.
As entity, you can specify either:
space, project, vocabulary, term, sampleType, experimentType, dataSetType
"""
if entity:
self.cache[entity] = {}
else:
self.cache = {}
def _object_cache(self, entity=None, code=None, value=None):
# return the value, if no value provided
if value is None:
if entity in self.cache:
return self.cache[entity].get(code)
else:
if entity not in self.cache:
self.cache[entity] = {}
self.cache[entity][code] = value
def get_terms(self, vocabulary=None, start_with=None, count=None, use_cache=True):
"""Returns information about existing vocabulary terms.
If a vocabulary code is provided, it only returns the terms of that vocabulary.
use_cache
and self.use_cache
and vocabulary is not None
and start_with is None
and count is None
):
voc = self._object_cache(entity="term", code=vocabulary)
if voc:
return voc
search_request = {}
if vocabulary is not None:
search_request = _gen_search_criteria(
{
"vocabulary": "VocabularyTerm",
"criteria": [{"vocabulary": "Vocabulary", "code": vocabulary}],
}
)
search_request[
"@type"
] = "as.dto.vocabulary.search.VocabularyTermSearchCriteria"
fetchopts = fetch_option["vocabularyTerm"]