From c042be9fc1535c9e187c358a52c40f859c713c42 Mon Sep 17 00:00:00 2001
From: alaskowski <alaskowski@ethz.ch>
Date: Fri, 24 Feb 2023 11:14:15 +0100
Subject: [PATCH] SSDM-13330: Added searching datasets flow for PHYSICAL data.
Refactored code.
---
.../src/python/obis/dm/commands/search.py | 56 +++++++++++++++--
.../src/python/obis/dm/data_mgmt.py | 35 +++++++++--
.../src/python/obis/scripts/cli.py | 60 ++++++++++++++-----
3 files changed, 127 insertions(+), 24 deletions(-)
diff --git a/app-openbis-command-line/src/python/obis/dm/commands/search.py b/app-openbis-command-line/src/python/obis/dm/commands/search.py
index 0fc3faa76e3..5c94e538e43 100644
--- a/app-openbis-command-line/src/python/obis/dm/commands/search.py
+++ b/app-openbis-command-line/src/python/obis/dm/commands/search.py
@@ -13,6 +13,8 @@
# limitations under the License.
#
+import os
+
from .openbis_command import OpenbisCommand
from ..command_result import CommandResult
from ..utils import cd
@@ -21,7 +23,7 @@ from ...scripts.click_util import click_echo
class Search(OpenbisCommand):
"""
- Command to search objects in openBIS.
+ Command to search data in openBIS.
"""
def __init__(self, dm, type_code, space, project, experiment, property_code, property_value,
@@ -61,12 +63,58 @@ class Search(OpenbisCommand):
where=properties,
props="*" # Fetch all properties
)
- click_echo("Search found {} samples".format(len(search_results)))
+ click_echo(f"Objects found: {len(search_results)}")
if self.save_path is not None:
- click_echo("Saving search results in {}".format(self.save_path), with_timestamp=True)
+ click_echo(f"Saving search results in {self.save_path}")
with cd(self.data_mgmt.invocation_path):
search_results.df.to_csv(self.save_path, index=False)
else:
- click_echo("Search results: %s" % search_results)
+ click_echo(f"Search results:\n{search_results}")
+
+ return CommandResult(returncode=0, output="Search completed.")
+
+ def search_data_sets(self):
+ if self.save_path is not None and self.fileservice_url() is None:
+ return CommandResult(returncode=-1,
+ output="Configuration fileservice_url needs to be set for download.")
+
+ properties = None
+ if self.property_code is not None and self.property_value is not None:
+ properties = {
+ self.property_code: self.property_value,
+ }
+
+ search_results = self.openbis.get_samples(
+ space=self.space,
+ project=self.project, # Not Supported with Project Samples disabled
+ experiment=self.experiment,
+ type=self.type_code,
+ where=properties,
+ props="*" # Fetch all properties
+ )
+ click_echo(f"Number of objects matching criteria: {len(search_results)}")
+ click_echo("Looking for data sets")
+ datasets = []
+ for sample in search_results:
+ ds = sample.get_datasets()
+ datasets += ds.objects
+
+ click_echo(f"Data sets found: {len(datasets)}")
+ if self.save_path is not None:
+ with cd(self.data_mgmt.invocation_path):
+ if os.path.exists(self.save_path) is True and os.path.isdir(
+ self.save_path) is False:
+ return CommandResult(returncode=-1,
+ output=f"File {self.save_path} is not a directory")
+ if os.path.isdir(self.save_path) is False:
+ click_echo(f"Creating directory {self.save_path}")
+ os.makedirs(self.save_path)
+ click_echo(
+ f"Saving search results in {os.path.join(self.data_mgmt.invocation_path, self.save_path)}")
+ for dataset in datasets:
+ dataset.download(destination=self.save_path,
+ linked_dataset_fileservice_url=self.fileservice_url() + "/download")
+ else:
+ click_echo(f"Search results:\n{datasets}")
return CommandResult(returncode=0, output="Search completed.")
diff --git a/app-openbis-command-line/src/python/obis/dm/data_mgmt.py b/app-openbis-command-line/src/python/obis/dm/data_mgmt.py
index a6a26f148a3..93ed3746e1c 100644
--- a/app-openbis-command-line/src/python/obis/dm/data_mgmt.py
+++ b/app-openbis-command-line/src/python/obis/dm/data_mgmt.py
@@ -218,7 +218,8 @@ class AbstractDataMgmt(metaclass=abc.ABCMeta):
return
@abc.abstractmethod
- def search(self, type_code, space, project, experiment, property_code, property_value, save):
+ def search_object(self, type_code, space, project, experiment, property_code, property_value,
+ save):
"""Search for objects in openBIS using filtering criteria.
:param type_code: Type of searched object.
:param space: Space path to filter object.
@@ -230,6 +231,19 @@ class AbstractDataMgmt(metaclass=abc.ABCMeta):
"""
return
+ def search_data_set(self, type_code, space, project, experiment, property_code, property_value,
+ save):
+ """Search for datasets in openBIS using filtering criteria.
+ :param type_code: Type of searched object.
+ :param space: Space path to filter object.
+ :param project: Project path to filter object.
+ :param experiment: Experiment path to filter object.
+ :param property_code: Custom property code to search by, property_value must be set as well.
+ :param property_value: Custom property value to search by, property_code must be set as well.
+ :param save: File path to save results. If missing, search results will not be saved.
+ """
+ return
+
class NoGitDataMgmt(AbstractDataMgmt):
"""DataMgmt operations when git is not available -- show error messages."""
@@ -270,7 +284,10 @@ class NoGitDataMgmt(AbstractDataMgmt):
def download(self, data_set_id, content_copy_index, file, skip_integrity_check):
self.error_raise("download", "No git command found.")
- def search(self, *_):
+ def search_object(self, *_):
+ self.error_raise("search", "No git command found.")
+
+ def search_data_set(self, *_):
self.error_raise("search", "No git command found.")
@@ -552,7 +569,10 @@ class GitDataMgmt(AbstractDataMgmt):
else:
return CommandResult(returncode=0, output="")
- def search(self, *_):
+ def search_object(self, *_):
+ self.error_raise("search", "This functionality is not implemented for data of LINK type.")
+
+ def search_data_set(self, *_):
self.error_raise("search", "This functionality is not implemented for data of LINK type.")
@@ -604,7 +624,14 @@ class PhysicalDataMgmt(AbstractDataMgmt):
cmd = DownloadPhysical(self, data_set_id, file)
return cmd.run()
- def search(self, type_code, space, project, experiment, property_code, property_value, save):
+ def search_object(self, type_code, space, project, experiment, property_code, property_value,
+ save):
cmd = Search(self, type_code, space, project, experiment, property_code, property_value,
save)
return cmd.search_samples()
+
+ def search_data_set(self, type_code, space, project, experiment, property_code, property_value,
+ save):
+ cmd = Search(self, type_code, space, project, experiment, property_code, property_value,
+ save)
+ return cmd.search_data_sets()
diff --git a/app-openbis-command-line/src/python/obis/scripts/cli.py b/app-openbis-command-line/src/python/obis/scripts/cli.py
index 877d7827912..b081071c8e1 100644
--- a/app-openbis-command-line/src/python/obis/scripts/cli.py
+++ b/app-openbis-command-line/src/python/obis/scripts/cli.py
@@ -268,6 +268,18 @@ def repository_clear(ctx, settings):
# data_set: type, properties
+_search_params = [
+ click.option('-type', '--type', 'type_code', default=None, help='Type code to filter by'),
+ click.option('-space', '--space', default=None, help='Space code'),
+ click.option('-project', '--project', default=None, help='Full project identification code'),
+ click.option('-experiment', '--experiment', default=None, help='Full experiment code'),
+ click.option('-property', '--property', 'property_code', default=None, help='Property code'),
+ click.option('-property-value', '--property-value', 'property_value', default=None,
+ help='Property value'),
+ click.option('-save', '--save', default=None, help='Filename to save results'),
+]
+
+
@cli.group('data_set')
@click.option('-g', '--is_global', default=False, is_flag=True, help='Set/get global or local.')
@click.option('-p', '--is_data_set_property', default=False, is_flag=True,
@@ -304,6 +316,35 @@ def data_set_clear(ctx, data_set_settings):
return ctx.obj['runner'].run("data_set_clear", lambda dm: _clear(ctx, data_set_settings))
+@data_set.command('search', short_help="Search for datasets using a filtering criteria.")
+@add_params(_search_params)
+@click.pass_context
+def data_set_search(ctx, type_code, space, project, experiment, property_code, property_value,
+ save):
+ if all(v is None for v in
+ [type_code, space, project, experiment, property_code, property_value]):
+ click_echo("You must provide at least one filtering criteria!")
+ return -1
+ if (property_code is None and property_value is not None) or (
+ property_code is not None and property_value is None):
+ click_echo("Property code and property value need to be specified!")
+ return -1
+ ctx.obj['runner'] = DataMgmtRunner(ctx.obj, halt_on_error_log=False)
+ ctx.invoke(_data_set_search, type_code=type_code, space=space,
+ project=project, experiment=experiment, property_code=property_code,
+ property_value=property_value, save=save)
+
+
+@add_params(_search_params)
+@click.pass_context
+def _data_set_search(ctx, type_code, space, project, experiment, property_code, property_value,
+ save):
+ return ctx.obj['runner'].run("data_set_search",
+ lambda dm: dm.search_data_set(type_code, space, project,
+ experiment, property_code,
+ property_value, save)),
+
+
# # object: object_id
@@ -340,19 +381,7 @@ def object_clear(ctx, object_settings):
return ctx.obj['runner'].run("object_clear", lambda dm: _clear(ctx, object_settings))
-_search_params = [
- click.option('-type', '--type', 'type_code', default=None, help='Type code to filter by'),
- click.option('-space', '--space', default=None, help='Space code'),
- click.option('-project', '--project', default=None, help='Full project identification code'),
- click.option('-experiment', '--experiment', default=None, help='Full experiment code'),
- click.option('-property', '--property', 'property_code', default=None, help='Property code'),
- click.option('-property-value', '--property-value', 'property_value', default=None,
- help='Property value'),
- click.option('-save', '--save', default=None, help='Filename to save results'),
-]
-
-
-@cli.command(short_help="Download files of a linked data set.")
+@object.command('search', short_help="Search for samples using a filtering criteria.")
@add_params(_search_params)
@click.pass_context
def object_search(ctx, type_code, space, project, experiment, property_code, property_value, save):
@@ -370,13 +399,12 @@ def object_search(ctx, type_code, space, project, experiment, property_code, pro
property_value=property_value, save=save)
-@object.command('search')
@add_params(_search_params)
@click.pass_context
def _object_search(ctx, type_code, space, project, experiment, property_code, property_value, save):
return ctx.obj['runner'].run("object_search",
- lambda dm: dm.search(type_code, space, project, experiment,
- property_code, property_value, save)),
+ lambda dm: dm.search_object(type_code, space, project, experiment,
+ property_code, property_value, save)),
# # collection: collection_id
--
GitLab