From 030caf8ac9286b34dc8579dd302fd11ebb007d3d Mon Sep 17 00:00:00 2001
From: alaskowski <alaskowski@ethz.ch>
Date: Thu, 23 Feb 2023 16:00:14 +0100
Subject: [PATCH] SSDM-13330: Added searching samples flow for PHYSICAL data.
Refactored code.
---
.../src/python/obis/dm/commands/search.py | 72 ++++++++++
.../src/python/obis/dm/data_mgmt.py | 46 +++++--
.../src/python/obis/scripts/cli.py | 126 +++++++++++++-----
3 files changed, 200 insertions(+), 44 deletions(-)
create mode 100644 app-openbis-command-line/src/python/obis/dm/commands/search.py
diff --git a/app-openbis-command-line/src/python/obis/dm/commands/search.py b/app-openbis-command-line/src/python/obis/dm/commands/search.py
new file mode 100644
index 00000000000..0fc3faa76e3
--- /dev/null
+++ b/app-openbis-command-line/src/python/obis/dm/commands/search.py
@@ -0,0 +1,72 @@
+# Copyright ETH 2023 Zürich, Scientific IT Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from .openbis_command import OpenbisCommand
+from ..command_result import CommandResult
+from ..utils import cd
+from ...scripts.click_util import click_echo
+
+
+class Search(OpenbisCommand):
+ """
+ Command to search objects in openBIS.
+ """
+
+ def __init__(self, dm, type_code, space, project, experiment, property_code, property_value,
+ save_path):
+ """
+ :param dm: data management
+ :param type_code: Filter by type code
+ :param space: Filter by space path
+ :param project: Filter by project path
+ :param experiment: Filter by experiment
+ :param property_code: Filter by property_code, needs to be set together with property_value
+ :param property_value: Filter by property_value, needs to be set together with property_code
+ :param save_path: Path to save results. If not set, results will not be saved.
+ """
+ self.property_value = property_value
+ self.property_code = property_code
+ self.experiment = experiment
+ self.project = project
+ self.space = space
+ self.type_code = type_code
+ self.save_path = save_path
+ self.load_global_config(dm)
+ super(Search, self).__init__(dm)
+
+ def search_samples(self):
+ properties = None
+ if self.property_code is not None and self.property_value is not None:
+ properties = {
+ self.property_code: self.property_value,
+ }
+
+ search_results = self.openbis.get_samples(
+ space=self.space,
+ project=self.project, # Not Supported with Project Samples disabled
+ experiment=self.experiment,
+ type=self.type_code,
+ where=properties,
+ props="*" # Fetch all properties
+ )
+ click_echo("Search found {} samples".format(len(search_results)))
+ if self.save_path is not None:
+ click_echo("Saving search results in {}".format(self.save_path), with_timestamp=True)
+ with cd(self.data_mgmt.invocation_path):
+ search_results.df.to_csv(self.save_path, index=False)
+ else:
+ click_echo("Search results: %s" % search_results)
+
+ return CommandResult(returncode=0, output="Search completed.")
diff --git a/app-openbis-command-line/src/python/obis/dm/data_mgmt.py b/app-openbis-command-line/src/python/obis/dm/data_mgmt.py
index ab26c845343..a6a26f148a3 100644
--- a/app-openbis-command-line/src/python/obis/dm/data_mgmt.py
+++ b/app-openbis-command-line/src/python/obis/dm/data_mgmt.py
@@ -34,6 +34,7 @@ from .commands.download_physical import DownloadPhysical
from .commands.move import Move
from .commands.openbis_sync import OpenbisSync
from .commands.removeref import Removeref
+from .commands.search import Search
from .git import GitWrapper
from .utils import Type
from .utils import cd
@@ -216,6 +217,19 @@ class AbstractDataMgmt(metaclass=abc.ABCMeta):
"""
return
+ @abc.abstractmethod
+ def search(self, type_code, space, project, experiment, property_code, property_value, save):
+ """Search for objects in openBIS using filtering criteria.
+ :param type_code: Type of searched object.
+ :param space: Space path to filter object.
+ :param project: Project path to filter object.
+ :param experiment: Experiment path to filter object.
+ :param property_code: Custom property code to search by, property_value must be set as well.
+ :param property_value: Custom property value to search by, property_code must be set as well.
+ :param save: File path to save results. If missing, search results will not be saved.
+ """
+ return
+
class NoGitDataMgmt(AbstractDataMgmt):
"""DataMgmt operations when git is not available -- show error messages."""
@@ -256,6 +270,9 @@ class NoGitDataMgmt(AbstractDataMgmt):
def download(self, data_set_id, content_copy_index, file, skip_integrity_check):
self.error_raise("download", "No git command found.")
+ def search(self, *_):
+ self.error_raise("search", "No git command found.")
+
def restore_signal_handler(data_mgmt):
data_mgmt.restore()
@@ -332,8 +349,7 @@ class GitDataMgmt(AbstractDataMgmt):
def init_data(self, desc=None):
# check that repository does not already exist
- # TODO remove .git check after physical flow is implemented
- if os.path.exists('.obis') and os.path.exists('.git'):
+ if os.path.exists('.obis'):
return CommandResult(returncode=-1, output="Folder is already an obis repository.")
result = self.git_wrapper.git_init()
if result.failure():
@@ -536,6 +552,9 @@ class GitDataMgmt(AbstractDataMgmt):
else:
return CommandResult(returncode=0, output="")
+ def search(self, *_):
+ self.error_raise("search", "This functionality is not implemented for data of LINK type.")
+
class PhysicalDataMgmt(AbstractDataMgmt):
"""DataMgmt operations for DSS-stored data."""
@@ -544,7 +563,7 @@ class PhysicalDataMgmt(AbstractDataMgmt):
return dm_config.SettingsResolver()
def setup_local_settings(self, all_settings):
- self.error_raise("setup local settings", "Not implemented.")
+ self.error_raise("setup local settings", "Not implemented for PHYSICAL data.")
def init_data(self, desc=None):
if os.path.exists('.obis'):
@@ -558,29 +577,34 @@ class PhysicalDataMgmt(AbstractDataMgmt):
return CommandResult(returncode=0, output="Physical obis repository initialized.")
def init_analysis(self, parent_folder, desc=None):
- self.error_raise("init analysis", "Not implemented.")
+ self.error_raise("init analysis", "Not implemented for PHYSICAL data.")
def commit(self, msg, auto_add=True, sync=True):
- self.error_raise("commit", "Not implemented.")
+ self.error_raise("commit", "Not implemented for PHYSICAL data.")
def sync(self):
- self.error_raise("sync", "Not implemented.")
+ self.error_raise("sync", "Not implemented for PHYSICAL data.")
def status(self):
- self.error_raise("status", "Not implemented.")
+ self.error_raise("status", "Not implemented for PHYSICAL data.")
def clone(self, data_set_id, ssh_user, content_copy_index, skip_integrity_check):
- self.error_raise("clone", "Not implemented.")
+ self.error_raise("clone", "Not implemented for PHYSICAL data.")
def move(self, data_set_id, ssh_user, content_copy_index, skip_integrity_check):
- self.error_raise("move", "Not implemented.")
+ self.error_raise("move", "Not implemented for PHYSICAL data.")
def addref(self):
- self.error_raise("addref", "Not implemented.")
+ self.error_raise("addref", "Not implemented for PHYSICAL data.")
def removeref(self, data_set_id=None):
- self.error_raise("removeref", "Not implemented.")
+ self.error_raise("removeref", "Not implemented for PHYSICAL data.")
def download(self, data_set_id, _content_copy_index, file, _skip_integrity_check):
cmd = DownloadPhysical(self, data_set_id, file)
return cmd.run()
+
+ def search(self, type_code, space, project, experiment, property_code, property_value, save):
+ cmd = Search(self, type_code, space, project, experiment, property_code, property_value,
+ save)
+ return cmd.search_samples()
diff --git a/app-openbis-command-line/src/python/obis/scripts/cli.py b/app-openbis-command-line/src/python/obis/scripts/cli.py
index 3fc36956e21..877d7827912 100644
--- a/app-openbis-command-line/src/python/obis/scripts/cli.py
+++ b/app-openbis-command-line/src/python/obis/scripts/cli.py
@@ -50,13 +50,15 @@ def add_params(params):
for param in reversed(params):
func = param(func)
return func
+
return _add_params
@click.group()
@click.version_option(version=None)
@click.option('-q', '--quiet', default=False, is_flag=True, help='Suppress status reporting.')
-@click.option('-s', '--skip_verification', default=False, is_flag=True, help='Do not verify cerficiates')
+@click.option('-s', '--skip_verification', default=False, is_flag=True,
+ help='Do not verify cerficiates')
@click.option('-d', '--debug', default=False, is_flag=True, help="Show stack trace on error.")
@click.pass_context
def cli(ctx, quiet, skip_verification, debug):
@@ -88,7 +90,8 @@ def init_analysis_impl(ctx, parent, repository, description):
analysis_dir = os.path.join(os.getcwd(), repository)
parent = os.path.relpath(parent_dir, analysis_dir)
parent = '..' if parent is None else parent
- return ctx.obj['runner'].run("init_analysis", lambda dm: dm.init_analysis(parent, description), repository)
+ return ctx.obj['runner'].run("init_analysis", lambda dm: dm.init_analysis(parent, description),
+ repository)
# settings commands
@@ -267,7 +270,8 @@ def repository_clear(ctx, settings):
@cli.group('data_set')
@click.option('-g', '--is_global', default=False, is_flag=True, help='Set/get global or local.')
-@click.option('-p', '--is_data_set_property', default=False, is_flag=True, help='Configure data set property.')
+@click.option('-p', '--is_data_set_property', default=False, is_flag=True,
+ help='Configure data set property.')
@click.pass_context
def data_set(ctx, is_global, is_data_set_property):
""" Get/set settings related to the data set.
@@ -280,27 +284,27 @@ def data_set(ctx, is_global, is_data_set_property):
@data_set.command('set')
-@click.argument('settings', type=SettingsSet(), nargs=-1)
+@click.argument('data_set_settings', type=SettingsSet(), nargs=-1)
@click.pass_context
-def data_set_set(ctx, settings):
- return ctx.obj['runner'].run("data_set_set", lambda dm: _set(ctx, settings))
+def data_set_set(ctx, data_set_settings):
+ return ctx.obj['runner'].run("data_set_set", lambda dm: _set(ctx, data_set_settings))
@data_set.command('get')
-@click.argument('settings', type=SettingsGet(), nargs=-1)
+@click.argument('data_set_settings', type=SettingsGet(), nargs=-1)
@click.pass_context
-def data_set_get(ctx, settings):
- return ctx.obj['runner'].run("data_set_get", lambda dm: _get(ctx, settings))
+def data_set_get(ctx, data_set_settings):
+ return ctx.obj['runner'].run("data_set_get", lambda dm: _get(ctx, data_set_settings))
@data_set.command('clear')
-@click.argument('settings', type=SettingsClear(), nargs=-1)
+@click.argument('data_set_settings', type=SettingsClear(), nargs=-1)
@click.pass_context
-def data_set_clear(ctx, settings):
- return ctx.obj['runner'].run("data_set_clear", lambda dm: _clear(ctx, settings))
+def data_set_clear(ctx, data_set_settings):
+ return ctx.obj['runner'].run("data_set_clear", lambda dm: _clear(ctx, data_set_settings))
-## object: object_id
+# # object: object_id
@cli.group()
@@ -316,27 +320,66 @@ def object(ctx, is_global):
@object.command('set')
-@click.argument('settings', type=SettingsSet(), nargs=-1)
+@click.argument('object_settings', type=SettingsSet(), nargs=-1)
@click.pass_context
-def object_set(ctx, settings):
- return ctx.obj['runner'].run("object_set", lambda dm: _set(ctx, settings))
+def object_set(ctx, object_settings):
+ return ctx.obj['runner'].run("object_set", lambda dm: _set(ctx, object_settings))
@object.command('get')
-@click.argument('settings', type=SettingsGet(), nargs=-1)
+@click.argument('object_settings', type=SettingsGet(), nargs=-1)
@click.pass_context
-def object_get(ctx, settings):
- return ctx.obj['runner'].run("object_get", lambda dm: _get(ctx, settings))
+def object_get(ctx, object_settings):
+ return ctx.obj['runner'].run("object_get", lambda dm: _get(ctx, object_settings))
@object.command('clear')
-@click.argument('settings', type=SettingsClear(), nargs=-1)
+@click.argument('object_settings', type=SettingsClear(), nargs=-1)
+@click.pass_context
+def object_clear(ctx, object_settings):
+ return ctx.obj['runner'].run("object_clear", lambda dm: _clear(ctx, object_settings))
+
+
+_search_params = [
+ click.option('-type', '--type', 'type_code', default=None, help='Type code to filter by'),
+ click.option('-space', '--space', default=None, help='Space code'),
+ click.option('-project', '--project', default=None, help='Full project identification code'),
+ click.option('-experiment', '--experiment', default=None, help='Full experiment code'),
+ click.option('-property', '--property', 'property_code', default=None, help='Property code'),
+ click.option('-property-value', '--property-value', 'property_value', default=None,
+ help='Property value'),
+ click.option('-save', '--save', default=None, help='Filename to save results'),
+]
+
+
+@cli.command(short_help="Download files of a linked data set.")
+@add_params(_search_params)
+@click.pass_context
+def object_search(ctx, type_code, space, project, experiment, property_code, property_value, save):
+ if all(v is None for v in
+ [type_code, space, project, experiment, property_code, property_value]):
+ click_echo("You must provide at least one filtering criteria!")
+ return -1
+ if (property_code is None and property_value is not None) or (
+ property_code is not None and property_value is None):
+ click_echo("Property code and property value need to be specified!")
+ return -1
+ ctx.obj['runner'] = DataMgmtRunner(ctx.obj, halt_on_error_log=False)
+ ctx.invoke(_object_search, type_code=type_code, space=space,
+ project=project, experiment=experiment, property_code=property_code,
+ property_value=property_value, save=save)
+
+
+@object.command('search')
+@add_params(_search_params)
@click.pass_context
-def object_clear(ctx, settings):
- return ctx.obj['runner'].run("object_clear", lambda dm: _clear(ctx, settings))
+def _object_search(ctx, type_code, space, project, experiment, property_code, property_value, save):
+ return ctx.obj['runner'].run("object_search",
+ lambda dm: dm.search(type_code, space, project, experiment,
+ property_code, property_value, save)),
-## collection: collection_id
+# # collection: collection_id
@cli.group()
@@ -428,7 +471,9 @@ _commit_params = [
@click.pass_context
@add_params(_commit_params)
def repository_commit(ctx, msg, auto_add, ignore_missing_parent, repository):
- return ctx.obj['runner'].run("commit", lambda dm: dm.commit(msg, auto_add, ignore_missing_parent), repository)
+ return ctx.obj['runner'].run("commit",
+ lambda dm: dm.commit(msg, auto_add, ignore_missing_parent),
+ repository)
@cli.command(short_help="Commit the repository to git and inform openBIS.")
@@ -439,6 +484,7 @@ def commit(ctx, msg, auto_add, ignore_missing_parent, repository):
ctx.invoke(repository_commit, msg=msg, auto_add=auto_add,
ignore_missing_parent=ignore_missing_parent, repository=repository)
+
# init
@@ -470,6 +516,7 @@ def init(ctx, repository_path, description, is_physical):
ctx.obj['runner'] = DataMgmtRunner(ctx.obj, halt_on_error_log=False, is_physical=is_physical)
ctx.invoke(repository_init, repository_path=repository_path, description=description)
+
# init analysis
@@ -495,6 +542,7 @@ def init_analysis(ctx, parent, repository_path, description):
ctx.invoke(repository_init_analysis, parent=parent,
repository_path=repository_path, description=description)
+
# status
@@ -518,6 +566,7 @@ def status(ctx, repository):
ctx.obj['runner'] = DataMgmtRunner(ctx.obj, halt_on_error_log=False)
ctx.invoke(repository_status, repository=repository)
+
# sync
@@ -538,7 +587,8 @@ def _repository_sync(dm, ignore_missing_parent):
@click.pass_context
@add_params(_sync_params)
def repository_sync(ctx, ignore_missing_parent, repository):
- return ctx.obj['runner'].run("sync", lambda dm: _repository_sync(dm, ignore_missing_parent), repository)
+ return ctx.obj['runner'].run("sync", lambda dm: _repository_sync(dm, ignore_missing_parent),
+ repository)
@cli.command(short_help="Sync the repository with openBIS.")
@@ -591,13 +641,13 @@ def new_token(ctx, session_name=None, **kwargs):
validFrom = datetime.now()
if kwargs.get("validity_months"):
validTo = validFrom + \
- relativedelta(months=int(kwargs.get("validity_months")))
+ relativedelta(months=int(kwargs.get("validity_months")))
elif kwargs.get("validity_weeks"):
validTo = validFrom + \
- relativedelta(weeks=int(kwargs.get("validity_weeks")))
+ relativedelta(weeks=int(kwargs.get("validity_weeks")))
elif kwargs.get("validity_days"):
validTo = validFrom + \
- relativedelta(days=int(kwargs.get("validity_days")))
+ relativedelta(days=int(kwargs.get("validity_days")))
else:
serverinfo = o.get_server_information()
seconds = serverinfo.personal_access_tokens_max_validity_period
@@ -637,6 +687,7 @@ def addref(ctx, repository):
ctx.obj['runner'] = DataMgmtRunner(ctx.obj, halt_on_error_log=False)
ctx.invoke(repository_addref, repository=repository)
+
# removeref
@@ -648,14 +699,16 @@ _removeref_params = [
]
-@repository.command("removeref", short_help="Remove the reference to the given repository from openBIS.")
+@repository.command("removeref",
+ short_help="Remove the reference to the given repository from openBIS.")
@click.pass_context
@add_params(_removeref_params)
def repository_removeref(ctx, data_set_id, repository):
if data_set_id is not None and repository is not None:
click_echo("Only provide the data_set id OR the repository.")
return -1
- return ctx.obj['runner'].run("removeref", lambda dm: dm.removeref(data_set_id=data_set_id), repository)
+ return ctx.obj['runner'].run("removeref", lambda dm: dm.removeref(data_set_id=data_set_id),
+ repository)
@cli.command(short_help="Remove the reference to the given repository from openBIS.")
@@ -686,7 +739,9 @@ _download_params = [
@add_params(_download_params)
@click.pass_context
def data_set_download(ctx, content_copy_index, file, data_set_id, skip_integrity_check):
- return ctx.obj['runner'].run("download", lambda dm: dm.download(data_set_id, content_copy_index, file, skip_integrity_check))
+ return ctx.obj['runner'].run("download",
+ lambda dm: dm.download(data_set_id, content_copy_index, file,
+ skip_integrity_check))
@cli.command(short_help="Download files of a linked data set.")
@@ -697,6 +752,7 @@ def download(ctx, content_copy_index, file, data_set_id, skip_integrity_check):
ctx.invoke(data_set_download, content_copy_index=content_copy_index, file=file,
data_set_id=data_set_id, skip_integrity_check=skip_integrity_check)
+
# clone
@@ -715,7 +771,9 @@ _clone_move_params = [
@click.pass_context
@add_params(_clone_move_params)
def data_set_clone(ctx, ssh_user, content_copy_index, data_set_id, skip_integrity_check):
- return ctx.obj['runner'].run("clone", lambda dm: dm.clone(data_set_id, ssh_user, content_copy_index, skip_integrity_check))
+ return ctx.obj['runner'].run("clone",
+ lambda dm: dm.clone(data_set_id, ssh_user, content_copy_index,
+ skip_integrity_check))
@cli.command(short_help="Clone the repository found in the given data set id.")
@@ -733,7 +791,9 @@ def clone(ctx, ssh_user, content_copy_index, data_set_id, skip_integrity_check):
@click.pass_context
@add_params(_clone_move_params)
def data_set_move(ctx, ssh_user, content_copy_index, data_set_id, skip_integrity_check):
- return ctx.obj['runner'].run("move", lambda dm: dm.move(data_set_id, ssh_user, content_copy_index, skip_integrity_check))
+ return ctx.obj['runner'].run("move",
+ lambda dm: dm.move(data_set_id, ssh_user, content_copy_index,
+ skip_integrity_check))
@cli.command(short_help="Move the repository found in the given data set id.")
--
GitLab