From f8e379be13aa9c23ec832bfb0466b93c18f30d30 Mon Sep 17 00:00:00 2001
From: alaskowski <alaskowski@ethz.ch>
Date: Mon, 27 Feb 2023 12:55:34 +0100
Subject: [PATCH] SSDM-13330: Added object set/get commands flow for physical
data management
---
.../src/python/obis/dm/commands/object.py | 105 ++++++++++++++++++
.../src/python/obis/dm/commands/search.py | 20 +++-
.../src/python/obis/dm/data_mgmt.py | 6 +-
.../src/python/obis/dm/utils.py | 23 +++-
4 files changed, 148 insertions(+), 6 deletions(-)
create mode 100644 app-openbis-command-line/src/python/obis/dm/commands/object.py
diff --git a/app-openbis-command-line/src/python/obis/dm/commands/object.py b/app-openbis-command-line/src/python/obis/dm/commands/object.py
new file mode 100644
index 00000000000..6d9785a3212
--- /dev/null
+++ b/app-openbis-command-line/src/python/obis/dm/commands/object.py
@@ -0,0 +1,105 @@
+# Copyright ETH 2023 Zürich, Scientific IT Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+
+from .openbis_command import OpenbisCommand
+from ..utils import is_valid_perm_id, OperationType
+from ...scripts.click_util import click_echo
+
+
+class Object(OpenbisCommand):
+ """
+ Command to operate on parent object of downloaded physical datasets.
+ """
+
+ def __init__(self, dm, operation_type, prop, value):
+ """
+ :param dm: data management
+ :param operation_type: type of operation to perform: get/set
+ :param prop: property to operate on
+ :param value: value to set for property prop
+ """
+ self.operation_type = operation_type
+ self.prop = prop
+ self.value = value
+ self.load_global_config(dm)
+ self.PERM_ID_DATE_FORMAT_PATTERN = "%Y%m%d%H%M%S%f"
+ super(Object, self).__init__(dm)
+
+ def run(self):
+ if self.operation_type is OperationType.GET:
+ return self.get()
+ if self.operation_type is OperationType.SET:
+ return self.set()
+ else:
+ click_echo(f"Operation {self.operation_type} is not supported!")
+ return -1
+
+ def get(self):
+ dataset_perm_ids = self.get_downloaded_datasets()
+ datasets = []
+ for perm_id in dataset_perm_ids:
+ ds = self.get_dataset(perm_id)
+ datasets += [ds] if ds is not None and ds.sample is not None else []
+ datasets = set(datasets)
+ for dataset in datasets:
+ sample = dataset.sample
+ click_echo(f"Object: {sample.permId} '{self.prop}' = {sample.props[self.prop]}")
+ return 0
+
+ def set(self):
+ dataset_perm_ids = self.get_downloaded_datasets()
+ datasets = []
+ for perm_id in dataset_perm_ids:
+ ds = self.get_dataset(perm_id)
+ datasets += [ds] if ds is not None and ds.sample is not None else []
+ datasets = set(datasets)
+ for dataset in datasets:
+ sample = dataset.sample
+ if self.prop == "parents":
+ sample.parents = self.empty_or_split()
+ click_echo(
+ f"Setting object: {sample.permId} parents to {self.empty_or_split()}")
+ elif self.prop == "children":
+ sample.children = self.empty_or_split()
+ click_echo(
+ f"Setting object: {sample.permId} children to {self.empty_or_split()}")
+ else:
+ sample.props[self.prop] = self.value
+ click_echo(
+ f"Setting object: {sample.permId} property '{self.prop}' to '{sample.props[self.prop]}'")
+ sample.save()
+ return 0
+
+ def empty_or_split(self):
+ if self.value == "":
+ return []
+ return self.value.split(',')
+
+ def get_downloaded_datasets(self):
+ result = set()
+ for root, dirs, files in os.walk(self.data_mgmt.invocation_path):
+ for dir_name in dirs:
+ if is_valid_perm_id(dir_name) is True:
+ result.add(dir_name)
+ return result
+
+ def get_dataset(self, perm_id):
+ try:
+ return self.openbis.get_dataset(perm_id, props="*")
+ except ValueError as e:
+ click_echo(f"Could not get dataset! {e}")
+ return None
diff --git a/app-openbis-command-line/src/python/obis/dm/commands/search.py b/app-openbis-command-line/src/python/obis/dm/commands/search.py
index 5c94e538e43..95b2cea3d33 100644
--- a/app-openbis-command-line/src/python/obis/dm/commands/search.py
+++ b/app-openbis-command-line/src/python/obis/dm/commands/search.py
@@ -92,12 +92,28 @@ class Search(OpenbisCommand):
where=properties,
props="*" # Fetch all properties
)
- click_echo(f"Number of objects matching criteria: {len(search_results)}")
+
+ collections = self.openbis.get_collections(
+ space=self.space,
+ project=self.project,
+ type=self.type_code,
+ where=properties,
+ props="*" # Fetch all properties
+ )
+
click_echo("Looking for data sets")
datasets = []
+ perm_ids = set()
for sample in search_results:
ds = sample.get_datasets()
- datasets += ds.objects
+ for ds_object in ds.objects:
+ datasets += [ds_object] if ds_object.permId not in perm_ids else []
+ perm_ids.add(ds_object.permId)
+ for collection in collections:
+ ds = collection.get_datasets()
+ for ds_object in ds.objects:
+ datasets += [ds_object] if ds_object.permId not in perm_ids else []
+ perm_ids.add(ds_object.permId)
click_echo(f"Data sets found: {len(datasets)}")
if self.save_path is not None:
diff --git a/app-openbis-command-line/src/python/obis/dm/data_mgmt.py b/app-openbis-command-line/src/python/obis/dm/data_mgmt.py
index bd729684a10..ee0d19bf848 100644
--- a/app-openbis-command-line/src/python/obis/dm/data_mgmt.py
+++ b/app-openbis-command-line/src/python/obis/dm/data_mgmt.py
@@ -31,6 +31,7 @@ from .commands.addref import Addref
from .commands.clone import Clone
from .commands.download_physical import DownloadPhysical
from .commands.move import Move
+from .commands.object import Object
from .commands.openbis_sync import OpenbisSync
from .commands.removeref import Removeref
from .commands.search import Search
@@ -683,7 +684,10 @@ class PhysicalDataMgmt(AbstractDataMgmt):
self.error_raise(f"{category} clear",
"This command is only available for External Manager Data")
- if category == "object" or category == "collection":
+ if category == "object":
+ cmd = Object(self, operation_type, prop, value)
+ return cmd.run()
+ elif category == "collection":
click_echo("Not yet implemented.")
return 0
else:
diff --git a/app-openbis-command-line/src/python/obis/dm/utils.py b/app-openbis-command-line/src/python/obis/dm/utils.py
index f5e6ccb11f8..580e7e2cbcc 100644
--- a/app-openbis-command-line/src/python/obis/dm/utils.py
+++ b/app-openbis-command-line/src/python/obis/dm/utils.py
@@ -15,6 +15,7 @@
import os
import subprocess
from contextlib import contextmanager
+from datetime import datetime
from enum import Enum
from .command_result import CommandResult, CommandException
@@ -45,8 +46,10 @@ def complete_openbis_config(config, resolver, local_only=True):
config['token'] = None
if config.get('is_physical') is None:
config['is_physical'] = None
- if config.get('allow_http_but_do_not_use_this_in_production_and_only_within_safe_networks') is None:
- config['allow_http_but_do_not_use_this_in_production_and_only_within_safe_networks'] = not config_dict['allow_only_https']
+ if config.get(
+ 'allow_http_but_do_not_use_this_in_production_and_only_within_safe_networks') is None:
+ config['allow_http_but_do_not_use_this_in_production_and_only_within_safe_networks'] = not \
+ config_dict['allow_only_https']
def complete_git_config(config):
@@ -69,7 +72,9 @@ def default_echo(details):
def run_shell(args, shell=False, strip_leading_whitespace=True, raise_exception_on_failure=False):
- result = CommandResult(subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell), strip_leading_whitespace=strip_leading_whitespace)
+ result = CommandResult(
+ subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell),
+ strip_leading_whitespace=strip_leading_whitespace)
if raise_exception_on_failure == True and result.failure():
raise CommandException(result)
return result
@@ -94,3 +99,15 @@ def cd(newdir):
yield
finally:
os.chdir(prevdir)
+
+
+def is_valid_perm_id(name):
+ if "-" not in name:
+ return False
+ split = name.split("-")
+ try:
+ datetime.strptime(split[0], "%Y%m%d%H%M%S%f")
+ int(split[1])
+ return True
+ except ValueError:
+ return False
--
GitLab