SSDM-13330: Added object set/get commands flow for physical data management

app-openbis-command-line/src/python/obis/dm/commands/object.py

+#   Copyright ETH 2023 Zürich, Scientific IT Services
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+import os
+
+from .openbis_command import OpenbisCommand
+from ..utils import is_valid_perm_id, OperationType
+from ...scripts.click_util import click_echo
+
+
+class Object(OpenbisCommand):
+    """
+    Command to operate on parent object of downloaded physical datasets.
+    """
+
+    def __init__(self, dm, operation_type, prop, value):
+        """
+        :param dm: data management
+        :param operation_type: type of operation to perform: get/set
+        :param prop: property to operate on
+        :param value: value to set for property prop
+        """
+        self.operation_type = operation_type
+        self.prop = prop
+        self.value = value
+        self.load_global_config(dm)
+        self.PERM_ID_DATE_FORMAT_PATTERN = "%Y%m%d%H%M%S%f"
+        super(Object, self).__init__(dm)
+
+    def run(self):
+        if self.operation_type is OperationType.GET:
+            return self.get()
+        if self.operation_type is OperationType.SET:
+            return self.set()
+        else:
+            click_echo(f"Operation {self.operation_type} is not supported!")
+            return -1
+
+    def get(self):
+        dataset_perm_ids = self.get_downloaded_datasets()
+        datasets = []
+        for perm_id in dataset_perm_ids:
+            ds = self.get_dataset(perm_id)
+            datasets += [ds] if ds is not None and ds.sample is not None else []
+        datasets = set(datasets)
+        for dataset in datasets:
+            sample = dataset.sample
+            click_echo(f"Object: {sample.permId} '{self.prop}' = {sample.props[self.prop]}")
+        return 0
+
+    def set(self):
+        dataset_perm_ids = self.get_downloaded_datasets()
+        datasets = []
+        for perm_id in dataset_perm_ids:
+            ds = self.get_dataset(perm_id)
+            datasets += [ds] if ds is not None and ds.sample is not None else []
+        datasets = set(datasets)
+        for dataset in datasets:
+            sample = dataset.sample
+            if self.prop == "parents":
+                sample.parents = self.empty_or_split()
+                click_echo(
+                    f"Setting object: {sample.permId} parents to {self.empty_or_split()}")
+            elif self.prop == "children":
+                sample.children = self.empty_or_split()
+                click_echo(
+                    f"Setting object: {sample.permId} children to {self.empty_or_split()}")
+            else:
+                sample.props[self.prop] = self.value
+                click_echo(
+                    f"Setting object: {sample.permId} property '{self.prop}' to '{sample.props[self.prop]}'")
+            sample.save()
+        return 0
+
+    def empty_or_split(self):
+        if self.value == "":
+            return []
+        return self.value.split(',')
+
+    def get_downloaded_datasets(self):
+        result = set()
+        for root, dirs, files in os.walk(self.data_mgmt.invocation_path):
+            for dir_name in dirs:
+                if is_valid_perm_id(dir_name) is True:
+                    result.add(dir_name)
+        return result
+
+    def get_dataset(self, perm_id):
+        try:
+            return self.openbis.get_dataset(perm_id, props="*")
+        except ValueError as e:
+            click_echo(f"Could not get dataset! {e}")
+            return None

app-openbis-command-line/src/python/obis/dm/commands/search.py

@@ -92,12 +92,28 @@ class Search(OpenbisCommand):
             where=properties,
             props="*"  # Fetch all properties
         )
-        click_echo(f"Number of objects matching criteria: {len(search_results)}")
+
+        collections = self.openbis.get_collections(
+            space=self.space,
+            project=self.project,
+            type=self.type_code,
+            where=properties,
+            props="*"  # Fetch all properties
+        )
+
         click_echo("Looking for data sets")
         datasets = []
+        perm_ids = set()
         for sample in search_results:
             ds = sample.get_datasets()
-            datasets += ds.objects
+            for ds_object in ds.objects:
+                datasets += [ds_object] if ds_object.permId not in perm_ids else []
+                perm_ids.add(ds_object.permId)
+        for collection in collections:
+            ds = collection.get_datasets()
+            for ds_object in ds.objects:
+                datasets += [ds_object] if ds_object.permId not in perm_ids else []
+                perm_ids.add(ds_object.permId)
 
         click_echo(f"Data sets found: {len(datasets)}")
         if self.save_path is not None:

app-openbis-command-line/src/python/obis/dm/data_mgmt.py

@@ -31,6 +31,7 @@ from .commands.addref import Addref
 from .commands.clone import Clone
 from .commands.download_physical import DownloadPhysical
 from .commands.move import Move
+from .commands.object import Object
 from .commands.openbis_sync import OpenbisSync
 from .commands.removeref import Removeref
 from .commands.search import Search
@@ -683,7 +684,10 @@ class PhysicalDataMgmt(AbstractDataMgmt):
             self.error_raise(f"{category} clear",
                              "This command is only available for External Manager Data")
 
-        if category == "object" or category == "collection":
+        if category == "object":
+            cmd = Object(self, operation_type, prop, value)
+            return cmd.run()
+        elif category == "collection":
             click_echo("Not yet implemented.")
             return 0
         else:

app-openbis-command-line/src/python/obis/dm/utils.py

@@ -15,6 +15,7 @@
 import os
 import subprocess
 from contextlib import contextmanager
+from datetime import datetime
 from enum import Enum
 
 from .command_result import CommandResult, CommandException
@@ -45,8 +46,10 @@ def complete_openbis_config(config, resolver, local_only=True):
         config['token'] = None
     if config.get('is_physical') is None:
         config['is_physical'] = None
-    if config.get('allow_http_but_do_not_use_this_in_production_and_only_within_safe_networks') is None:
-        config['allow_http_but_do_not_use_this_in_production_and_only_within_safe_networks'] = not config_dict['allow_only_https']
+    if config.get(
+            'allow_http_but_do_not_use_this_in_production_and_only_within_safe_networks') is None:
+        config['allow_http_but_do_not_use_this_in_production_and_only_within_safe_networks'] = not \
+        config_dict['allow_only_https']
 
 
 def complete_git_config(config):
@@ -69,7 +72,9 @@ def default_echo(details):
 
 
 def run_shell(args, shell=False, strip_leading_whitespace=True, raise_exception_on_failure=False):
-    result = CommandResult(subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell), strip_leading_whitespace=strip_leading_whitespace)
+    result = CommandResult(
+        subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell),
+        strip_leading_whitespace=strip_leading_whitespace)
     if raise_exception_on_failure == True and result.failure():
         raise CommandException(result)
     return result
@@ -94,3 +99,15 @@ def cd(newdir):
         yield
     finally:
         os.chdir(prevdir)
+
+
+def is_valid_perm_id(name):
+    if "-" not in name:
+        return False
+    split = name.split("-")
+    try:
+        datetime.strptime(split[0], "%Y%m%d%H%M%S%f")
+        int(split[1])
+        return True
+    except ValueError:
+        return False