SSDM-13300: Added filtering by modification_date and registration_date to...

SSDM-13300: Added filtering by modification_date and registration_date to object/dataset search in obis

app-openbis-command-line/src/python/obis/dm/commands/download_physical.py

@@ -60,15 +60,21 @@ class DownloadPhysical(OpenbisCommand):
         return CommandResult(returncode=0, output="Download completed.")
 
     def download_dataset(self, perm_id):
+        click_echo(f"Downloading dataset {perm_id}")
         data_set = self.openbis.get_dataset(perm_id)
         files = self.files if self.files is not None else data_set.file_list
         with cd(self.data_mgmt.invocation_path):
-            target_folder = data_set.download(files, destination=self.data_mgmt.invocation_path)
+            try:
+                target_folder = data_set.download(files, destination=self.data_mgmt.invocation_path)
+            except Exception as e:
+                print(e)
+                raise e
+
             if self.skip_integrity_check is not True:
                 invalid_files = validate_checksum(self.openbis, files, data_set.permId,
                                                   target_folder, None)
                 self.redownload_invalid_files_on_demand(invalid_files, target_folder, perm_id)
-                click_echo(f"Files from dataset {perm_id} has been downloaded to {target_folder}")
+        click_echo(f"Files from dataset {perm_id} has been downloaded to {target_folder}")
 
     def redownload_invalid_files_on_demand(self, invalid_files, target_folder, perm_id):
         if len(invalid_files) == 0:

app-openbis-command-line/src/python/obis/dm/commands/search.py

@@ -24,43 +24,38 @@ class Search(OpenbisCommand):
     Command to search data in openBIS.
     """
 
-    def __init__(self, dm, type_code, space, project, experiment, property_code, property_value,
-                 save_path):
+    def __init__(self, dm, filters, save_path):
         """
         :param dm: data management
-        :param type_code: Filter by type code
-        :param space: Filter by space path
-        :param project: Filter by project path
-        :param experiment: Filter by experiment
-        :param property_code: Filter by property_code, needs to be set together with property_value
-        :param property_value: Filter by property_value, needs to be set together with property_code
+        :param filters: Dictionary of filter to be used during search
         :param save_path: Path to save results. If not set, results will not be saved.
         """
-        self.property_value = property_value
-        self.property_code = property_code
-        self.experiment = experiment
-        self.project = project
-        self.space = space
-        self.type_code = type_code
+        self.filters = filters
         self.save_path = save_path
         self.load_global_config(dm)
         super(Search, self).__init__(dm)
 
     def search_samples(self):
         properties = None
-        if self.property_code is not None and self.property_value is not None:
+        if self.filters['property_code'] is not None and self.filters['property_value'] is not None:
             properties = {
-                self.property_code: self.property_value,
+                self.filters['property_code']: self.filters['property_value'],
             }
 
-        search_results = self.openbis.get_samples(
-            space=self.space,
-            project=self.project,  # Not Supported with Project Samples disabled
-            experiment=self.experiment,
-            type=self.type_code,
-            where=properties,
-            props="*"  # Fetch all properties
-        )
+        args = dict(space=self.filters['space'],
+                    project=self.filters['project'],  # Not Supported with Project Samples disabled
+                    experiment=self.filters['experiment'],
+                    type=self.filters['type_code'],
+                    where=properties,
+                    attrs=["parents", "children"],
+                    props="*")  # Fetch all properties
+
+        if self.filters['registration_date'] is not None:
+            args['registrationDate'] = self.filters['registration_date']
+        if self.filters['modification_date'] is not None:
+            args['modificationDate'] = self.filters['modification_date']
+
+        search_results = self.openbis.get_samples(**args)
         click_echo(f"Objects found: {len(search_results)}")
         if self.save_path is not None:
             click_echo(f"Saving search results in {self.save_path}")
@@ -77,20 +72,24 @@ class Search(OpenbisCommand):
                                  output="Configuration fileservice_url needs to be set for download.")
 
         properties = None
-        if self.property_code is not None and self.property_value is not None:
+        if self.filters['property_code'] is not None and self.filters['property_value'] is not None:
             properties = {
-                self.property_code: self.property_value,
+                self.filters['property_code']: self.filters['property_value'],
             }
+        args = dict(space=self.filters['space'],
+                    project=self.filters['project'],  # Not Supported with Project Samples disabled
+                    experiment=self.filters['experiment'],
+                    type=self.filters['type_code'],
+                    where=properties,
+                    attrs=["parents", "children"],
+                    props="*")  # Fetch all properties
+
+        if self.filters['registration_date'] is not None:
+            args['registrationDate'] = self.filters['registration_date']
+        if self.filters['modification_date'] is not None:
+            args['modificationDate'] = self.filters['modification_date']
 
-        datasets = self.openbis.get_datasets(
-            space=self.space,
-            project=self.project,  # Not Supported with Project Samples disabled
-            experiment=self.experiment,
-            type=self.type_code,
-            where=properties,
-            attrs=["parents", "children"],
-            props="*"  # Fetch all properties
-        )
+        datasets = self.openbis.get_datasets(**args)
 
         click_echo(f"Data sets found: {len(datasets)}")
         if self.save_path is not None:

app-openbis-command-line/src/python/obis/dm/data_mgmt.py

@@ -229,29 +229,17 @@ class AbstractDataMgmt(metaclass=abc.ABCMeta):
         return
 
     @abc.abstractmethod
-    def search_object(self, type_code, space, project, experiment, property_code, property_value,
-                      save):
+    def search_object(self, filters, save):
         """Search for objects in openBIS using filtering criteria.
-        :param type_code: Type of searched object.
-        :param space: Space path to filter object.
-        :param project: Project path to filter object.
-        :param experiment: Experiment path to filter object.
-        :param property_code: Custom property code to search by, property_value must be set as well.
-        :param property_value: Custom property value to search by, property_code must be set as well.
+        :param filters: dictionary of filter parameters
         :param save: File path to save results. If missing, search results will not be saved.
         """
         return
 
     @abc.abstractmethod
-    def search_data_set(self, type_code, space, project, experiment, property_code, property_value,
-                        save):
+    def search_data_set(self, filters, save):
         """Search for datasets in openBIS using filtering criteria.
-        :param type_code: Type of searched object.
-        :param space: Space path to filter object.
-        :param project: Project path to filter object.
-        :param experiment: Experiment path to filter object.
-        :param property_code: Custom property code to search by, property_value must be set as well.
-        :param property_value: Custom property value to search by, property_code must be set as well.
+        :param filters: dictionary of filter parameters
         :param save: File path to save results. If missing, search results will not be saved.
         """
         return
@@ -654,16 +642,12 @@ class PhysicalDataMgmt(AbstractDataMgmt):
         cmd = Upload(self, sample_id, data_set_type, files)
         return cmd.run()
 
-    def search_object(self, type_code, space, project, experiment, property_code, property_value,
-                      save):
-        cmd = Search(self, type_code, space, project, experiment, property_code, property_value,
-                     save)
+    def search_object(self,filters, save):
+        cmd = Search(self, filters, save)
         return cmd.search_samples()
 
-    def search_data_set(self, type_code, space, project, experiment, property_code, property_value,
-                        save):
-        cmd = Search(self, type_code, space, project, experiment, property_code, property_value,
-                     save)
+    def search_data_set(self, filters, save):
+        cmd = Search(self, filters, save)
         return cmd.search_data_sets()
 
     def config(self, category, is_global, is_data_set_property, operation_type, prop=None,

app-openbis-command-line/src/python/obis/scripts/cli.py

@@ -279,6 +279,10 @@ _search_params = [
     click.option('-property', 'property_code', default=None, help='Property code'),
     click.option('-property-value', 'property_value', default=None,
                  help='Property value'),
+    click.option('-registration-date', '--registration-date', 'registration_date', default=None,
+                 help='Registration date, it can be in the format "oYYYY-MM-DD" (e.g. ">2023-01-01")'),
+    click.option('-modification-date', '--modification-date', 'modification_date', default=None,
+                 help='Modification date, it can be in the format "oYYYY-MM-DD" (e.g. ">2023-01-01")'),
     click.option('-save', '--save', default=None, help='Filename to save results'),
 ]
 
@@ -322,10 +326,11 @@ def data_set_clear(ctx, data_set_settings):
 @data_set.command('search', short_help="Search for datasets using a filtering criteria.")
 @add_params(_search_params)
 @click.pass_context
-def data_set_search(ctx, type_code, space, project, experiment, property_code, property_value,
-                    save):
+def data_set_search(ctx, type_code, space, project, experiment, registration_date,
+                    modification_date, property_code, property_value, save):
     if all(v is None for v in
-           [type_code, space, project, experiment, property_code, property_value]):
+           [type_code, space, project, experiment, registration_date, modification_date,
+            property_code, property_value]):
         click_echo("You must provide at least one filtering criteria!")
         return -1
     if (property_code is None and property_value is not None) or (
@@ -333,19 +338,12 @@ def data_set_search(ctx, type_code, space, project, experiment, property_code, p
         click_echo("Property code and property value need to be specified!")
         return -1
     ctx.obj['runner'] = DataMgmtRunner(ctx.obj, halt_on_error_log=False)
-    ctx.invoke(_data_set_search, type_code=type_code, space=space,
-               project=project, experiment=experiment, property_code=property_code,
-               property_value=property_value, save=save)
-
-
-@add_params(_search_params)
-@click.pass_context
-def _data_set_search(ctx, type_code, space, project, experiment, property_code, property_value,
-                     save):
+    filters = dict(type_code=type_code, space=space,
+                   project=project, experiment=experiment, property_code=property_code,
+                   registration_date=registration_date, modification_date=modification_date,
+                   property_value=property_value)
     return ctx.obj['runner'].run("data_set_search",
-                                 lambda dm: dm.search_data_set(type_code, space, project,
-                                                               experiment, property_code,
-                                                               property_value, save)),
+                                 lambda dm: dm.search_data_set(filters, save)),
 
 
 # # object: object_id
@@ -387,9 +385,11 @@ def object_clear(ctx, object_settings):
 @object.command('search', short_help="Search for samples using a filtering criteria.")
 @add_params(_search_params)
 @click.pass_context
-def object_search(ctx, type_code, space, project, experiment, property_code, property_value, save):
+def object_search(ctx, type_code, space, project, experiment, registration_date,
+                  modification_date, property_code, property_value, save):
     if all(v is None for v in
-           [type_code, space, project, experiment, property_code, property_value]):
+           [type_code, space, project, experiment, registration_date, modification_date,
+            property_code, property_value]):
         click_echo("You must provide at least one filtering criteria!")
         return -1
     if (property_code is None and property_value is not None) or (
@@ -397,17 +397,12 @@ def object_search(ctx, type_code, space, project, experiment, property_code, pro
         click_echo("Property code and property value need to be specified!")
         return -1
     ctx.obj['runner'] = DataMgmtRunner(ctx.obj, halt_on_error_log=False)
-    ctx.invoke(_object_search, type_code=type_code, space=space,
-               project=project, experiment=experiment, property_code=property_code,
-               property_value=property_value, save=save)
-
-
-@add_params(_search_params)
-@click.pass_context
-def _object_search(ctx, type_code, space, project, experiment, property_code, property_value, save):
+    filters = dict(type_code=type_code, space=space,
+                   project=project, experiment=experiment, property_code=property_code,
+                   registration_date=registration_date, modification_date=modification_date,
+                   property_value=property_value)
     return ctx.obj['runner'].run("object_search",
-                                 lambda dm: dm.search_object(type_code, space, project, experiment,
-                                                             property_code, property_value, save)),
+                                 lambda dm: dm.search_object(filters, save))
 
 
 # # collection: collection_id
@@ -764,7 +759,8 @@ _download_params = [
                  help='An output .CSV file from `obis data_set search` command with the list of' +
                       ' objects to download datasets from'),
     click.option(
-        '-f', '--file', help='File in the data set to download - downloading all if not given.'),
+        '-f', '--file', 'file',
+        help='File in the data set to download - downloading all if not given.'),
     click.option('-s', '--skip_integrity_check', default=False, is_flag=True,
                  help='Flag to skip file integrity check with checksums'),
 ]

docs/app-openbis-command-line/README.md

@@ -205,6 +205,12 @@ Options:
   -project, --project TEXT        Full project identification code
   -experiment, --experiment TEXT  Full experiment code
   -type, --type TEXT              Type code
+  -registration-date, --registration-date TEXT
+                                  Registration date, it can be in the format
+                                  "oYYYY-MM-DD" (e.g. ">2023-01-31", "=2023-01-31", "<2023-01-31")
+  -modification-date, --modification-date TEXT
+                                  Modification date, it can be in the format
+                                  "oYYYY-MM-DD" (e.g. ">2023-01-31", "=2023-01-31", "<2023-01-31")
   -property TEXT                  Property code
   -property-value TEXT            Property value
   -save, --save TEXT              Directory name to save results
@@ -269,7 +275,13 @@ Options:
   -type, --type TEXT              Type code to filter by
   -space, --space TEXT            Space code
   -project, --project TEXT        Full project identification code
-  -experiment, --experiment TEXT  Full experiment code
+  -experiment, --experiment TEXT  Full experiment 
+  -registration-date, --registration-date TEXT
+                                  Registration date, it can be in the format
+                                  "oYYYY-MM-DD" (e.g. ">2023-01-31", "=2023-01-31", "<2023-01-31")
+  -modification-date, --modification-date TEXT
+                                  Modification date, it can be in the format
+                                  "oYYYY-MM-DD" (e.g. ">2023-01-31", "=2023-01-31", "<2023-01-31")
   -property TEXT                  Property code
   -property-value TEXT            Property value
   -save, --save TEXT              File name to save results in csv format