diff --git a/app-openbis-command-line/src/python/obis/dm/commands/download_physical.py b/app-openbis-command-line/src/python/obis/dm/commands/download_physical.py
index 44845cf12abe87dfa2f7e1823c0424cc679cc1c2..1b5e94c4878e2d35613278acaa94125ba492e74e 100644
--- a/app-openbis-command-line/src/python/obis/dm/commands/download_physical.py
+++ b/app-openbis-command-line/src/python/obis/dm/commands/download_physical.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+import csv
import os
from .openbis_command import OpenbisCommand
@@ -26,12 +27,16 @@ class DownloadPhysical(OpenbisCommand):
Command to download physical files of a data set.
"""
- def __init__(self, dm, data_set_id, file, skip_integrity_check):
+ def __init__(self, dm, data_set_id, from_file, file, skip_integrity_check):
"""
- :param dm: data management
- :param data_set_id: permId of the data set to be cloned
+ :param dm: data management.
+ :param data_set_id: permId of the data set to be cloned.
+ :param from_file: Path to a CSV file with a list of datasets to download.
+ :param file: path to a specific file to download from a dataset.
+ :param skip_integrity_check: boolean flag indicating whether to skip checksum validation.
"""
self.data_set_id = data_set_id
+ self.from_file = from_file
self.files = [file] if file is not None else None
self.skip_integrity_check = skip_integrity_check
self.load_global_config(dm)
@@ -42,25 +47,37 @@ class DownloadPhysical(OpenbisCommand):
return CommandResult(returncode=-1,
output="Configuration fileservice_url needs to be set for download.")
- data_set = self.openbis.get_dataset(self.data_set_id)
- files = self.files if self.files is not None else data_set.file_list
+ if self.from_file is not None:
+ with cd(self.data_mgmt.invocation_path):
+ with open(self.from_file, newline='') as csvfile:
+ reader = csv.DictReader(csvfile)
+ for row in reader:
+ self.download_dataset(row['permId'])
+ if reader.line_num == 1: # First row contains headers
+ click_echo("No data sets were found in provided file!")
+ else:
+ self.download_dataset(self.data_set_id)
+ return CommandResult(returncode=0, output="Download completed.")
+ def download_dataset(self, perm_id):
+ data_set = self.openbis.get_dataset(perm_id)
+ files = self.files if self.files is not None else data_set.file_list
with cd(self.data_mgmt.invocation_path):
target_folder = data_set.download(files, destination=self.data_mgmt.invocation_path)
if self.skip_integrity_check is not True:
invalid_files = validate_checksum(self.openbis, files, data_set.permId,
target_folder, None)
- self.redownload_invalid_files_on_demand(invalid_files, target_folder)
- return CommandResult(returncode=0, output="Files downloaded to: %s" % target_folder)
+ self.redownload_invalid_files_on_demand(invalid_files, target_folder, perm_id)
+ click_echo(f"Files from dataset {perm_id} has been downloaded to {target_folder}")
- def redownload_invalid_files_on_demand(self, invalid_files, target_folder):
+ def redownload_invalid_files_on_demand(self, invalid_files, target_folder, perm_id):
if len(invalid_files) == 0:
return
yes_or_no = None
while yes_or_no != "yes" and yes_or_no != "no":
- click_echo("Integrity check failed for following files:\n" +
+ click_echo(f"Integrity check failed for following files in dataset {perm_id}:\n" +
str(invalid_files) + "\n" +
- "Either the download failed or the files where changed after committing to openBIS.\n" +
+ "Either the download failed or the files where changed in the OpenBIS.\n" +
"Should the files be downloaded again? (yes/no)")
yes_or_no = input('> ')
if yes_or_no == "yes":
@@ -68,4 +85,4 @@ class DownloadPhysical(OpenbisCommand):
filename_dest = os.path.join(target_folder, file)
os.remove(filename_dest)
self.files = invalid_files
- return self.run()
+ return self.download_dataset(perm_id)
diff --git a/app-openbis-command-line/src/python/obis/dm/commands/search.py b/app-openbis-command-line/src/python/obis/dm/commands/search.py
index 4ef0d1e865fa5a972fce32d10f8f0a2078b061ed..d4b2ad454ed0b47011402a553240b5977f78c67b 100644
--- a/app-openbis-command-line/src/python/obis/dm/commands/search.py
+++ b/app-openbis-command-line/src/python/obis/dm/commands/search.py
@@ -13,8 +13,6 @@
# limitations under the License.
#
-import os
-
from .openbis_command import OpenbisCommand
from ..command_result import CommandResult
from ..utils import cd
@@ -84,7 +82,7 @@ class Search(OpenbisCommand):
self.property_code: self.property_value,
}
- search_results = self.openbis.get_samples(
+ datasets = self.openbis.get_datasets(
space=self.space,
project=self.project, # Not Supported with Project Samples disabled
experiment=self.experiment,
@@ -94,43 +92,11 @@ class Search(OpenbisCommand):
props="*" # Fetch all properties
)
- collections = self.openbis.get_collections(
- space=self.space,
- project=self.project,
- type=self.type_code,
- where=properties,
- props="*" # Fetch all properties
- )
-
- click_echo("Looking for data sets")
- datasets = []
- perm_ids = set()
- for sample in search_results:
- ds = sample.get_datasets()
- for ds_object in ds.objects:
- datasets += [ds_object] if ds_object.permId not in perm_ids else []
- perm_ids.add(ds_object.permId)
- for collection in collections:
- ds = collection.get_datasets()
- for ds_object in ds.objects:
- datasets += [ds_object] if ds_object.permId not in perm_ids else []
- perm_ids.add(ds_object.permId)
-
click_echo(f"Data sets found: {len(datasets)}")
if self.save_path is not None:
+ click_echo(f"Saving search results in {self.save_path}")
with cd(self.data_mgmt.invocation_path):
- if os.path.exists(self.save_path) is True and os.path.isdir(
- self.save_path) is False:
- return CommandResult(returncode=-1,
- output=f"File {self.save_path} is not a directory")
- if os.path.isdir(self.save_path) is False:
- click_echo(f"Creating directory {self.save_path}")
- os.makedirs(self.save_path)
- click_echo(
- f"Saving search results in {os.path.join(self.data_mgmt.invocation_path, self.save_path)}")
- for dataset in datasets:
- dataset.download(destination=self.save_path,
- linked_dataset_fileservice_url=self.fileservice_url() + "/download")
+ datasets.df.to_csv(self.save_path, index=False)
else:
click_echo(f"Search results:\n{datasets}")
diff --git a/app-openbis-command-line/src/python/obis/dm/data_mgmt.py b/app-openbis-command-line/src/python/obis/dm/data_mgmt.py
index 92464167e30ff43d178ccf88538fdaf7d15e0cd2..92c5a514662296f08e1d4eae3995624e1f0624f1 100644
--- a/app-openbis-command-line/src/python/obis/dm/data_mgmt.py
+++ b/app-openbis-command-line/src/python/obis/dm/data_mgmt.py
@@ -210,9 +210,10 @@ class AbstractDataMgmt(metaclass=abc.ABCMeta):
return
@abc.abstractmethod
- def download(self, data_set_id, file, skip_integrity_check):
+ def download(self, data_set_id, from_file, file, skip_integrity_check):
"""Download files of a repository without adding a content copy.
:param data_set_id: Id of the data set to download from.
+ :param from_file: Path of a file with a list of datasets to download.
:param file: Path of a file in the data set to download. All files are downloaded if it is None.
:param skip_integrity_check: Checksums of files are not verified if true.
"""
@@ -349,7 +350,7 @@ class NoGitDataMgmt(AbstractDataMgmt):
def removeref(self, data_set_id=None):
self.error_raise("removeref", "No git command found.")
- def download(self, data_set_id, file, skip_integrity_check):
+ def download(self, *_):
self.error_raise("download", "No git command found.")
def search_object(self, *_):
@@ -559,7 +560,7 @@ class GitDataMgmt(AbstractDataMgmt):
cmd = Removeref(self, data_set_id=data_set_id)
return cmd.run()
- def download(self, data_set_id, file, skip_integrity_check):
+ def download(self, data_set_id, from_file, file, skip_integrity_check):
self.error_raise("download", "This command is only available for Manager Data.")
#
@@ -645,8 +646,8 @@ class PhysicalDataMgmt(AbstractDataMgmt):
def removeref(self, data_set_id=None):
self.error_raise("removeref", "This command is only available for External Manager Data")
- def download(self, data_set_id, file, skip_integrity_check):
- cmd = DownloadPhysical(self, data_set_id, file, skip_integrity_check)
+ def download(self, data_set_id, from_file, file, skip_integrity_check):
+ cmd = DownloadPhysical(self, data_set_id, from_file, file, skip_integrity_check)
return cmd.run()
def upload(self, sample_id, data_set_type, files):
diff --git a/app-openbis-command-line/src/python/obis/scripts/cli.py b/app-openbis-command-line/src/python/obis/scripts/cli.py
index be6804e1cc66ecd2c56b5690f95a31fceeb4f1a8..3c626e9762e4f1df9d1da63a14add25f12497598 100644
--- a/app-openbis-command-line/src/python/obis/scripts/cli.py
+++ b/app-openbis-command-line/src/python/obis/scripts/cli.py
@@ -275,6 +275,7 @@ _search_params = [
click.option('-space', '--space', default=None, help='Space code'),
click.option('-project', '--project', default=None, help='Full project identification code'),
click.option('-experiment', '--experiment', default=None, help='Full experiment code'),
+ click.option('-type', '--type', 'type_code', default=None, help='Type code'),
click.option('-property', 'property_code', default=None, help='Property code'),
click.option('-property-value', 'property_value', default=None,
help='Property value'),
@@ -758,7 +759,10 @@ def removeref(ctx, data_set_id, repository):
# download
_download_params = [
- click.argument('data_set_id'),
+ click.argument('data_set_id', required=False),
+ click.option('-from-file', '--from-file', 'from_file',
+ help='An output .CSV file from `obis data_set search` command with the list of' +
+ ' objects to download datasets from'),
click.option(
'-f', '--file', help='File in the data set to download - downloading all if not given.'),
click.option('-s', '--skip_integrity_check', default=False, is_flag=True,
@@ -766,22 +770,21 @@ _download_params = [
]
-@data_set.command("download", short_help="Download files of a data set.")
-@add_params(_download_params)
-@click.pass_context
-def data_set_download(ctx, file, data_set_id, skip_integrity_check):
- return ctx.obj['runner'].run("download",
- lambda dm: dm.download(data_set_id=data_set_id, file=file,
- skip_integrity_check=skip_integrity_check))
-
-
@cli.command("download", short_help="Download files of a data set.")
@add_params(_download_params)
@click.pass_context
-def download(ctx, file, data_set_id, skip_integrity_check):
+def download(ctx, data_set_id, from_file, file, skip_integrity_check):
+ """ Downloads dataset files from OpenBIS instance.\n
+ DATA_SET Unique identifier of dataset within OpenBIS instance."""
+ if (data_set_id is None and from_file is None) or (
+ data_set_id is not None and from_file is not None):
+ click_echo("'data_set_id' or 'from_file' must be provided!")
+ return -1
ctx.obj['runner'] = DataMgmtRunner(ctx.obj, halt_on_error_log=False)
- ctx.invoke(data_set_download, file=file,
- data_set_id=data_set_id, skip_integrity_check=skip_integrity_check)
+ return ctx.obj['runner'].run("download",
+ lambda dm: dm.download(data_set_id=data_set_id,
+ from_file=from_file, file=file,
+ skip_integrity_check=skip_integrity_check))
# upload
@@ -795,20 +798,17 @@ _upload_params = [
]
-@data_set.command("upload", short_help="Upload files to form a data set.")
-@add_params(_upload_params)
-@click.pass_context
-def data_set_upload(ctx, sample_id, data_set_type, files):
- return ctx.obj['runner'].run("upload",
- lambda dm: dm.upload(sample_id, data_set_type, files))
-
-
@cli.command("upload", short_help="Upload files to form a data set.")
@add_params(_upload_params)
@click.pass_context
-def download(ctx, sample_id, data_set_type, files):
+def upload(ctx, sample_id, data_set_type, files):
+ """ Creates data set under object and upload files to it.\n
+ SAMPLE_ID Unique identifier an object in OpenBIS.\n
+ DATA_SET_TYPE Newly created data set type.
+ """
ctx.obj['runner'] = DataMgmtRunner(ctx.obj, halt_on_error_log=False)
- ctx.invoke(data_set_upload, files=files, sample_id=sample_id, data_set_type=data_set_type)
+ ctx.obj['runner'].run("upload",
+ lambda dm: dm.upload(sample_id, data_set_type, files))
# clone
diff --git a/docs/app-openbis-command-line/README.md b/docs/app-openbis-command-line/README.md
index 62505db94e28bfce7854a9bac0370995cae98986..4b8b3566a277bbdb91be4ca2ed9fa6bcee055e38 100644
--- a/docs/app-openbis-command-line/README.md
+++ b/docs/app-openbis-command-line/README.md
@@ -204,6 +204,7 @@ Options:
-space, --space TEXT Space code
-project, --project TEXT Full project identification code
-experiment, --experiment TEXT Full experiment code
+ -type, --type TEXT Type code
-property TEXT Property code
-property-value TEXT Property value
-save, --save TEXT Directory name to save results
@@ -211,7 +212,7 @@ Options:
With `data_set search` command, obis connects to a configured OpenBIS instance and searches for all
data sets that fulfill given filtering criteria.
-At least one filtering criteria must be specified. Resulting data set files can be downloaded by
+At least one filtering criteria must be specified. Search results can be downloaded by
using `save` option.
*Note: Filtering by `-project` may not work when `Project Samples` are disabled in OpenBIS
@@ -221,6 +222,15 @@ configuration.*
```
obis download [options] [data_set_id]
+
+Options:
+ -from-file, --from-file TEXT An output .CSV file from `obis data_set search`
+ command with the list of objects to download
+ data sets from
+ -f, --file TEXT File in the data set to download - downloading
+ all if not given.
+ -s, --skip_integrity_check Flag to skip file integrity check with
+ checksums
```
The `download` command downloads, the files of a given data set from the OpenBIS instance specified