From a00774e4bbbe3237d45f2e6f8fa82940a0491154 Mon Sep 17 00:00:00 2001
From: Yves Noirjean <yves.noirjean@id.ethz.ch>
Date: Thu, 31 May 2018 16:27:23 +0200
Subject: [PATCH] obis: added skip_integrity_check
---
src/python/OBis/obis/dm/commands/clone.py | 6 +++--
src/python/OBis/obis/dm/commands/download.py | 6 +++--
src/python/OBis/obis/dm/data_mgmt.py | 23 +++++++++++++-------
src/python/OBis/obis/scripts/cli.py | 20 ++++++++---------
4 files changed, 33 insertions(+), 22 deletions(-)
diff --git a/src/python/OBis/obis/dm/commands/clone.py b/src/python/OBis/obis/dm/commands/clone.py
index 7331d048903..9efb4fc8ee3 100644
--- a/src/python/OBis/obis/dm/commands/clone.py
+++ b/src/python/OBis/obis/dm/commands/clone.py
@@ -16,11 +16,12 @@ class Clone(OpenbisCommand):
and adds the local copy as a new content copy using the addref command.
"""
- def __init__(self, dm, data_set_id, ssh_user, content_copy_index):
+ def __init__(self, dm, data_set_id, ssh_user, content_copy_index, skip_integrity_check):
self.data_set_id = data_set_id
self.ssh_user = ssh_user
self.content_copy_index = content_copy_index
self.load_global_config(dm)
+ self.skip_integrity_check = skip_integrity_check
super(Clone, self).__init__(dm)
@@ -56,7 +57,8 @@ class Clone(OpenbisCommand):
if result.failure():
return result
data_set = self.openbis.get_dataset(self.data_set_id)
- validate_checksum(self.openbis, data_set.file_list, data_set.permId, repository_folder)
+ if self.skip_integrity_check != True:
+ validate_checksum(self.openbis, data_set.file_list, data_set.permId, repository_folder)
return self.add_content_copy_to_openbis(repository_folder)
diff --git a/src/python/OBis/obis/dm/commands/download.py b/src/python/OBis/obis/dm/commands/download.py
index 36a1c1906ad..50e14c47878 100644
--- a/src/python/OBis/obis/dm/commands/download.py
+++ b/src/python/OBis/obis/dm/commands/download.py
@@ -12,10 +12,11 @@ class Download(OpenbisCommand):
"""
- def __init__(self, dm, data_set_id, content_copy_index, file):
+ def __init__(self, dm, data_set_id, content_copy_index, file, skip_integrity_check):
self.data_set_id = data_set_id
self.content_copy_index = content_copy_index
self.file = file
+ self.skip_integrity_check = skip_integrity_check
self.load_global_config(dm)
super(Download, self).__init__(dm)
@@ -29,5 +30,6 @@ class Download(OpenbisCommand):
content_copy_index = ContentCopySelector(data_set, self.content_copy_index, get_index=True).select()
files = [self.file] if self.file is not None else data_set.file_list
destination = data_set.download(files, linked_dataset_fileservice_url=self.fileservice_url(), content_copy_index=content_copy_index)
- validate_checksum(self.openbis, files, data_set.permId, os.path.join(destination, data_set.permId))
+ if self.skip_integrity_check != True:
+ validate_checksum(self.openbis, files, data_set.permId, os.path.join(destination, data_set.permId))
return CommandResult(returncode=0, output="Files downloaded to: %s" % os.path.join(destination, data_set.permId))
diff --git a/src/python/OBis/obis/dm/data_mgmt.py b/src/python/OBis/obis/dm/data_mgmt.py
index 3bf2a5d2455..49fa63c27fb 100644
--- a/src/python/OBis/obis/dm/data_mgmt.py
+++ b/src/python/OBis/obis/dm/data_mgmt.py
@@ -120,11 +120,12 @@ class AbstractDataMgmt(metaclass=abc.ABCMeta):
return
@abc.abstractmethod
- def clone(self, data_set_id, ssh_user, content_copy_index):
+ def clone(self, data_set_id, ssh_user, content_copy_index, skip_integrity_check):
"""Clone / copy a repository related to the given data set id.
:param data_set_id:
:param ssh_user: ssh user for remote clone (optional)
:param content_copy_index: index of content copy in case there are multiple copies (optional)
+ :param skip_integrity_check: if true, the file checksums will not be checked
:return: A CommandResult.
"""
return
@@ -144,7 +145,13 @@ class AbstractDataMgmt(metaclass=abc.ABCMeta):
return
@abc.abstractmethod
- def download(self, data_set_id, content_copy_index, file):
+ def download(self, data_set_id, content_copy_index, file, skip_integrity_check):
+ """Download files of a repository without adding a content copy.
+ :param data_set_id: Id of the data set to download from.
+ :param content_copy_index: Index of the content copy to download from.
+ :param file: Path of a file in the data set to download. All files are downloaded if it is None.
+ :param skip_integrity_check: Checksums of files are not verified if true.
+ """
return
@@ -166,7 +173,7 @@ class NoGitDataMgmt(AbstractDataMgmt):
def status(self):
self.error_raise("status", "No git command found.")
- def clone(self, data_set_id, ssh_user, content_copy_index):
+ def clone(self, data_set_id, ssh_user, content_copy_index, skip_integrity_check):
self.error_raise("clone", "No git command found.")
def addref(self):
@@ -175,7 +182,7 @@ class NoGitDataMgmt(AbstractDataMgmt):
def removeref(self):
self.error_raise("removeref", "No git command found.")
- def download(self, data_set_id, content_copy_index, file):
+ def download(self, data_set_id, content_copy_index, file, skip_integrity_check):
self.error_raise("download", "No git command found.")
@@ -351,8 +358,8 @@ class GitDataMgmt(AbstractDataMgmt):
self.git_wrapper.git_checkout(properties_path)
self.git_wrapper.git_delete_if_untracked(properties_path)
- def clone(self, data_set_id, ssh_user, content_copy_index):
- cmd = Clone(self, data_set_id, ssh_user, content_copy_index)
+ def clone(self, data_set_id, ssh_user, content_copy_index, skip_integrity_check):
+ cmd = Clone(self, data_set_id, ssh_user, content_copy_index, skip_integrity_check)
return cmd.run()
def addref(self):
@@ -363,6 +370,6 @@ class GitDataMgmt(AbstractDataMgmt):
cmd = Removeref(self)
return cmd.run()
- def download(self, data_set_id, content_copy_index, file):
- cmd = Download(self, data_set_id, content_copy_index, file)
+ def download(self, data_set_id, content_copy_index, file, skip_integrity_check):
+ cmd = Download(self, data_set_id, content_copy_index, file, skip_integrity_check)
return cmd.run()
diff --git a/src/python/OBis/obis/scripts/cli.py b/src/python/OBis/obis/scripts/cli.py
index 58744918390..e5ad17ad92c 100644
--- a/src/python/OBis/obis/scripts/cli.py
+++ b/src/python/OBis/obis/scripts/cli.py
@@ -693,55 +693,55 @@ def removeref(ctx, repository):
## download
-# TODO --skip_integrity_check flag for download, clone and move
-
_download_params = [
click.option('-c', '--content_copy_index', type=int, default=None, help='Index of the content copy to download from.'),
click.option('-f', '--file', help='File in the data set to download - downloading all if not given.'),
+ click.option('-s', '--skip_integrity_check', default=False, is_flag=True, help='Skip file integrity check with checksums.'),
click.argument('data_set_id'),
]
@data_set.command("download")
@add_params(_download_params)
@click.pass_context
-def data_set_download(ctx, content_copy_index, file, data_set_id):
+def data_set_download(ctx, content_copy_index, file, data_set_id, skip_integrity_check):
""" Download files of a linked data set.
"""
data_mgmt = shared_data_mgmt(ctx.obj)
- return check_result("download", run(ctx, lambda: data_mgmt.download(data_set_id, content_copy_index, file)))
+ return check_result("download", run(ctx, lambda: data_mgmt.download(data_set_id, content_copy_index, file, skip_integrity_check)))
@cli.command()
@add_params(_download_params)
@click.pass_context
-def download(ctx, content_copy_index, file, data_set_id):
+def download(ctx, content_copy_index, file, data_set_id, skip_integrity_check):
""" Download files of a linked data set.
"""
- ctx.invoke(data_set_download, content_copy_index=content_copy_index, file=file, data_set_id=data_set_id)
+ ctx.invoke(data_set_download, content_copy_index=content_copy_index, file=file, data_set_id=data_set_id, skip_integrity_check=skip_integrity_check)
## clone
_clone_params = [
click.option('-u', '--ssh_user', default=None, help='User to connect to remote systems via ssh'),
click.option('-c', '--content_copy_index', type=int, default=None, help='Index of the content copy to clone from in case there are multiple copies'),
+ click.option('-s', '--skip_integrity_check', default=False, is_flag=True, help='Skip file integrity check with checksums.'),
click.argument('data_set_id'),
]
@data_set.command("clone")
@click.pass_context
@add_params(_clone_params)
-def data_set_clone(ctx, ssh_user, content_copy_index, data_set_id):
+def data_set_clone(ctx, ssh_user, content_copy_index, data_set_id, skip_integrity_check):
"""Clone the repository found in the given data set id.
"""
data_mgmt = shared_data_mgmt(ctx.obj)
- return check_result("clone", run(ctx, lambda: data_mgmt.clone(data_set_id, ssh_user, content_copy_index)))
+ return check_result("clone", run(ctx, lambda: data_mgmt.clone(data_set_id, ssh_user, content_copy_index, skip_integrity_check)))
@cli.command()
@click.pass_context
@add_params(_clone_params)
-def clone(ctx, ssh_user, content_copy_index, data_set_id):
+def clone(ctx, ssh_user, content_copy_index, data_set_id, skip_integrity_check):
"""Clone the repository found in the given data set id.
"""
- ctx.invoke(data_set_clone, ssh_user=ssh_user, content_copy_index=content_copy_index, data_set_id=data_set_id)
+ ctx.invoke(data_set_clone, ssh_user=ssh_user, content_copy_index=content_copy_index, data_set_id=data_set_id, skip_integrity_check=skip_integrity_check)
def main():
--
GitLab