diff --git a/src/python/OBis/obis/dm/checksum.py b/src/python/OBis/obis/dm/checksum.py
index b10fba4225a2f00e13290bb5bd333bde4b6c5a9e..45a0bc7c829f8b08ea6b26d875ee1029241cb586 100644
--- a/src/python/OBis/obis/dm/checksum.py
+++ b/src/python/OBis/obis/dm/checksum.py
@@ -3,7 +3,7 @@ import json
import os
from abc import ABC, abstractmethod
from .utils import run_shell
-from .command_result import CommandException
+from .command_result import CommandResult, CommandException
def get_checksum_generator(checksum_type, default=None):
@@ -19,6 +19,27 @@ def get_checksum_generator(checksum_type, default=None):
return None
+def validate_checksum(openbis, files, data_set_id, folder):
+ dataset_files = openbis.search_files(data_set_id)['objects']
+ dataset_files_by_path = {}
+ for dataset_file in dataset_files:
+ dataset_files_by_path[dataset_file['path']] = dataset_file
+ for filename in files:
+ dataset_file = dataset_files_by_path[filename]
+ filename_dest = os.path.join(folder, filename)
+ checksum_generator = None
+ if dataset_file['checksumCRC32'] is not None and dataset_file['checksumCRC32'] > 0:
+ checksum_generator = ChecksumGeneratorCrc32()
+ expected_checksum = dataset_file['checksumCRC32']
+ elif dataset_file['checksumType'] is not None:
+ checksum_generator = get_checksum_generator(dataset_file['checksumType'])
+ expected_checksum = dataset_file['checksum']
+ if checksum_generator is not None:
+ checksum = checksum_generator.get_checksum(filename_dest)['checksum']
+ if checksum != expected_checksum:
+ raise CommandException(CommandResult(returncode=-1, output="Checksum wrong for file {}. Expected {} but was {}.".format(filename_dest, expected_checksum, checksum)))
+
+
class ChecksumGeneratorCrc32(object):
def get_checksum(self, file):
result = run_shell(['cksum', file])
diff --git a/src/python/OBis/obis/dm/commands/clone.py b/src/python/OBis/obis/dm/commands/clone.py
index 103a3f5a88ebe5a6e5a2a429bf36443618cd91ae..7331d048903d849eeb6b1d0ed2d99c609006d964 100644
--- a/src/python/OBis/obis/dm/commands/clone.py
+++ b/src/python/OBis/obis/dm/commands/clone.py
@@ -2,6 +2,7 @@ import socket
import os
import pybis
from .openbis_command import OpenbisCommand, ContentCopySelector
+from ..checksum import validate_checksum
from ..command_result import CommandResult
from ..utils import cd
from ..utils import run_shell
@@ -54,6 +55,8 @@ class Clone(OpenbisCommand):
result = self.checkout_commit(content_copy, path)
if result.failure():
return result
+ data_set = self.openbis.get_dataset(self.data_set_id)
+ validate_checksum(self.openbis, data_set.file_list, data_set.permId, repository_folder)
return self.add_content_copy_to_openbis(repository_folder)
diff --git a/src/python/OBis/obis/dm/commands/download.py b/src/python/OBis/obis/dm/commands/download.py
index 131c204af89cb94d10d1a5b889475f8b02453579..36a1c1906ad4aeb9af4fe1c5d55835cf3a5391fd 100644
--- a/src/python/OBis/obis/dm/commands/download.py
+++ b/src/python/OBis/obis/dm/commands/download.py
@@ -2,7 +2,7 @@ import os
import pybis
from .openbis_command import OpenbisCommand, ContentCopySelector
from ..command_result import CommandResult
-from ..checksum import get_checksum_generator, ChecksumGeneratorCrc32
+from ..checksum import validate_checksum
class Download(OpenbisCommand):
"""
@@ -29,29 +29,5 @@ class Download(OpenbisCommand):
content_copy_index = ContentCopySelector(data_set, self.content_copy_index, get_index=True).select()
files = [self.file] if self.file is not None else data_set.file_list
destination = data_set.download(files, linked_dataset_fileservice_url=self.fileservice_url(), content_copy_index=content_copy_index)
- result = self._validate_checksum(files, data_set, destination)
- if result.failure():
- return result
+ validate_checksum(self.openbis, files, data_set.permId, os.path.join(destination, data_set.permId))
return CommandResult(returncode=0, output="Files downloaded to: %s" % os.path.join(destination, data_set.permId))
-
-
- def _validate_checksum(self, files, data_set, destination):
- dataset_files = self.openbis.search_files(data_set.permId)['objects']
- dataset_files_by_path = {}
- for dataset_file in dataset_files:
- dataset_files_by_path[dataset_file['path']] = dataset_file
- for filename in files:
- dataset_file = dataset_files_by_path[filename]
- filename_dest = os.path.join(destination, data_set.permId, filename)
- checksum_generator = None
- if dataset_file['checksumCRC32'] is not None and dataset_file['checksumCRC32'] > 0:
- checksum_generator = ChecksumGeneratorCrc32()
- expected_checksum = dataset_file['checksumCRC32']
- elif dataset_file['checksumType'] is not None:
- checksum_generator = get_checksum_generator(dataset_file['checksumType'])
- expected_checksum = dataset_file['checksum']
- if checksum_generator is not None:
- checksum = checksum_generator.get_checksum(filename_dest)['checksum']
- if checksum != expected_checksum:
- return CommandResult(returncode=-1, output="Checksum wrong for file {}. Expected {} but was {}.".format(filename_dest, expected_checksum, checksum))
- return CommandResult(returncode=0, output="")
diff --git a/src/python/OBis/obis/dm/data_mgmt.py b/src/python/OBis/obis/dm/data_mgmt.py
index 711869b41abeafbbc36ca0ad26eabaa2426e74b5..3bf2a5d2455b40528e82b12ace1b0327189a67b0 100644
--- a/src/python/OBis/obis/dm/data_mgmt.py
+++ b/src/python/OBis/obis/dm/data_mgmt.py
@@ -34,7 +34,7 @@ from ..scripts import cli
# noinspection PyPep8Naming
-def DataMgmt(echo_func=None, settings_resolver=None, openbis_config={}, git_config={}, openbis=None):
+def DataMgmt(echo_func=None, settings_resolver=None, openbis_config={}, git_config={}, openbis=None, debug=False):
"""Factory method for DataMgmt instances"""
echo_func = echo_func if echo_func is not None else default_echo
@@ -51,7 +51,7 @@ def DataMgmt(echo_func=None, settings_resolver=None, openbis_config={}, git_conf
settings_resolver.set_resolver_location_roots('data_set', result.output)
complete_openbis_config(openbis_config, settings_resolver)
- return GitDataMgmt(settings_resolver, openbis_config, git_wrapper, openbis)
+ return GitDataMgmt(settings_resolver, openbis_config, git_wrapper, openbis, debug)
class AbstractDataMgmt(metaclass=abc.ABCMeta):
@@ -60,11 +60,12 @@ class AbstractDataMgmt(metaclass=abc.ABCMeta):
All operations throw an exepction if they fail.
"""
- def __init__(self, settings_resolver, openbis_config, git_wrapper, openbis):
+ def __init__(self, settings_resolver, openbis_config, git_wrapper, openbis, debug=False):
self.settings_resolver = settings_resolver
self.openbis_config = openbis_config
self.git_wrapper = git_wrapper
self.openbis = openbis
+ self.debug = debug
def error_raise(self, command, reason):
"""Raise an exception."""
@@ -194,6 +195,8 @@ def with_restore(f):
return result
except Exception as e:
self.restore()
+ if self.debug == True:
+ raise e
return CommandResult(returncode=-1, output="Error: " + str(e))
return f_with_restore
diff --git a/src/python/OBis/obis/scripts/cli.py b/src/python/OBis/obis/scripts/cli.py
index f1df55e16901b2e8d8a795d5dfb068a23c276504..58744918390189f38d22cbc4c678432966039c19 100644
--- a/src/python/OBis/obis/scripts/cli.py
+++ b/src/python/OBis/obis/scripts/cli.py
@@ -49,7 +49,7 @@ def shared_data_mgmt(context={}):
openbis_config = {}
if context.get('verify_certificates') is not None:
openbis_config['verify_certificates'] = context['verify_certificates']
- return dm.DataMgmt(openbis_config=openbis_config, git_config=git_config)
+ return dm.DataMgmt(openbis_config=openbis_config, git_config=git_config, debug=context['debug'])
def check_result(command, result):
@@ -60,23 +60,27 @@ def check_result(command, result):
return result.returncode
-def run(function):
+def run(ctx, function):
try:
return function()
except CommandException as e:
return e.command_result
except Exception as e:
+ if ctx.obj['debug'] == True:
+ raise e
return CommandResult(returncode=-1, output="Error: " + str(e))
@click.group()
@click.option('-q', '--quiet', default=False, is_flag=True, help='Suppress status reporting.')
@click.option('-s', '--skip_verification', default=False, is_flag=True, help='Do not verify cerficiates')
+@click.option('-d', '--debug', default=False, is_flag=True, help="Show stack trace on error.")
@click.pass_context
-def cli(ctx, quiet, skip_verification):
+def cli(ctx, quiet, skip_verification, debug):
ctx.obj['quiet'] = quiet
if skip_verification:
ctx.obj['verify_certificates'] = False
+ ctx.obj['debug'] = debug
def set_property(data_mgmt, resolver, prop, value, is_global, is_data_set_property=False):
@@ -88,6 +92,8 @@ def set_property(data_mgmt, resolver, prop, value, is_global, is_data_set_proper
else:
resolver.set_value_for_parameter(prop, value, loc)
except ValueError as e:
+ if data_mgmt.debug == True:
+ raise e
return CommandResult(returncode=-1, output="Error: " + str(e))
if not is_global:
return data_mgmt.commit_metadata_updates(prop)
@@ -102,7 +108,7 @@ def init_data_impl(ctx, object_id, collection_id, repository, desc):
click_echo("init_data {}".format(repository))
data_mgmt = shared_data_mgmt(ctx.obj)
desc = desc if desc != "" else None
- result = run(lambda: data_mgmt.init_data(repository, desc, create=True))
+ result = run(ctx, lambda: data_mgmt.init_data(repository, desc, create=True))
init_handle_cleanup(result, object_id, collection_id, repository, data_mgmt)
@@ -110,7 +116,7 @@ def init_analysis_impl(ctx, parent, object_id, collection_id, repository, descri
click_echo("init_analysis {}".format(repository))
data_mgmt = shared_data_mgmt(ctx.obj)
description = description if description != "" else None
- result = run(lambda: data_mgmt.init_analysis(repository, parent, description, create=True))
+ result = run(ctx, lambda: data_mgmt.init_analysis(repository, parent, description, create=True))
init_handle_cleanup(result, object_id, collection_id, repository, data_mgmt)
@@ -327,21 +333,21 @@ def repository(ctx, is_global):
@click.argument('settings', type=SettingsSet(), nargs=-1)
@click.pass_context
def repository_set(ctx, settings):
- return check_result("repository_set", run(lambda: _set(ctx, settings)))
+ return check_result("repository_set", run(ctx, lambda: _set(ctx, settings)))
@repository.command('get')
@click.argument('settings', type=SettingsGet(), nargs=-1)
@click.pass_context
def repository_get(ctx, settings):
- return check_result("repository_get", run(lambda: _get(ctx, settings)))
+ return check_result("repository_get", run(ctx, lambda: _get(ctx, settings)))
@repository.command('clear')
@click.argument('settings', type=SettingsClear(), nargs=-1)
@click.pass_context
def repository_clear(ctx, settings):
- return check_result("repository_clear", run(lambda: _clear(ctx, settings)))
+ return check_result("repository_clear", run(ctx, lambda: _clear(ctx, settings)))
## data_set: type, properties
@@ -364,21 +370,21 @@ def data_set(ctx, is_global, is_data_set_property):
@click.argument('settings', type=SettingsSet(), nargs=-1)
@click.pass_context
def data_set_set(ctx, settings):
- return check_result("data_set_set", run(lambda: _set(ctx, settings)))
+ return check_result("data_set_set", run(ctx, lambda: _set(ctx, settings)))
@data_set.command('get')
@click.argument('settings', type=SettingsGet(), nargs=-1)
@click.pass_context
def data_set_get(ctx, settings):
- return check_result("data_set_get", run(lambda: _get(ctx, settings)))
+ return check_result("data_set_get", run(ctx, lambda: _get(ctx, settings)))
@data_set.command('clear')
@click.argument('settings', type=SettingsClear(), nargs=-1)
@click.pass_context
def data_set_clear(ctx, settings):
- return check_result("data_set_clear", run(lambda: _clear(ctx, settings)))
+ return check_result("data_set_clear", run(ctx, lambda: _clear(ctx, settings)))
## object: object_id
@@ -399,21 +405,21 @@ def object(ctx, is_global):
@click.argument('settings', type=SettingsSet(), nargs=-1)
@click.pass_context
def object_set(ctx, settings):
- return check_result("object_set", run(lambda: _set(ctx, settings)))
+ return check_result("object_set", run(ctx, lambda: _set(ctx, settings)))
@object.command('get')
@click.argument('settings', type=SettingsGet(), nargs=-1)
@click.pass_context
def object_get(ctx, settings):
- return check_result("object_get", run(lambda: _get(ctx, settings)))
+ return check_result("object_get", run(ctx, lambda: _get(ctx, settings)))
@object.command('clear')
@click.argument('settings', type=SettingsClear(), nargs=-1)
@click.pass_context
def object_clear(ctx, settings):
- return check_result("object_clear", run(lambda: _clear(ctx, settings)))
+ return check_result("object_clear", run(ctx, lambda: _clear(ctx, settings)))
## collection: collection_id
@@ -434,21 +440,21 @@ def collection(ctx, is_global):
@click.argument('settings', type=SettingsSet(), nargs=-1)
@click.pass_context
def collection_set(ctx, settings):
- return check_result("collection_set", run(lambda: _set(ctx, settings)))
+ return check_result("collection_set", run(ctx, lambda: _set(ctx, settings)))
@collection.command('get')
@click.argument('settings', type=SettingsGet(), nargs=-1)
@click.pass_context
def collection_get(ctx, settings):
- return check_result("collection_get", run(lambda: _get(ctx, settings)))
+ return check_result("collection_get", run(ctx, lambda: _get(ctx, settings)))
@collection.command('clear')
@click.argument('settings', type=SettingsClear(), nargs=-1)
@click.pass_context
def collection_clear(ctx, settings):
- return check_result("collection_clear", run(lambda: _clear(ctx, settings)))
+ return check_result("collection_clear", run(ctx, lambda: _clear(ctx, settings)))
## config: fileservice_url, git_annex_hash_as_checksum, hostname, openbis_url, user, verify_certificates
@@ -469,21 +475,21 @@ def config(ctx, is_global):
@click.argument('settings', type=SettingsSet(), nargs=-1)
@click.pass_context
def config_set(ctx, settings):
- return check_result("config_set", run(lambda: _set(ctx, settings)))
+ return check_result("config_set", run(ctx, lambda: _set(ctx, settings)))
@config.command('get')
@click.argument('settings', type=SettingsGet(), nargs=-1)
@click.pass_context
def config_get(ctx, settings):
- return check_result("config_get", run(lambda: _get(ctx, settings)))
+ return check_result("config_get", run(ctx, lambda: _get(ctx, settings)))
@config.command('clear')
@click.argument('settings', type=SettingsClear(), nargs=-1)
@click.pass_context
def config_clear(ctx, settings):
- return check_result("config_clear", run(lambda: _clear(ctx, settings)))
+ return check_result("config_clear", run(ctx, lambda: _clear(ctx, settings)))
# repository commands: status, sync, commit, init, addref, removeref, init_analysis
@@ -499,7 +505,7 @@ _commit_params = [
def _repository_commit(ctx, msg, auto_add, ignore_missing_parent):
data_mgmt = shared_data_mgmt(ctx.obj)
- return check_result("commit", run(lambda: data_mgmt.commit(msg, auto_add, ignore_missing_parent)))
+ return check_result("commit", run(ctx, lambda: data_mgmt.commit(msg, auto_add, ignore_missing_parent)))
@repository.command("commit")
@click.pass_context
@@ -572,7 +578,7 @@ _status_params = [
def _repository_status(ctx):
data_mgmt = shared_data_mgmt(ctx.obj)
- result = run(data_mgmt.status)
+ result = run(ctx, data_mgmt.status)
click.echo(result.output)
@repository.command("status")
@@ -603,7 +609,7 @@ _sync_params = [
def _repository_sync(ctx, ignore_missing_parent):
data_mgmt = shared_data_mgmt(ctx.obj)
- return check_result("sync", run(lambda: data_mgmt.sync(ignore_missing_parent)))
+ return check_result("sync", run(ctx, lambda: data_mgmt.sync(ignore_missing_parent)))
@repository.command("sync")
@click.pass_context
@@ -632,7 +638,7 @@ _addref_params = [
def _repository_addref(ctx):
data_mgmt = shared_data_mgmt(ctx.obj)
- return check_result("addref", run(data_mgmt.addref))
+ return check_result("addref", run(ctx, data_mgmt.addref))
@repository.command("addref")
@click.pass_context
@@ -661,7 +667,7 @@ _removeref_params = [
def _repository_removeref(ctx):
data_mgmt = shared_data_mgmt(ctx.obj)
- return check_result("addref", run(data_mgmt.removeref))
+ return check_result("addref", run(ctx, data_mgmt.removeref))
@repository.command("removeref")
@click.pass_context
@@ -687,6 +693,8 @@ def removeref(ctx, repository):
## download
+# TODO --skip_integrity_check flag for download, clone and move
+
_download_params = [
click.option('-c', '--content_copy_index', type=int, default=None, help='Index of the content copy to download from.'),
click.option('-f', '--file', help='File in the data set to download - downloading all if not given.'),
@@ -700,7 +708,7 @@ def data_set_download(ctx, content_copy_index, file, data_set_id):
""" Download files of a linked data set.
"""
data_mgmt = shared_data_mgmt(ctx.obj)
- return check_result("download", run(lambda: data_mgmt.download(data_set_id, content_copy_index, file)))
+ return check_result("download", run(ctx, lambda: data_mgmt.download(data_set_id, content_copy_index, file)))
@cli.command()
@add_params(_download_params)
@@ -725,7 +733,7 @@ def data_set_clone(ctx, ssh_user, content_copy_index, data_set_id):
"""Clone the repository found in the given data set id.
"""
data_mgmt = shared_data_mgmt(ctx.obj)
- return check_result("clone", run(lambda: data_mgmt.clone(data_set_id, ssh_user, content_copy_index)))
+ return check_result("clone", run(ctx, lambda: data_mgmt.clone(data_set_id, ssh_user, content_copy_index)))
@cli.command()
@click.pass_context