From a8bb971cb2731e6b07a6e6a41a59bb0208a7f1d4 Mon Sep 17 00:00:00 2001
From: alaskowski <alaskowski@ethz.ch>
Date: Thu, 23 Feb 2023 12:19:47 +0100
Subject: [PATCH] SSDM-13330: Added downloading flow for PHYSICAL data
---
.../obis/dm/commands/download_physical.py | 46 ++++++++++++++
.../obis/dm/commands/openbis_command.py | 23 +++++--
.../src/python/obis/dm/data_mgmt.py | 62 +++++++++++--------
.../src/python/obis/dm/git.py | 9 ++-
.../python/obis/scripts/data_mgmt_runner.py | 40 ++++++------
5 files changed, 123 insertions(+), 57 deletions(-)
create mode 100644 app-openbis-command-line/src/python/obis/dm/commands/download_physical.py
diff --git a/app-openbis-command-line/src/python/obis/dm/commands/download_physical.py b/app-openbis-command-line/src/python/obis/dm/commands/download_physical.py
new file mode 100644
index 00000000000..43f084b5f3e
--- /dev/null
+++ b/app-openbis-command-line/src/python/obis/dm/commands/download_physical.py
@@ -0,0 +1,46 @@
+# Copyright ETH 2023 Zürich, Scientific IT Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from .openbis_command import OpenbisCommand
+from ..command_result import CommandResult
+from ..utils import cd
+
+
+class DownloadPhysical(OpenbisCommand):
+ """
+ Command to download physical files of a data set.
+ """
+
+ def __init__(self, dm, data_set_id, file):
+ """
+ :param dm: data management
+ :param data_set_id: permId of the data set to be cloned
+ """
+ self.data_set_id = data_set_id
+ self.files = [file] if file is not None else None
+ self.load_global_config(dm)
+ super(DownloadPhysical, self).__init__(dm)
+
+ def run(self):
+ if self.fileservice_url() is None:
+ return CommandResult(returncode=-1,
+ output="Configuration fileservice_url needs to be set for download.")
+
+ data_set = self.openbis.get_dataset(self.data_set_id)
+ files = self.files if self.files is not None else data_set.file_list
+
+ with cd(self.data_mgmt.invocation_path):
+ target_folder = data_set.download(files)
+ return CommandResult(returncode=0, output="Files downloaded to: %s" % target_folder)
diff --git a/app-openbis-command-line/src/python/obis/dm/commands/openbis_command.py b/app-openbis-command-line/src/python/obis/dm/commands/openbis_command.py
index 8aa0a3ce1f2..68e7db6d4ed 100644
--- a/app-openbis-command-line/src/python/obis/dm/commands/openbis_command.py
+++ b/app-openbis-command-line/src/python/obis/dm/commands/openbis_command.py
@@ -16,12 +16,13 @@ import getpass
import hashlib
import os
import socket
+
import pybis
-from ..command_result import CommandResult
-from ..command_result import CommandException
+
from .. import config as dm_config
+from ..command_result import CommandException
+from ..command_result import CommandResult
from ..utils import complete_openbis_config
-from ...scripts import cli
class OpenbisCommand(object):
@@ -36,7 +37,14 @@ class OpenbisCommand(object):
self.config_dict = dm.settings_resolver.config_dict()
if self.openbis is None and dm.openbis_config.get('url') is not None:
- self.openbis = pybis.Openbis(**dm.openbis_config)
+ self.openbis = pybis.Openbis(url=dm.openbis_config.get('url'),
+ verify_certificates=dm.openbis_config.get(
+ 'verify_certificates'),
+ token=dm.openbis_config.get('token'),
+ use_cache=dm.openbis_config.get('use_cache'),
+ allow_http_but_do_not_use_this_in_production_and_only_within_safe_networks=dm.openbis_config.get(
+ 'allow_http_but_do_not_use_this_in_production_and_only_within_safe_networks'),
+ )
if self.user() is not None:
result = self.login()
if result.failure():
@@ -198,8 +206,11 @@ class OpenbisCommand(object):
except ValueError:
# external dms does not exist - create it
try:
- external_dms = self.openbis.create_external_data_management_system(external_dms_id, external_dms_id,
- "{}:/{}".format(hostname, edms_path))
+ external_dms = self.openbis.create_external_data_management_system(external_dms_id,
+ external_dms_id,
+ "{}:/{}".format(
+ hostname,
+ edms_path))
except Exception as error:
return CommandResult(returncode=-1, output=str(error))
return CommandResult(returncode=0, output=external_dms)
diff --git a/app-openbis-command-line/src/python/obis/dm/data_mgmt.py b/app-openbis-command-line/src/python/obis/dm/data_mgmt.py
index 23202e3f36d..ab26c845343 100644
--- a/app-openbis-command-line/src/python/obis/dm/data_mgmt.py
+++ b/app-openbis-command-line/src/python/obis/dm/data_mgmt.py
@@ -30,6 +30,7 @@ from .command_result import CommandResult
from .commands.addref import Addref
from .commands.clone import Clone
from .commands.download import Download
+from .commands.download_physical import DownloadPhysical
from .commands.move import Move
from .commands.openbis_sync import OpenbisSync
from .commands.removeref import Removeref
@@ -67,17 +68,20 @@ def DataMgmt(echo_func=None, settings_resolver=None, openbis_config={}, git_conf
repository_type = Type.LINK
if repository_type == Type.PHYSICAL:
- return PhysicalDataMgmt(settings_resolver, None, None, openbis, log, data_path, metadata_path, invocation_path)
+ return PhysicalDataMgmt(settings_resolver, None, None, openbis, log, data_path,
+ metadata_path, invocation_path)
else:
complete_git_config(git_config)
git_wrapper = GitWrapper(**git_config)
if not git_wrapper.can_run():
# TODO We could just as well throw an error here instead of creating
# creating the NoGitDataMgmt which will fail later.
- return NoGitDataMgmt(settings_resolver, None, git_wrapper, openbis, log, data_path, metadata_path, invocation_path)
+ return NoGitDataMgmt(settings_resolver, None, git_wrapper, openbis, log, data_path,
+ metadata_path, invocation_path)
complete_openbis_config(openbis_config, settings_resolver)
- return GitDataMgmt(settings_resolver, openbis_config, git_wrapper, openbis, log, data_path, metadata_path, invocation_path, debug, login)
+ return GitDataMgmt(settings_resolver, openbis_config, git_wrapper, openbis, log, data_path,
+ metadata_path, invocation_path, debug, login)
class AbstractDataMgmt(metaclass=abc.ABCMeta):
@@ -86,7 +90,8 @@ class AbstractDataMgmt(metaclass=abc.ABCMeta):
All operations throw an exepction if they fail.
"""
- def __init__(self, settings_resolver, openbis_config, git_wrapper, openbis, log, data_path, metadata_path, invocation_path, debug=False, login=True):
+ def __init__(self, settings_resolver, openbis_config, git_wrapper, openbis, log, data_path,
+ metadata_path, invocation_path, debug=False, login=True):
self.settings_resolver = settings_resolver
self.openbis_config = openbis_config
self.git_wrapper = git_wrapper
@@ -259,22 +264,25 @@ def restore_signal_handler(data_mgmt):
def with_log(f):
""" To be used with commands that use the CommandLog. """
+
def f_with_log(self, *args):
try:
result = f(self, *args)
except Exception as e:
self.log.log_error(str(e))
raise e
- if result.failure() == False:
+ if result.failure() == False:
self.log.success()
else:
self.log.log_error(result.output)
return result
+
return f_with_log
def with_restore(f):
""" Sets the restore point and restores on error. """
+
def f_with_restore(self, *args):
self.set_restorepoint()
try:
@@ -291,6 +299,7 @@ def with_restore(f):
raise e
self.clear_restorepoint()
return CommandResult(returncode=-1, output="Error: " + str(e))
+
return f_with_restore
@@ -305,7 +314,6 @@ class GitDataMgmt(AbstractDataMgmt):
settings_resolver.set_resolver_location_roots('data_set', relative_path)
return settings_resolver
-
# TODO add this to abstract / other class
def setup_local_settings(self, all_settings):
self.settings_resolver.set_resolver_location_roots('data_set', '.')
@@ -314,17 +322,14 @@ class GitDataMgmt(AbstractDataMgmt):
for key, value in settings.items():
resolver.set_value_for_parameter(key, value, 'local')
-
def get_data_set_id(self, relative_path):
settings_resolver = self.get_settings_resolver(relative_path)
return settings_resolver.repository.config_dict().get('data_set_id')
-
def get_repository_id(self, relative_path):
settings_resolver = self.get_settings_resolver(relative_path)
return settings_resolver.repository.config_dict().get('id')
-
def init_data(self, desc=None):
# check that repository does not already exist
# TODO remove .git check after physical flow is implemented
@@ -345,13 +350,13 @@ class GitDataMgmt(AbstractDataMgmt):
self.settings_resolver.copy_global_to_local()
return CommandResult(returncode=0, output="")
-
def init_analysis(self, parent_folder, desc=None):
# get data_set_id of parent from current folder or explicit parent argument
parent_data_set_id = self.get_data_set_id(parent_folder)
# check that parent repository has been added to openBIS
if self.get_repository_id(parent_folder) is None:
- return CommandResult(returncode=-1, output="Parent data set must be committed to openBIS before creating an analysis data set.")
+ return CommandResult(returncode=-1,
+ output="Parent data set must be committed to openBIS before creating an analysis data set.")
# init analysis repository
result = self.init_data(desc)
if result.failure():
@@ -366,20 +371,18 @@ class GitDataMgmt(AbstractDataMgmt):
self.git_wrapper.git_ignore(analysis_folder_relative)
# set data_set_id to analysis repository so it will be used as parent when committing
- self.set_property(self.settings_resolver.repository, "data_set_id", parent_data_set_id, False, False)
+ self.set_property(self.settings_resolver.repository, "data_set_id", parent_data_set_id,
+ False, False)
return result
-
@with_restore
def sync(self):
return self._sync()
-
def _sync(self):
cmd = OpenbisSync(self, self.ignore_missing_parent)
return cmd.run()
-
@with_restore
def commit(self, msg, auto_add=True, sync=True):
""" Git add, commit and sync with openBIS. """
@@ -398,7 +401,6 @@ class GitDataMgmt(AbstractDataMgmt):
result = self._sync()
return result
-
def status(self):
git_status = self.git_wrapper.git_status()
try:
@@ -454,7 +456,8 @@ class GitDataMgmt(AbstractDataMgmt):
# settings
#
- def config(self, category, is_global, is_data_set_property, prop=None, value=None, set=False, get=False, clear=False):
+ def config(self, category, is_global, is_data_set_property, prop=None, value=None, set=False,
+ get=False, clear=False):
"""
:param category: config, object, collection, data_set or repository
:param is_global: act on global settings - local if false
@@ -503,26 +506,31 @@ class GitDataMgmt(AbstractDataMgmt):
config_str = json.dumps(little_dict, indent=4, sort_keys=True)
click_echo("{}".format(config_str), with_timestamp=False)
elif set == True:
- return check_result("config", self.set_property(resolver, prop, value, is_global, is_data_set_property))
+ return check_result("config", self.set_property(resolver, prop, value, is_global,
+ is_data_set_property))
elif clear == True:
if prop is None:
returncode = 0
for prop in config_dict.keys():
- returncode += check_result("config", self.set_property(resolver, prop, None, is_global, is_data_set_property))
+ returncode += check_result("config",
+ self.set_property(resolver, prop, None, is_global,
+ is_data_set_property))
return returncode
else:
- return check_result("config", self.set_property(resolver, prop, None, is_global, is_data_set_property))
+ return check_result("config", self.set_property(resolver, prop, None, is_global,
+ is_data_set_property))
def set_property(self, resolver, prop, value, is_global, is_data_set_property=False):
"""Helper function to implement the property setting semantics."""
loc = 'global' if is_global else 'local'
try:
if is_data_set_property:
- resolver.set_value_for_json_parameter('properties', prop, value, loc, apply_rules=True)
+ resolver.set_value_for_json_parameter('properties', prop, value, loc,
+ apply_rules=True)
else:
resolver.set_value_for_parameter(prop, value, loc, apply_rules=True)
except Exception as e:
- if self.debug == True:
+ if self.debug == True:
raise e
return CommandResult(returncode=-1, output="Error: " + str(e))
else:
@@ -544,7 +552,10 @@ class PhysicalDataMgmt(AbstractDataMgmt):
self.settings_resolver.set_resolver_location_roots('data_set', '.')
self.settings_resolver.copy_global_to_local()
self.settings_resolver.config.set_value_for_parameter("is_physical", True, "local")
- return CommandResult(returncode=0, output="Physical obis repository initialized!")
+ openbis_url = self.settings_resolver.config.config_dict()['openbis_url']
+ self.settings_resolver.config.set_value_for_parameter("fileservice_url",
+ openbis_url, "local")
+ return CommandResult(returncode=0, output="Physical obis repository initialized.")
def init_analysis(self, parent_folder, desc=None):
self.error_raise("init analysis", "Not implemented.")
@@ -570,5 +581,6 @@ class PhysicalDataMgmt(AbstractDataMgmt):
def removeref(self, data_set_id=None):
self.error_raise("removeref", "Not implemented.")
- def download(self, data_set_id, content_copy_index, file, skip_integrity_check):
- self.error_raise("download", "Not implemented.")
\ No newline at end of file
+ def download(self, data_set_id, _content_copy_index, file, _skip_integrity_check):
+ cmd = DownloadPhysical(self, data_set_id, file)
+ return cmd.run()
diff --git a/app-openbis-command-line/src/python/obis/dm/git.py b/app-openbis-command-line/src/python/obis/dm/git.py
index ddc661904a2..c22df44daa7 100644
--- a/app-openbis-command-line/src/python/obis/dm/git.py
+++ b/app-openbis-command-line/src/python/obis/dm/git.py
@@ -12,12 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-import shutil
import os
-from pathlib import Path
-from .utils import run_shell, cd
-from .command_result import CommandResult, CommandException
+import shutil
+
from .checksum import ChecksumGeneratorCrc32, ChecksumGeneratorGitAnnex
+from .utils import run_shell
class GitWrapper(object):
@@ -77,7 +76,7 @@ class GitWrapper(object):
else:
return self._git(["annex", "status", path], strip_leading_whitespace=False)
- def git_annex_backend(self, desc, git_annex_backend=None):
+ def git_annex_init(self, desc, git_annex_backend=None):
""" Configures annex in a git repository."""
# We use annex --version=5 since that works better with big files. Version
diff --git a/app-openbis-command-line/src/python/obis/scripts/data_mgmt_runner.py b/app-openbis-command-line/src/python/obis/scripts/data_mgmt_runner.py
index 8ffabde717a..ea5e9ae58a8 100644
--- a/app-openbis-command-line/src/python/obis/scripts/data_mgmt_runner.py
+++ b/app-openbis-command-line/src/python/obis/scripts/data_mgmt_runner.py
@@ -26,7 +26,6 @@ from ..dm.utils import run_shell
class DataMgmtRunner(object):
-
def __init__(self, context, halt_on_error_log=True, data_path=None, bootstrap_settings=None,
check_result=True, login=True, openbis=None, is_physical=False):
self.context = context
@@ -40,7 +39,6 @@ class DataMgmtRunner(object):
self.openbis = openbis
self.repository_type = Type.PHYSICAL if is_physical else Type.UNKNOWN
-
def init_paths(self, repository=None):
# data path
if self.data_path is None:
@@ -62,19 +60,19 @@ class DataMgmtRunner(object):
if not os.path.exists(self.data_path):
os.makedirs(self.data_path)
-
def _validate_obis_metadata_folder(self, obis_metadata_folder):
if not os.path.isabs(obis_metadata_folder):
return CommandResult(
- returncode=-1,
- output="Ignoring obis_metadata_folder. Must be absolute but is: {}".format(obis_metadata_folder))
+ returncode=-1,
+ output="Ignoring obis_metadata_folder. Must be absolute but is: {}".format(
+ obis_metadata_folder))
if not os.path.exists(obis_metadata_folder):
return CommandResult(
- returncode=-1,
- output="Ignoring obis_metadata_folder. Folder does not exist: {}".format(obis_metadata_folder))
+ returncode=-1,
+ output="Ignoring obis_metadata_folder. Folder does not exist: {}".format(
+ obis_metadata_folder))
return CommandResult(returncode=0, output="")
-
def run(self, command, function, repository=None):
self.init_paths(repository)
with cd(self.metadata_path):
@@ -84,7 +82,6 @@ class DataMgmtRunner(object):
else:
return result
-
def _run(self, function):
try:
dm = self._get_dm()
@@ -98,13 +95,11 @@ class DataMgmtRunner(object):
raise e
return CommandResult(returncode=-1, output="Error: " + str(e))
-
def get_settings(self, repository=None, do_cd=True):
self.init_paths()
with cd(self.metadata_path):
return self.get_settings_resolver(do_cd).config_dict()
-
def get_settings_resolver(self, do_cd=True):
if do_cd:
self.init_paths()
@@ -113,25 +108,28 @@ class DataMgmtRunner(object):
else:
return self._get_dm().get_settings_resolver()
-
def config(self, resolver, is_global, is_data_set_property, prop, value, set, get, clear):
self.init_paths()
with cd(self.metadata_path):
- self._get_dm().config(resolver, is_global, is_data_set_property, prop, value, set, get, clear)
-
+ self._get_dm().config(resolver, is_global, is_data_set_property, prop, value, set, get,
+ clear)
def _get_dm(self):
git_config = {
- 'find_git': True,
- 'data_path': self.data_path,
- 'metadata_path': self.metadata_path,
- 'invocation_path': self.invocation_path
- }
+ 'find_git': True,
+ 'data_path': self.data_path,
+ 'metadata_path': self.metadata_path,
+ 'invocation_path': self.invocation_path
+ }
openbis_config = {}
if self.context.get('verify_certificates') is not None:
openbis_config['verify_certificates'] = self.context['verify_certificates']
log = CommandLog()
if self.halt_on_error_log and log.any_log_exists():
- click_echo("Error: A previous command did not finish. Please check the log ({}) and remove it when you want to continue using obis".format(log.folder_path))
+ click_echo(
+ "Error: A previous command did not finish. Please check the log ({}) and remove it when you want to continue using obis".format(
+ log.folder_path))
sys.exit(-1)
- return dm.DataMgmt(openbis_config=openbis_config, git_config=git_config, log=log, debug=self.context['debug'], login=self.login, openbis=self.openbis, repository_type=self.repository_type)
+ return dm.DataMgmt(openbis_config=openbis_config, git_config=git_config, log=log,
+ debug=self.context['debug'], login=self.login, openbis=self.openbis,
+ repository_type=self.repository_type)
--
GitLab