diff --git a/app-openbis-command-line/src/python/obis/dm/.obis/config.json b/app-openbis-command-line/src/python/obis/dm/.obis/config.json deleted file mode 100644 index de2e0d62f0bd0805cb3261752b6fea7398c42548..0000000000000000000000000000000000000000 --- a/app-openbis-command-line/src/python/obis/dm/.obis/config.json +++ /dev/null @@ -1 +0,0 @@ -{"data_set_id": "20170814175159563859-9212", "data_set_type": "DS_TYPE", "object_id": "/SAMPLE/ID", "openbis_url": "http://localhost:8888"} \ No newline at end of file diff --git a/app-openbis-command-line/src/python/obis/dm/config.py b/app-openbis-command-line/src/python/obis/dm/config.py index 8b9d5f268b0fb27a7fca39ae8a240ce7a05a31ec..20591a8d7575cc50de02da5d5ee8bca68a4243d1 100644 --- a/app-openbis-command-line/src/python/obis/dm/config.py +++ b/app-openbis-command-line/src/python/obis/dm/config.py @@ -135,6 +135,7 @@ class ConfigEnv(object): self.add_param(ConfigParam(name='obis_metadata_folder')) self.add_param(ConfigParam(name='openbis_token')) self.add_param(ConfigParam(name='session_name')) + self.add_param(ConfigParam(name='is_physical', default_value=False)) def add_param(self, param): self.params[param.name] = param diff --git a/app-openbis-command-line/src/python/obis/dm/data_mgmt.py b/app-openbis-command-line/src/python/obis/dm/data_mgmt.py index a2cd2897e757c7dde93e96ca0f561052840bcf45..23202e3f36d6c108f21c40198fe43f851b6dea1e 100644 --- a/app-openbis-command-line/src/python/obis/dm/data_mgmt.py +++ b/app-openbis-command-line/src/python/obis/dm/data_mgmt.py @@ -2,59 +2,49 @@ # -*- coding: utf-8 -*- # Copyright ETH 2018 - 2023 Zürich, Scientific IT Services -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # - -""" -data_mgmt.py - -Module implementing data management operations. - -Created by Chandrasekhar Ramakrishnan on 2017-02-01. -Copyright (c) 2017 Chandrasekhar Ramakrishnan. All rights reserved. -""" import abc import json import os import shutil -import traceback -import pybis -import requests import signal import sys from pathlib import Path + +import requests + from . import config as dm_config +from .command_result import CommandResult from .commands.addref import Addref -from .commands.removeref import Removeref from .commands.clone import Clone +from .commands.download import Download from .commands.move import Move from .commands.openbis_sync import OpenbisSync -from .commands.download import Download -from .command_log import CommandLog -from .command_result import CommandResult -from .command_result import CommandException +from .commands.removeref import Removeref from .git import GitWrapper -from .utils import default_echo +from .utils import Type +from .utils import cd from .utils import complete_git_config from .utils import complete_openbis_config -from .utils import cd -from ..scripts import cli +from .utils import default_echo from ..scripts.click_util import click_echo, check_result # noinspection PyPep8Naming -def DataMgmt(echo_func=None, settings_resolver=None, openbis_config={}, git_config={}, openbis=None, log=None, debug=False, login=True): +def DataMgmt(echo_func=None, settings_resolver=None, openbis_config={}, git_config={}, + openbis=None, log=None, debug=False, login=True, repository_type=Type.UNKNOWN): """Factory method for DataMgmt instances""" echo_func = echo_func if echo_func is not None else default_echo @@ -63,19 +53,31 @@ def DataMgmt(echo_func=None, settings_resolver=None, openbis_config={}, git_conf metadata_path = git_config['metadata_path'] invocation_path = git_config['invocation_path'] - complete_git_config(git_config) - git_wrapper = GitWrapper(**git_config) - if not git_wrapper.can_run(): - # TODO We could just as well throw an error here instead of creating - # creating the NoGitDataMgmt which will fail later. - return NoGitDataMgmt(settings_resolver, None, git_wrapper, openbis, log, data_path, metadata_path, invocation_path) - if settings_resolver is None: settings_resolver = dm_config.SettingsResolver() - complete_openbis_config(openbis_config, settings_resolver) + if repository_type == Type.UNKNOWN: + if os.path.exists('.obis'): + config_dict = settings_resolver.config.config_dict() + if config_dict['is_physical'] is True: + repository_type = Type.PHYSICAL + else: + repository_type = Type.LINK + else: + repository_type = Type.LINK + + if repository_type == Type.PHYSICAL: + return PhysicalDataMgmt(settings_resolver, None, None, openbis, log, data_path, metadata_path, invocation_path) + else: + complete_git_config(git_config) + git_wrapper = GitWrapper(**git_config) + if not git_wrapper.can_run(): + # TODO We could just as well throw an error here instead of creating + # creating the NoGitDataMgmt which will fail later. + return NoGitDataMgmt(settings_resolver, None, git_wrapper, openbis, log, data_path, metadata_path, invocation_path) - return GitDataMgmt(settings_resolver, openbis_config, git_wrapper, openbis, log, data_path, metadata_path, invocation_path, debug, login) + complete_openbis_config(openbis_config, settings_resolver) + return GitDataMgmt(settings_resolver, openbis_config, git_wrapper, openbis, log, data_path, metadata_path, invocation_path, debug, login) class AbstractDataMgmt(metaclass=abc.ABCMeta): @@ -325,6 +327,7 @@ class GitDataMgmt(AbstractDataMgmt): def init_data(self, desc=None): # check that repository does not already exist + # TODO remove .git check after physical flow is implemented if os.path.exists('.obis') and os.path.exists('.git'): return CommandResult(returncode=-1, output="Folder is already an obis repository.") result = self.git_wrapper.git_init() @@ -524,3 +527,48 @@ class GitDataMgmt(AbstractDataMgmt): return CommandResult(returncode=-1, output="Error: " + str(e)) else: return CommandResult(returncode=0, output="") + + +class PhysicalDataMgmt(AbstractDataMgmt): + """DataMgmt operations for DSS-stored data.""" + + def get_settings_resolver(self): + return dm_config.SettingsResolver() + + def setup_local_settings(self, all_settings): + self.error_raise("setup local settings", "Not implemented.") + + def init_data(self, desc=None): + if os.path.exists('.obis'): + return CommandResult(returncode=-1, output="Folder is already an obis repository.") + self.settings_resolver.set_resolver_location_roots('data_set', '.') + self.settings_resolver.copy_global_to_local() + self.settings_resolver.config.set_value_for_parameter("is_physical", True, "local") + return CommandResult(returncode=0, output="Physical obis repository initialized!") + + def init_analysis(self, parent_folder, desc=None): + self.error_raise("init analysis", "Not implemented.") + + def commit(self, msg, auto_add=True, sync=True): + self.error_raise("commit", "Not implemented.") + + def sync(self): + self.error_raise("sync", "Not implemented.") + + def status(self): + self.error_raise("status", "Not implemented.") + + def clone(self, data_set_id, ssh_user, content_copy_index, skip_integrity_check): + self.error_raise("clone", "Not implemented.") + + def move(self, data_set_id, ssh_user, content_copy_index, skip_integrity_check): + self.error_raise("move", "Not implemented.") + + def addref(self): + self.error_raise("addref", "Not implemented.") + + def removeref(self, data_set_id=None): + self.error_raise("removeref", "Not implemented.") + + def download(self, data_set_id, content_copy_index, file, skip_integrity_check): + self.error_raise("download", "Not implemented.") \ No newline at end of file diff --git a/app-openbis-command-line/src/python/obis/dm/git.py b/app-openbis-command-line/src/python/obis/dm/git.py index a62b293eaafa9f231315b16a9d5caea0d84ed54a..ddc661904a2e9d9603fb6a0d37e747c696ce3f0f 100644 --- a/app-openbis-command-line/src/python/obis/dm/git.py +++ b/app-openbis-command-line/src/python/obis/dm/git.py @@ -77,7 +77,7 @@ class GitWrapper(object): else: return self._git(["annex", "status", path], strip_leading_whitespace=False) - def git_annex_init(self, desc, git_annex_backend=None): + def git_annex_backend(self, desc, git_annex_backend=None): """ Configures annex in a git repository.""" # We use annex --version=5 since that works better with big files. Version diff --git a/app-openbis-command-line/src/python/obis/dm/utils.py b/app-openbis-command-line/src/python/obis/dm/utils.py index fc74dcfac36f7a76a392ca58924e41c8518d8703..81eafa8f991cf520bb0881d5c03e84e5c2c45e2d 100644 --- a/app-openbis-command-line/src/python/obis/dm/utils.py +++ b/app-openbis-command-line/src/python/obis/dm/utils.py @@ -12,12 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import subprocess import os +import subprocess from contextlib import contextmanager +from enum import Enum + from .command_result import CommandResult, CommandException +class Type(Enum): + """Enum representing which what type of repository it is""" + LINK = 1, + PHYSICAL = 2, + UNKNOWN = 3 + def complete_openbis_config(config, resolver, local_only=True): """Add default values for empty entries in the config.""" config_dict = resolver.config.config_dict(local_only) @@ -27,6 +35,8 @@ def complete_openbis_config(config, resolver, local_only=True): config['verify_certificates'] = config_dict['verify_certificates'] if config.get('token') is None: config['token'] = None + if config.get('is_physical') is None: + config['is_physical'] = None if config.get('allow_http_but_do_not_use_this_in_production_and_only_within_safe_networks') is None: config['allow_http_but_do_not_use_this_in_production_and_only_within_safe_networks'] = not config_dict['allow_only_https'] diff --git a/app-openbis-command-line/src/python/obis/scripts/cli.py b/app-openbis-command-line/src/python/obis/scripts/cli.py index 131bc54e64a676ae1bc85e63b8af0b91c67a4fd5..3fc36956e214503812be3aed9080114954d3c9bd 100644 --- a/app-openbis-command-line/src/python/obis/scripts/cli.py +++ b/app-openbis-command-line/src/python/obis/scripts/cli.py @@ -20,24 +20,19 @@ cli.py The module that implements the CLI for obis. - - -Created by Chandrasekhar Ramakrishnan on 2017-01-27. -Copyright (c) 2017 Chandrasekhar Ramakrishnan. All rights reserved. """ import json import os from datetime import datetime -from requests import ConnectionError import click from dateutil.relativedelta import relativedelta from pybis import Openbis +from requests import ConnectionError -from ..dm.command_result import CommandResult -from ..dm.utils import cd from .click_util import click_echo from .data_mgmt_runner import DataMgmtRunner +from ..dm.command_result import CommandResult def click_progress(progress_data): @@ -448,25 +443,32 @@ def commit(ctx, msg, auto_add, ignore_missing_parent, repository): _init_params = [ - click.argument('repository', type=click.Path( + click.argument('repository_path', type=click.Path( exists=False, file_okay=False), required=False), click.argument('description', default=""), + ] @repository.command("init", short_help="Initialize the folder as a data repository.") @click.pass_context @add_params(_init_params) -def repository_init(ctx, repository, description): - return init_data_impl(ctx, repository, description) +def repository_init(ctx, repository_path, description): + return init_data_impl(ctx, repository_path, description) + + +_init_params_physical = \ + _init_params + \ + [click.option('-p', '--physical', 'is_physical', default=False, is_flag=True, + help='If parent data set is missing, ignore it.')] @cli.command(short_help="Initialize the folder as a data repository.") @click.pass_context -@add_params(_init_params) -def init(ctx, repository, description): - ctx.obj['runner'] = DataMgmtRunner(ctx.obj, halt_on_error_log=False) - ctx.invoke(repository_init, repository=repository, description=description) +@add_params(_init_params_physical) +def init(ctx, repository_path, description, is_physical): + ctx.obj['runner'] = DataMgmtRunner(ctx.obj, halt_on_error_log=False, is_physical=is_physical) + ctx.invoke(repository_init, repository_path=repository_path, description=description) # init analysis @@ -481,17 +483,17 @@ _init_analysis_params += _init_params @repository.command("init_analysis", short_help="Initialize the folder as an analysis folder.") @click.pass_context @add_params(_init_analysis_params) -def repository_init_analysis(ctx, parent, repository, description): - return init_analysis_impl(ctx, parent, repository, description) +def repository_init_analysis(ctx, parent, repository_path, description): + return init_analysis_impl(ctx, parent, repository_path, description) @cli.command(name='init_analysis', short_help="Initialize the folder as an analysis folder.") @click.pass_context @add_params(_init_analysis_params) -def init_analysis(ctx, parent, repository, description): +def init_analysis(ctx, parent, repository_path, description): ctx.obj['runner'] = DataMgmtRunner(ctx.obj, halt_on_error_log=False) ctx.invoke(repository_init_analysis, parent=parent, - repository=repository, description=description) + repository_path=repository_path, description=description) # status diff --git a/app-openbis-command-line/src/python/obis/scripts/data_mgmt_runner.py b/app-openbis-command-line/src/python/obis/scripts/data_mgmt_runner.py index 98d5581491ab890918aa803da1cc43a744293fd8..8ffabde717a466084ed3c292e329a2125a5fa5b0 100644 --- a/app-openbis-command-line/src/python/obis/scripts/data_mgmt_runner.py +++ b/app-openbis-command-line/src/python/obis/scripts/data_mgmt_runner.py @@ -12,25 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import click -import json import os import sys -from datetime import datetime - +from .click_util import click_echo, check_result from .. import dm -from ..dm.utils import cd -from ..dm.command_result import CommandResult, CommandException from ..dm.command_log import CommandLog +from ..dm.command_result import CommandResult, CommandException +from ..dm.utils import Type +from ..dm.utils import cd from ..dm.utils import run_shell -from .click_util import click_echo, check_result class DataMgmtRunner(object): - def __init__(self, context, halt_on_error_log=True, data_path=None, bootstrap_settings=None, check_result=True, login=True, openbis=None): + def __init__(self, context, halt_on_error_log=True, data_path=None, bootstrap_settings=None, + check_result=True, login=True, openbis=None, is_physical=False): self.context = context self.halt_on_error_log = halt_on_error_log self.data_path = data_path @@ -40,6 +38,7 @@ class DataMgmtRunner(object): self.check_result = check_result self.login = login self.openbis = openbis + self.repository_type = Type.PHYSICAL if is_physical else Type.UNKNOWN def init_paths(self, repository=None): @@ -135,4 +134,4 @@ class DataMgmtRunner(object): if self.halt_on_error_log and log.any_log_exists(): click_echo("Error: A previous command did not finish. Please check the log ({}) and remove it when you want to continue using obis".format(log.folder_path)) sys.exit(-1) - return dm.DataMgmt(openbis_config=openbis_config, git_config=git_config, log=log, debug=self.context['debug'], login=self.login, openbis=self.openbis) + return dm.DataMgmt(openbis_config=openbis_config, git_config=git_config, log=log, debug=self.context['debug'], login=self.login, openbis=self.openbis, repository_type=self.repository_type)