SSDM-13330: Added PhysicalDataMgmt for handling data stored in DSS,...

SSDM-13330: Added PhysicalDataMgmt for handling data stored in DSS, implemented init command for physical data. Refactored code.

app-openbis-command-line/src/python/obis/dm/.obis/config.json

-{"data_set_id": "20170814175159563859-9212", "data_set_type": "DS_TYPE", "object_id": "/SAMPLE/ID", "openbis_url": "http://localhost:8888"}
\ No newline at end of file

app-openbis-command-line/src/python/obis/dm/config.py

@@ -135,6 +135,7 @@ class ConfigEnv(object):
         self.add_param(ConfigParam(name='obis_metadata_folder'))
         self.add_param(ConfigParam(name='openbis_token'))
         self.add_param(ConfigParam(name='session_name'))
+        self.add_param(ConfigParam(name='is_physical', default_value=False))
 
     def add_param(self, param):
         self.params[param.name] = param

app-openbis-command-line/src/python/obis/dm/data_mgmt.py

@@ -2,59 +2,49 @@
 # -*- coding: utf-8 -*-
 
 #   Copyright ETH 2018 - 2023 Zürich, Scientific IT Services
-# 
+#
 #   Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
 #   You may obtain a copy of the License at
-# 
+#
 #        http://www.apache.org/licenses/LICENSE-2.0
-#   
+#
 #   Unless required by applicable law or agreed to in writing, software
 #   distributed under the License is distributed on an "AS IS" BASIS,
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 #
-
-"""
-data_mgmt.py
-
-Module implementing data management operations.
-
-Created by Chandrasekhar Ramakrishnan on 2017-02-01.
-Copyright (c) 2017 Chandrasekhar Ramakrishnan. All rights reserved.
-"""
 import abc
 import json
 import os
 import shutil
-import traceback
-import pybis
-import requests
 import signal
 import sys
 from pathlib import Path
+
+import requests
+
 from . import config as dm_config
+from .command_result import CommandResult
 from .commands.addref import Addref
-from .commands.removeref import Removeref
 from .commands.clone import Clone
+from .commands.download import Download
 from .commands.move import Move
 from .commands.openbis_sync import OpenbisSync
-from .commands.download import Download
-from .command_log import CommandLog
-from .command_result import CommandResult
-from .command_result import CommandException
+from .commands.removeref import Removeref
 from .git import GitWrapper
-from .utils import default_echo
+from .utils import Type
+from .utils import cd
 from .utils import complete_git_config
 from .utils import complete_openbis_config
-from .utils import cd
-from ..scripts import cli
+from .utils import default_echo
 from ..scripts.click_util import click_echo, check_result
 
 
 # noinspection PyPep8Naming
-def DataMgmt(echo_func=None, settings_resolver=None, openbis_config={}, git_config={}, openbis=None, log=None, debug=False, login=True):
+def DataMgmt(echo_func=None, settings_resolver=None, openbis_config={}, git_config={},
+             openbis=None, log=None, debug=False, login=True, repository_type=Type.UNKNOWN):
     """Factory method for DataMgmt instances"""
 
     echo_func = echo_func if echo_func is not None else default_echo
@@ -63,19 +53,31 @@ def DataMgmt(echo_func=None, settings_resolver=None, openbis_config={}, git_conf
     metadata_path = git_config['metadata_path']
     invocation_path = git_config['invocation_path']
 
-    complete_git_config(git_config)
-    git_wrapper = GitWrapper(**git_config)
-    if not git_wrapper.can_run():
-        # TODO We could just as well throw an error here instead of creating
-        #      creating the NoGitDataMgmt which will fail later.
-        return NoGitDataMgmt(settings_resolver, None, git_wrapper, openbis, log, data_path, metadata_path, invocation_path)
-
     if settings_resolver is None:
         settings_resolver = dm_config.SettingsResolver()
 
-    complete_openbis_config(openbis_config, settings_resolver)
+    if repository_type == Type.UNKNOWN:
+        if os.path.exists('.obis'):
+            config_dict = settings_resolver.config.config_dict()
+            if config_dict['is_physical'] is True:
+                repository_type = Type.PHYSICAL
+            else:
+                repository_type = Type.LINK
+        else:
+            repository_type = Type.LINK
+
+    if repository_type == Type.PHYSICAL:
+        return PhysicalDataMgmt(settings_resolver, None, None, openbis, log, data_path, metadata_path, invocation_path)
+    else:
+        complete_git_config(git_config)
+        git_wrapper = GitWrapper(**git_config)
+        if not git_wrapper.can_run():
+            # TODO We could just as well throw an error here instead of creating
+            #      creating the NoGitDataMgmt which will fail later.
+            return NoGitDataMgmt(settings_resolver, None, git_wrapper, openbis, log, data_path, metadata_path, invocation_path)
 
-    return GitDataMgmt(settings_resolver, openbis_config, git_wrapper, openbis, log, data_path, metadata_path, invocation_path, debug, login)
+        complete_openbis_config(openbis_config, settings_resolver)
+        return GitDataMgmt(settings_resolver, openbis_config, git_wrapper, openbis, log, data_path, metadata_path, invocation_path, debug, login)
 
 
 class AbstractDataMgmt(metaclass=abc.ABCMeta):
@@ -325,6 +327,7 @@ class GitDataMgmt(AbstractDataMgmt):
 
     def init_data(self, desc=None):
         # check that repository does not already exist
+        # TODO remove .git check after physical flow is implemented
         if os.path.exists('.obis') and os.path.exists('.git'):
             return CommandResult(returncode=-1, output="Folder is already an obis repository.")
         result = self.git_wrapper.git_init()
@@ -524,3 +527,48 @@ class GitDataMgmt(AbstractDataMgmt):
             return CommandResult(returncode=-1, output="Error: " + str(e))
         else:
             return CommandResult(returncode=0, output="")
+
+
+class PhysicalDataMgmt(AbstractDataMgmt):
+    """DataMgmt operations for DSS-stored data."""
+
+    def get_settings_resolver(self):
+        return dm_config.SettingsResolver()
+
+    def setup_local_settings(self, all_settings):
+        self.error_raise("setup local settings", "Not implemented.")
+
+    def init_data(self, desc=None):
+        if os.path.exists('.obis'):
+            return CommandResult(returncode=-1, output="Folder is already an obis repository.")
+        self.settings_resolver.set_resolver_location_roots('data_set', '.')
+        self.settings_resolver.copy_global_to_local()
+        self.settings_resolver.config.set_value_for_parameter("is_physical", True, "local")
+        return CommandResult(returncode=0, output="Physical obis repository initialized!")
+
+    def init_analysis(self, parent_folder, desc=None):
+        self.error_raise("init analysis", "Not implemented.")
+
+    def commit(self, msg, auto_add=True, sync=True):
+        self.error_raise("commit", "Not implemented.")
+
+    def sync(self):
+        self.error_raise("sync", "Not implemented.")
+
+    def status(self):
+        self.error_raise("status", "Not implemented.")
+
+    def clone(self, data_set_id, ssh_user, content_copy_index, skip_integrity_check):
+        self.error_raise("clone", "Not implemented.")
+
+    def move(self, data_set_id, ssh_user, content_copy_index, skip_integrity_check):
+        self.error_raise("move", "Not implemented.")
+
+    def addref(self):
+        self.error_raise("addref", "Not implemented.")
+
+    def removeref(self, data_set_id=None):
+        self.error_raise("removeref", "Not implemented.")
+
+    def download(self, data_set_id, content_copy_index, file, skip_integrity_check):
+        self.error_raise("download", "Not implemented.")
\ No newline at end of file

app-openbis-command-line/src/python/obis/dm/git.py

@@ -77,7 +77,7 @@ class GitWrapper(object):
         else:
             return self._git(["annex", "status", path], strip_leading_whitespace=False)
 
-    def git_annex_init(self, desc, git_annex_backend=None):
+    def git_annex_backend(self, desc, git_annex_backend=None):
         """ Configures annex in a git repository."""
 
         # We use annex --version=5 since that works better with big files. Version 

app-openbis-command-line/src/python/obis/dm/utils.py

@@ -12,12 +12,20 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 #
-import subprocess
 import os
+import subprocess
 from contextlib import contextmanager
+from enum import Enum
+
 from .command_result import CommandResult, CommandException
 
 
+class Type(Enum):
+    """Enum representing which what type of repository it is"""
+    LINK = 1,
+    PHYSICAL = 2,
+    UNKNOWN = 3
+
 def complete_openbis_config(config, resolver, local_only=True):
     """Add default values for empty entries in the config."""
     config_dict = resolver.config.config_dict(local_only)
@@ -27,6 +35,8 @@ def complete_openbis_config(config, resolver, local_only=True):
         config['verify_certificates'] = config_dict['verify_certificates']
     if config.get('token') is None:
         config['token'] = None
+    if config.get('is_physical') is None:
+        config['is_physical'] = None
     if config.get('allow_http_but_do_not_use_this_in_production_and_only_within_safe_networks') is None:
         config['allow_http_but_do_not_use_this_in_production_and_only_within_safe_networks'] = not config_dict['allow_only_https']
 

app-openbis-command-line/src/python/obis/scripts/cli.py

@@ -20,24 +20,19 @@
 cli.py
 
 The module that implements the CLI for obis.
-
-
-Created by Chandrasekhar Ramakrishnan on 2017-01-27.
-Copyright (c) 2017 Chandrasekhar Ramakrishnan. All rights reserved.
 """
 import json
 import os
 from datetime import datetime
-from requests import ConnectionError
 
 import click
 from dateutil.relativedelta import relativedelta
 from pybis import Openbis
+from requests import ConnectionError
 
-from ..dm.command_result import CommandResult
-from ..dm.utils import cd
 from .click_util import click_echo
 from .data_mgmt_runner import DataMgmtRunner
+from ..dm.command_result import CommandResult
 
 
 def click_progress(progress_data):
@@ -448,25 +443,32 @@ def commit(ctx, msg, auto_add, ignore_missing_parent, repository):
 
 
 _init_params = [
-    click.argument('repository', type=click.Path(
+    click.argument('repository_path', type=click.Path(
         exists=False, file_okay=False), required=False),
     click.argument('description', default=""),
+
 ]
 
 
 @repository.command("init", short_help="Initialize the folder as a data repository.")
 @click.pass_context
 @add_params(_init_params)
-def repository_init(ctx, repository, description):
-    return init_data_impl(ctx, repository, description)
+def repository_init(ctx, repository_path, description):
+    return init_data_impl(ctx, repository_path, description)
+
+
+_init_params_physical = \
+    _init_params + \
+    [click.option('-p', '--physical', 'is_physical', default=False, is_flag=True,
+                  help='If parent data set is missing, ignore it.')]
 
 
 @cli.command(short_help="Initialize the folder as a data repository.")
 @click.pass_context
-@add_params(_init_params)
-def init(ctx, repository, description):
-    ctx.obj['runner'] = DataMgmtRunner(ctx.obj, halt_on_error_log=False)
-    ctx.invoke(repository_init, repository=repository, description=description)
+@add_params(_init_params_physical)
+def init(ctx, repository_path, description, is_physical):
+    ctx.obj['runner'] = DataMgmtRunner(ctx.obj, halt_on_error_log=False, is_physical=is_physical)
+    ctx.invoke(repository_init, repository_path=repository_path, description=description)
 
 # init analysis
 
@@ -481,17 +483,17 @@ _init_analysis_params += _init_params
 @repository.command("init_analysis", short_help="Initialize the folder as an analysis folder.")
 @click.pass_context
 @add_params(_init_analysis_params)
-def repository_init_analysis(ctx, parent, repository, description):
-    return init_analysis_impl(ctx, parent, repository, description)
+def repository_init_analysis(ctx, parent, repository_path, description):
+    return init_analysis_impl(ctx, parent, repository_path, description)
 
 
 @cli.command(name='init_analysis', short_help="Initialize the folder as an analysis folder.")
 @click.pass_context
 @add_params(_init_analysis_params)
-def init_analysis(ctx, parent, repository, description):
+def init_analysis(ctx, parent, repository_path, description):
     ctx.obj['runner'] = DataMgmtRunner(ctx.obj, halt_on_error_log=False)
     ctx.invoke(repository_init_analysis, parent=parent,
-               repository=repository, description=description)
+               repository_path=repository_path, description=description)
 
 # status
 

app-openbis-command-line/src/python/obis/scripts/data_mgmt_runner.py

@@ -12,25 +12,23 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 #
-import click
-import json
 import os
 import sys
 
-from datetime import datetime
-
+from .click_util import click_echo, check_result
 from .. import dm
-from ..dm.utils import cd
-from ..dm.command_result import CommandResult, CommandException
 from ..dm.command_log import CommandLog
+from ..dm.command_result import CommandResult, CommandException
+from ..dm.utils import Type
+from ..dm.utils import cd
 from ..dm.utils import run_shell
-from .click_util import click_echo, check_result
 
 
 class DataMgmtRunner(object):
 
 
-    def __init__(self, context, halt_on_error_log=True, data_path=None, bootstrap_settings=None, check_result=True, login=True, openbis=None):
+    def __init__(self, context, halt_on_error_log=True, data_path=None, bootstrap_settings=None,
+                 check_result=True, login=True, openbis=None, is_physical=False):
         self.context = context
         self.halt_on_error_log = halt_on_error_log
         self.data_path = data_path
@@ -40,6 +38,7 @@ class DataMgmtRunner(object):
         self.check_result = check_result
         self.login = login
         self.openbis = openbis
+        self.repository_type = Type.PHYSICAL if is_physical else Type.UNKNOWN
 
 
     def init_paths(self, repository=None):
@@ -135,4 +134,4 @@ class DataMgmtRunner(object):
         if self.halt_on_error_log and log.any_log_exists():
             click_echo("Error: A previous command did not finish. Please check the log ({}) and remove it when you want to continue using obis".format(log.folder_path))
             sys.exit(-1)
-        return dm.DataMgmt(openbis_config=openbis_config, git_config=git_config, log=log, debug=self.context['debug'], login=self.login, openbis=self.openbis)
+        return dm.DataMgmt(openbis_config=openbis_config, git_config=git_config, log=log, debug=self.context['debug'], login=self.login, openbis=self.openbis, repository_type=self.repository_type)