Skip to content
Snippets Groups Projects
Commit d4fa3862 authored by yvesn's avatar yvesn
Browse files

SSDM-7140: using --git-dir and --work-tree to point git to data / metadata folder

parent f73d0cc4
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
obis/src/python/obis/dm/commands/clone.py
+ 1
− 2
View file @ d4fa3862
...@@ -73,8 +73,7 @@ class Clone(OpenbisCommand): ...@@ -73,8 +73,7 @@ class Clone(OpenbisCommand):
""" """
commit_hash = content_copy['gitCommitHash'] commit_hash = content_copy['gitCommitHash']
repository_folder = path.split('/')[-1] repository_folder = path.split('/')[-1]
with cd(repository_folder): return self.git_wrapper.git_checkout(commit_hash, relative_repo_path=repository_folder)
return self.git_wrapper.git_checkout(commit_hash)
def add_content_copy_to_openbis(self, repository_folder): def add_content_copy_to_openbis(self, repository_folder):
......
obis/src/python/obis/dm/data_mgmt.py
+ 19
− 10
View file @ d4fa3862
...@@ -77,6 +77,11 @@ class AbstractDataMgmt(metaclass=abc.ABCMeta): ...@@ -77,6 +77,11 @@ class AbstractDataMgmt(metaclass=abc.ABCMeta):
message = "'{}' failed. {}".format(command, reason) message = "'{}' failed. {}".format(command, reason)
raise ValueError(message) raise ValueError(message)
@abc.abstractmethod
def get_settings_resolver(self):
""" Get the settings resolver """
return
@abc.abstractmethod @abc.abstractmethod
def init_data(self, path, desc=None, create=True): def init_data(self, path, desc=None, create=True):
"""Initialize a data repository at the path with the description. """Initialize a data repository at the path with the description.
...@@ -175,6 +180,9 @@ class AbstractDataMgmt(metaclass=abc.ABCMeta): ...@@ -175,6 +180,9 @@ class AbstractDataMgmt(metaclass=abc.ABCMeta):
class NoGitDataMgmt(AbstractDataMgmt): class NoGitDataMgmt(AbstractDataMgmt):
"""DataMgmt operations when git is not available -- show error messages.""" """DataMgmt operations when git is not available -- show error messages."""
def get_settings_resolver(self):
self.error_raise("get settings resolver", "No git command found.")
def init_data(self, path, desc=None, create=True): def init_data(self, path, desc=None, create=True):
self.error_raise("init data", "No git command found.") self.error_raise("init data", "No git command found.")
...@@ -246,6 +254,10 @@ def with_restore(f): ...@@ -246,6 +254,10 @@ def with_restore(f):
class GitDataMgmt(AbstractDataMgmt): class GitDataMgmt(AbstractDataMgmt):
"""DataMgmt operations in normal state.""" """DataMgmt operations in normal state."""
def get_settings_resolver(self):
return self.settings_resolver
def setup_local_settings(self, all_settings): def setup_local_settings(self, all_settings):
self.settings_resolver.set_resolver_location_roots('data_set', '.') self.settings_resolver.set_resolver_location_roots('data_set', '.')
for resolver_type, settings in all_settings.items(): for resolver_type, settings in all_settings.items():
...@@ -273,8 +285,6 @@ class GitDataMgmt(AbstractDataMgmt): ...@@ -273,8 +285,6 @@ class GitDataMgmt(AbstractDataMgmt):
return self.settings_resolver.repository.config_dict().get('id') return self.settings_resolver.repository.config_dict().get('id')
def init_data(self, relative_path, desc=None, create=True, apply_config=False): def init_data(self, relative_path, desc=None, create=True, apply_config=False):
if not os.path.exists(relative_path) and create:
os.mkdir(relative_path)
result = self.git_wrapper.git_init(relative_path) result = self.git_wrapper.git_init(relative_path)
if result.failure(): if result.failure():
return result return result
...@@ -282,14 +292,13 @@ class GitDataMgmt(AbstractDataMgmt): ...@@ -282,14 +292,13 @@ class GitDataMgmt(AbstractDataMgmt):
result = self.git_wrapper.git_annex_init(relative_path, desc, git_annex_backend) result = self.git_wrapper.git_annex_init(relative_path, desc, git_annex_backend)
if result.failure(): if result.failure():
return result return result
with cd(relative_path): result = self.git_wrapper.initial_commit()
result = self.git_wrapper.initial_commit() if result.failure():
if result.failure(): return result
return result # Update the resolvers location
# Update the resolvers location self.settings_resolver.set_resolver_location_roots('data_set', '.')
self.settings_resolver.set_resolver_location_roots('data_set', '.') self.settings_resolver.copy_global_to_local()
self.settings_resolver.copy_global_to_local() self.commit_metadata_updates('local with global')
self.commit_metadata_updates('local with global')
return CommandResult(returncode=0, output="") return CommandResult(returncode=0, output="")
......
obis/src/python/obis/dm/git.py
+ 21
− 13
View file @ d4fa3862
...@@ -15,11 +15,14 @@ class GitWrapper(object): ...@@ -15,11 +15,14 @@ class GitWrapper(object):
self.data_path = data_path self.data_path = data_path
self.metadata_path = metadata_path self.metadata_path = metadata_path
def _git(self, params, strip_leading_whitespace=True): def _git(self, params, strip_leading_whitespace=True, relative_repo_path=''):
return run_shell([self.git_path] + params, strip_leading_whitespace=strip_leading_whitespace) cmd = [self.git_path]
if self.data_path is not None and self.metadata_path is not None:
git_dir = os.path.join(self.metadata_path, relative_repo_path, '.git') # TODO add repo folder if not already in
cmd += ['--work-tree', self.data_path, '--git-dir', git_dir]
cmd += params
return run_shell(cmd, strip_leading_whitespace=strip_leading_whitespace)
def _git_annex(self, params):
return run_shell([self.git_annex_path] + params)
def can_run(self): def can_run(self):
"""Return true if the perquisites are satisfied to run""" """Return true if the perquisites are satisfied to run"""
...@@ -30,13 +33,15 @@ class GitWrapper(object): ...@@ -30,13 +33,15 @@ class GitWrapper(object):
if self._git(['help']).failure(): if self._git(['help']).failure():
# git help should have a returncode of 0 # git help should have a returncode of 0
return False return False
if self._git_annex(['help']).failure(): if self._git(['annex', 'help']).failure():
# git help should have a returncode of 0 # git help should have a returncode of 0
return False return False
return True return True
# TODO remove path
def git_init(self, path): def git_init(self, path):
return self._git(["init", path]) return self._git(["init"])
# return self._git(["init", path])
def git_status(self, path=None): def git_status(self, path=None):
if path is None: if path is None:
...@@ -45,7 +50,7 @@ class GitWrapper(object): ...@@ -45,7 +50,7 @@ class GitWrapper(object):
return self._git(["annex", "status", path], strip_leading_whitespace=False) return self._git(["annex", "status", path], strip_leading_whitespace=False)
def git_annex_init(self, path, desc, git_annex_backend=None): def git_annex_init(self, path, desc, git_annex_backend=None):
cmd = ["-C", path, "annex", "init", "--version=5"] cmd = ["annex", "init", "--version=5"]
if desc is not None: if desc is not None:
cmd.append(desc) cmd.append(desc)
result = self._git(cmd) result = self._git(cmd)
...@@ -53,26 +58,26 @@ class GitWrapper(object): ...@@ -53,26 +58,26 @@ class GitWrapper(object):
return result return result
# annex.thin to avoid copying big files # annex.thin to avoid copying big files
cmd = ["-C", path, "config", "annex.thin", "true"] cmd = ["config", "annex.thin", "true"]
result = self._git(cmd) result = self._git(cmd)
if result.failure(): if result.failure():
return result return result
# direct mode so annex uses hard links instead of soft links # direct mode so annex uses hard links instead of soft links
cmd = ["-C", path, "annex", "direct"] cmd = ["annex", "direct"]
result = self._git(cmd) result = self._git(cmd)
if result.failure(): if result.failure():
return result return result
# re-enable the repository to be used with git directly # re-enable the repository to be used with git directly
# though we need to know what we do since annex can lead to unexpected behaviour # though we need to know what we do since annex can lead to unexpected behaviour
cmd = ["-C", path, "config", "--unset", "core.bare"] cmd = ["config", "--unset", "core.bare"]
result = self._git(cmd) result = self._git(cmd)
if result.failure(): if result.failure():
return result return result
attributes_src = os.path.join(os.path.dirname(__file__), "git-annex-attributes") attributes_src = os.path.join(os.path.dirname(__file__), "git-annex-attributes")
attributes_dst = os.path.join(path, ".git/info/attributes") attributes_dst = '.git/info/attributes'
shutil.copyfile(attributes_src, attributes_dst) shutil.copyfile(attributes_src, attributes_dst)
self._apply_git_annex_backend(attributes_dst, git_annex_backend) self._apply_git_annex_backend(attributes_dst, git_annex_backend)
...@@ -112,6 +117,7 @@ class GitWrapper(object): ...@@ -112,6 +117,7 @@ class GitWrapper(object):
def git_commit(self, msg): def git_commit(self, msg):
return self._git(["commit", '-m', msg]) return self._git(["commit", '-m', msg])
# TODO where is this used - does is make sense?
def git_top_level_path(self): def git_top_level_path(self):
return self._git(['rev-parse', '--show-toplevel']) return self._git(['rev-parse', '--show-toplevel'])
...@@ -121,8 +127,10 @@ class GitWrapper(object): ...@@ -121,8 +127,10 @@ class GitWrapper(object):
def git_ls_tree(self): def git_ls_tree(self):
return self._git(['ls-tree', '--full-tree', '-r', 'HEAD']) return self._git(['ls-tree', '--full-tree', '-r', 'HEAD'])
def git_checkout(self, path): def git_checkout(self, path_or_hash, relative_repo_path=''):
return self._git(["checkout", path]) if relative_repo_path:
return self._git(['checkout', path_or_hash], relative_repo_path=relative_repo_path)
return self._git(["checkout", path_or_hash])
def git_reset_to(self, commit_hash): def git_reset_to(self, commit_hash):
return self._git(['reset', commit_hash]) return self._git(['reset', commit_hash])
......
obis/src/python/obis/scripts/cli.py
+ 2
− 2
View file @ d4fa3862
...@@ -59,13 +59,13 @@ def init_data_impl(ctx, repository, desc): ...@@ -59,13 +59,13 @@ def init_data_impl(ctx, repository, desc):
repository = "." repository = "."
click_echo("init_data {}".format(repository)) click_echo("init_data {}".format(repository))
desc = desc if desc != "" else None desc = desc if desc != "" else None
return ctx.obj['runner'].run("init_data", lambda dm: dm.init_data(repository, desc, create=True)) return ctx.obj['runner'].run("init_data", lambda dm: dm.init_data(repository, desc, create=True), repository)
def init_analysis_impl(ctx, parent, repository, description): def init_analysis_impl(ctx, parent, repository, description):
click_echo("init_analysis {}".format(repository)) click_echo("init_analysis {}".format(repository))
description = description if description != "" else None description = description if description != "" else None
return ctx.obj['runner'].run("init_analysis", lambda dm: dm.init_analysis(repository, parent, description, create=True)) return ctx.obj['runner'].run("init_analysis", lambda dm: dm.init_analysis(repository, parent, description, create=True), repository)
# settings commands # settings commands
......
obis/src/python/obis/scripts/data_mgmt_runner.py
+ 27
− 17
View file @ d4fa3862
...@@ -21,38 +21,44 @@ class DataMgmtRunner(object): ...@@ -21,38 +21,44 @@ class DataMgmtRunner(object):
self.context = context self.context = context
self.halt_on_error_log = halt_on_error_log self.halt_on_error_log = halt_on_error_log
self.data_path = None self.data_path = None
self.metadta_path = None self.metadata_path = None
def init_paths(self, repository=None): def init_paths(self, repository=None):
if self.data_path is not None and self.metadata_path is not None:
return
# data path # data path
self.data_path = run_shell(['pwd'], raise_exception_on_failure=True).output self.data_path = run_shell(['pwd'], raise_exception_on_failure=True).output
if repository is not None: if repository is not None:
self.data_path = os.path.join(self.data_path, repository) self.data_path = os.path.join(self.data_path, repository)
# metadata path # metadata path
self.metadta_path = self.data_path self.metadata_path = self.data_path
obis_metadata_folder = self.get_settings_resolver(do_cd=False).config.config_dict().get('obis_metadata_folder') obis_metadata_folder = self.get_settings_resolver(do_cd=False).config.config_dict().get('obis_metadata_folder')
if obis_metadata_folder is not None: if obis_metadata_folder is not None:
self._validate_obis_metadata_folder(obis_metadata_folder) result = self._validate_obis_metadata_folder(obis_metadata_folder)
self.metadta_path = os.path.join(obis_metadata_folder, self.data_path[1:]) if result.failure():
if not os.path.exists(self.metadta_path): click_echo(result.output)
os.makedirs(self.metadta_path) else:
self.metadata_path = os.path.join(obis_metadata_folder, self.data_path[1:])
if not os.path.exists(self.metadata_path):
os.makedirs(self.metadata_path)
def _validate_obis_metadata_folder(self, obis_metadata_folder): def _validate_obis_metadata_folder(self, obis_metadata_folder):
if not os.path.isabs(obis_metadata_folder): if not os.path.isabs(obis_metadata_folder):
raise CommandException(CommandResult( return CommandResult(
returncode=-1, returncode=-1,
output="obis_metadata_folder must be absolute: {}".format(obis_metadata_folder))) output="Ignoring obis_metadata_folder. Must be absolute but is: {}".format(obis_metadata_folder))
if not os.path.exists(obis_metadata_folder): if not os.path.exists(obis_metadata_folder):
raise CommandException(CommandResult( return CommandResult(
returncode=-1, returncode=-1,
output="obis_metadata_folder does not exist: {}".format(obis_metadata_folder))) output="Ignoring obis_metadata_folder. Folder does not exist: {}".format(obis_metadata_folder))
return CommandResult(returncode=0, output="")
def run(self, command, function, repository=None): def run(self, command, function, repository=None):
self.init_paths(repository) self.init_paths(repository)
with cd(self.metadta_path): with cd(self.metadata_path):
result = self._run(function) result = self._run(function)
return check_result(command, result) return check_result(command, result)
...@@ -70,27 +76,31 @@ class DataMgmtRunner(object): ...@@ -70,27 +76,31 @@ class DataMgmtRunner(object):
def get_settings(self, repository=None): def get_settings(self, repository=None):
self.init_paths() self.init_paths()
with cd(self.metadta_path): with cd(self.metadata_path):
return self.get_settings_resolver().config_dict() return self.get_settings_resolver().config_dict()
def get_settings_resolver(self, do_cd=True): def get_settings_resolver(self, do_cd=True):
if do_cd: if do_cd:
self.init_paths() self.init_paths()
with cd(self.metadta_path): with cd(self.metadata_path):
return self._get_dm().settings_resolver return self._get_dm().get_settings_resolver()
else: else:
return self._get_dm().settings_resolver return self._get_dm().get_settings_resolver()
def config(self, resolver, is_global, is_data_set_property, prop, value, set, get, clear): def config(self, resolver, is_global, is_data_set_property, prop, value, set, get, clear):
self.init_paths() self.init_paths()
with cd(self.metadta_path): with cd(self.metadata_path):
self._get_dm().config(resolver, is_global, is_data_set_property, prop, value, set, get, clear) self._get_dm().config(resolver, is_global, is_data_set_property, prop, value, set, get, clear)
def _get_dm(self): def _get_dm(self):
git_config = {'find_git': True} git_config = {
'find_git': True,
'data_path': self.data_path,
'metadata_path': self.metadata_path
}
openbis_config = {} openbis_config = {}
if self.context.get('verify_certificates') is not None: if self.context.get('verify_certificates') is not None:
openbis_config['verify_certificates'] = self.context['verify_certificates'] openbis_config['verify_certificates'] = self.context['verify_certificates']
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment