SSDM-4670: Basic commands for working with git/git-annex

src/python/OBis/obis/dm/data_mgmt.py

@@ -9,10 +9,13 @@ Module implementing data management operations.
 Created by Chandrasekhar Ramakrishnan on 2017-02-01.
 Copyright (c) 2017 Chandrasekhar Ramakrishnan. All rights reserved.
 """
+import abc
+import os
+import shutil
 import subprocess
+from contextlib import contextmanager
 
 import pybis
-import abc
 
 
 # noinspection PyPep8Naming
@@ -68,6 +71,12 @@ class CommandResult(object):
         self.returncode = completed_process.returncode
         self.output = completed_process.stdout.decode('utf-8').strip()
 
+    def __str__(self):
+        return "CommandResult({},{})".format(self.returncode, self.output)
+
+    def __repr__(self):
+        return "CommandResult({},{})".format(self.returncode, self.output)
+
 
 def run_shell(args):
     return CommandResult(subprocess.run(args, stdout=subprocess.PIPE))
@@ -78,6 +87,17 @@ def locate_command(command):
     return run_shell(['type', '-p', command])
 
 
+@contextmanager
+def cd(newdir):
+    """Safe cd -- return to original dir after execution, even if an exception is raised."""
+    prevdir = os.getcwd()
+    os.chdir(os.path.expanduser(newdir))
+    try:
+        yield
+    finally:
+        os.chdir(prevdir)
+
+
 class AbstractDataMgmt(metaclass=abc.ABCMeta):
     """Abstract object that implements operations.
 
@@ -109,6 +129,25 @@ class AbstractDataMgmt(metaclass=abc.ABCMeta):
 
     @abc.abstractmethod
     def init_data(self, path, desc=None):
+        """Initialize a data repository at the path with the description."""
+        return
+
+    @abc.abstractmethod
+    def init_analysis(self, path):
+        """Initialize an analysis repository at the path."""
+        return
+
+    @abc.abstractmethod
+    def add_content(self, path):
+        """Add content to the repository."""
+        return
+
+    @abc.abstractmethod
+    def commit(self, msg):
+        """Commit the current repo.
+
+        This issues a git commit and connects to openBIS and creates a data set in openBIS.
+        """
         return
 
 
@@ -118,12 +157,20 @@ class NoGitDataMgmt(AbstractDataMgmt):
     def init_data(self, path, desc=None):
         self.error_raise("init data", "No git command found.")
 
+    def init_analysis(self, path):
+        self.error_raise("init analysis", "No git command found.")
+
+    def add_content(self, path):
+        self.error_raise("add", "No git command found.")
+
+    def commit(self, msg):
+        self.error_raise("commit", "No git command found.")
+
 
 class GitDataMgmt(AbstractDataMgmt):
     """DataMgmt operations in normal state."""
 
     def init_data(self, path, desc=None):
-        """Initialize a data repository at the path with the description."""
         result = self.git_wrapper.git_init(path)
         if result.returncode != 0:
             self.error(result.output)
@@ -133,6 +180,24 @@ class GitDataMgmt(AbstractDataMgmt):
             self.error(result.output)
         return result
 
+    def init_analysis(self, path):
+        result = self.git_wrapper.git_init(path)
+        if result.returncode != 0:
+            self.error(result.output)
+        return result
+
+    def add_content(self, path):
+        result = self.git_wrapper.git_add(path)
+        if result.returncode != 0:
+            self.error(result.output)
+        return result
+
+    def commit(self, msg):
+        result = self.git_wrapper.git_commit(msg)
+        if result.returncode != 0:
+            self.error(result.output)
+        return result
+
 
 class GitWrapper(object):
     """A wrapper on commands to git."""
@@ -159,7 +224,27 @@ class GitWrapper(object):
         return run_shell([self.git_path, "init", path])
 
     def git_annex_init(self, path, desc):
-        cmd = [self.git_path, "-C", path, "annex", "init"]
+        cmd = [self.git_path, "-C", path, "annex", "init", "--version=6"]
         if desc is not None:
             cmd.append(desc)
-        return run_shell(cmd)
+        result = run_shell(cmd)
+        if result.returncode != 0:
+            return result
+
+        attributes_src = os.path.join(os.path.dirname(__file__), "git-annex-attributes")
+        attributes_dst = os.path.join(path, ".gitattributes")
+        shutil.copyfile(attributes_src, attributes_dst)
+        cmd = [self.git_path, "-C", path, "add", ".gitattributes"]
+        result = run_shell(cmd)
+        if result.returncode != 0:
+            return result
+
+        cmd = [self.git_path, "-C", path, "commit", "-m", "Initial commit."]
+        result = run_shell(cmd)
+        return result
+
+    def git_add(self, path):
+        return run_shell([self.git_path, "add", path])
+
+    def git_commit(self, msg):
+        return run_shell([self.git_path, "commit", '-m', msg])

src/python/OBis/obis/dm/data_mgmt_test.py

@@ -8,6 +8,8 @@ data_mgmt_test.py
 Created by Chandrasekhar Ramakrishnan on 2017-02-02.
 Copyright (c) 2017 Chandrasekhar Ramakrishnan. All rights reserved.
 """
+import os
+import shutil
 
 from . import data_mgmt
 
@@ -30,18 +32,61 @@ def test_locate_command():
     assert result.returncode == 1
 
 
-def test_normal_use_case(shared_dm, tmpdir):
-    # The folder should not be a git repo at first.
-    result = data_mgmt.run_shell(['git', '-C', str(tmpdir), 'status', ])
-    assert result.returncode == 128
+def git_status(path=None, annex=False):
+    cmd = ['git']
+    if path:
+        cmd.extend(['-C', path])
+    if annex:
+        cmd.extend(['annex', 'status'])
+    else:
+        cmd.extend(['status', '--porcelain'])
+    return data_mgmt.run_shell(cmd)
 
-    result = shared_dm.init_data(str(tmpdir), "test")
-    assert result.returncode == 0
 
-    # The folder should be a git repo now
-    result = data_mgmt.run_shell(['git', '-C', str(tmpdir), 'status', str(tmpdir)])
-    assert result.returncode == 0
+def test_data_use_case(shared_dm, tmpdir):
+    tmp_dir_path = str(tmpdir)
+    assert git_status(tmp_dir_path).returncode == 128  # The folder should not be a git repo at first.
 
-    # ...and a git-annex repo as well.
-    result = data_mgmt.run_shell(['git', '-C', str(tmpdir), 'annex', 'status', str(tmpdir)])
+    result = shared_dm.init_data(tmp_dir_path, "test")
     assert result.returncode == 0
+
+    assert git_status(tmp_dir_path).returncode == 0  # The folder should be a git repo now
+    assert git_status(tmp_dir_path, annex=True).returncode == 0  # ...and a git-annex repo as well.
+
+    copy_test_data(tmpdir)
+
+    with data_mgmt.cd(tmp_dir_path):
+        result = shared_dm.add_content(".")
+        assert result.returncode == 0
+
+        result = git_status()
+        assert result.returncode == 0
+        assert result.output == "A  snb-data.zip\nA  test.txt"
+
+        result = shared_dm.commit("Added data.")
+        assert result.returncode == 0
+
+        # The zip should be in the annex
+        result = data_mgmt.run_shell(['git', 'annex', 'info', 'snb-data.zip'])
+        present_p = result.output.split('\n')[-1]
+        assert present_p == 'present: true'
+
+        # The txt files should be in git normally
+        result = data_mgmt.run_shell(['git', 'annex', 'info', 'test.txt'])
+        present_p = result.output.split(' ')[-1]
+        assert present_p == 'failed'
+
+
+def copy_test_data(tmpdir):
+    # Put some (binary) content into our new repository
+    test_data_folder = os.path.join(os.path.dirname(__file__), '..', 'test-data')
+    test_data_bin_src = os.path.join(test_data_folder, "snb-data.zip")
+    test_data_bin_path = str(tmpdir.join(os.path.basename(test_data_bin_src)))
+    shutil.copyfile(test_data_bin_src, test_data_bin_path)
+
+    # Put some text content into our new repository
+    test_data_txt_src = os.path.join(test_data_folder, "test.txt")
+    test_data_txt_path = str(tmpdir.join(os.path.basename(test_data_txt_src)))
+    shutil.copyfile(test_data_txt_src, test_data_txt_path)
+
+    return test_data_bin_path, test_data_bin_path

src/python/OBis/obis/dm/git-annex-attributes

+* annex.largefiles=(largerthan=100kb)
+*.zip annex.largefiles=anything
+*.py annex.largefiles=nothing
+*.r annex.largefiles=nothing
\ No newline at end of file

src/python/OBis/obis/test-data/test.txt

+Move along, nothing to see here.
\ No newline at end of file