diff --git a/src/python/OBis/obis/dm/data_mgmt.py b/src/python/OBis/obis/dm/data_mgmt.py
index c71bc541c3215f57432af879e347c1ce5d8c8b5c..e37fc61e02734ad7e73ff4b3b7aa0a64af4087f2 100644
--- a/src/python/OBis/obis/dm/data_mgmt.py
+++ b/src/python/OBis/obis/dm/data_mgmt.py
@@ -9,10 +9,13 @@ Module implementing data management operations.
Created by Chandrasekhar Ramakrishnan on 2017-02-01.
Copyright (c) 2017 Chandrasekhar Ramakrishnan. All rights reserved.
"""
+import abc
+import os
+import shutil
import subprocess
+from contextlib import contextmanager
import pybis
-import abc
# noinspection PyPep8Naming
@@ -68,6 +71,12 @@ class CommandResult(object):
self.returncode = completed_process.returncode
self.output = completed_process.stdout.decode('utf-8').strip()
+ def __str__(self):
+ return "CommandResult({},{})".format(self.returncode, self.output)
+
+ def __repr__(self):
+ return "CommandResult({},{})".format(self.returncode, self.output)
+
def run_shell(args):
return CommandResult(subprocess.run(args, stdout=subprocess.PIPE))
@@ -78,6 +87,17 @@ def locate_command(command):
return run_shell(['type', '-p', command])
+@contextmanager
+def cd(newdir):
+ """Safe cd -- return to original dir after execution, even if an exception is raised."""
+ prevdir = os.getcwd()
+ os.chdir(os.path.expanduser(newdir))
+ try:
+ yield
+ finally:
+ os.chdir(prevdir)
+
+
class AbstractDataMgmt(metaclass=abc.ABCMeta):
"""Abstract object that implements operations.
@@ -109,6 +129,25 @@ class AbstractDataMgmt(metaclass=abc.ABCMeta):
@abc.abstractmethod
def init_data(self, path, desc=None):
+ """Initialize a data repository at the path with the description."""
+ return
+
+ @abc.abstractmethod
+ def init_analysis(self, path):
+ """Initialize an analysis repository at the path."""
+ return
+
+ @abc.abstractmethod
+ def add_content(self, path):
+ """Add content to the repository."""
+ return
+
+ @abc.abstractmethod
+ def commit(self, msg):
+ """Commit the current repo.
+
+ This issues a git commit and connects to openBIS and creates a data set in openBIS.
+ """
return
@@ -118,12 +157,20 @@ class NoGitDataMgmt(AbstractDataMgmt):
def init_data(self, path, desc=None):
self.error_raise("init data", "No git command found.")
+ def init_analysis(self, path):
+ self.error_raise("init analysis", "No git command found.")
+
+ def add_content(self, path):
+ self.error_raise("add", "No git command found.")
+
+ def commit(self, msg):
+ self.error_raise("commit", "No git command found.")
+
class GitDataMgmt(AbstractDataMgmt):
"""DataMgmt operations in normal state."""
def init_data(self, path, desc=None):
- """Initialize a data repository at the path with the description."""
result = self.git_wrapper.git_init(path)
if result.returncode != 0:
self.error(result.output)
@@ -133,6 +180,24 @@ class GitDataMgmt(AbstractDataMgmt):
self.error(result.output)
return result
+ def init_analysis(self, path):
+ result = self.git_wrapper.git_init(path)
+ if result.returncode != 0:
+ self.error(result.output)
+ return result
+
+ def add_content(self, path):
+ result = self.git_wrapper.git_add(path)
+ if result.returncode != 0:
+ self.error(result.output)
+ return result
+
+ def commit(self, msg):
+ result = self.git_wrapper.git_commit(msg)
+ if result.returncode != 0:
+ self.error(result.output)
+ return result
+
class GitWrapper(object):
"""A wrapper on commands to git."""
@@ -159,7 +224,27 @@ class GitWrapper(object):
return run_shell([self.git_path, "init", path])
def git_annex_init(self, path, desc):
- cmd = [self.git_path, "-C", path, "annex", "init"]
+ cmd = [self.git_path, "-C", path, "annex", "init", "--version=6"]
if desc is not None:
cmd.append(desc)
- return run_shell(cmd)
+ result = run_shell(cmd)
+ if result.returncode != 0:
+ return result
+
+ attributes_src = os.path.join(os.path.dirname(__file__), "git-annex-attributes")
+ attributes_dst = os.path.join(path, ".gitattributes")
+ shutil.copyfile(attributes_src, attributes_dst)
+ cmd = [self.git_path, "-C", path, "add", ".gitattributes"]
+ result = run_shell(cmd)
+ if result.returncode != 0:
+ return result
+
+ cmd = [self.git_path, "-C", path, "commit", "-m", "Initial commit."]
+ result = run_shell(cmd)
+ return result
+
+ def git_add(self, path):
+ return run_shell([self.git_path, "add", path])
+
+ def git_commit(self, msg):
+ return run_shell([self.git_path, "commit", '-m', msg])
diff --git a/src/python/OBis/obis/dm/data_mgmt_test.py b/src/python/OBis/obis/dm/data_mgmt_test.py
index 920ae053008780d71c153469dedd3a0e76c31ab6..9cf0f8c24f642d409d440d058e95cedc42f47b30 100644
--- a/src/python/OBis/obis/dm/data_mgmt_test.py
+++ b/src/python/OBis/obis/dm/data_mgmt_test.py
@@ -8,6 +8,8 @@ data_mgmt_test.py
Created by Chandrasekhar Ramakrishnan on 2017-02-02.
Copyright (c) 2017 Chandrasekhar Ramakrishnan. All rights reserved.
"""
+import os
+import shutil
from . import data_mgmt
@@ -30,18 +32,61 @@ def test_locate_command():
assert result.returncode == 1
-def test_normal_use_case(shared_dm, tmpdir):
- # The folder should not be a git repo at first.
- result = data_mgmt.run_shell(['git', '-C', str(tmpdir), 'status', ])
- assert result.returncode == 128
+def git_status(path=None, annex=False):
+ cmd = ['git']
+ if path:
+ cmd.extend(['-C', path])
+ if annex:
+ cmd.extend(['annex', 'status'])
+ else:
+ cmd.extend(['status', '--porcelain'])
+ return data_mgmt.run_shell(cmd)
- result = shared_dm.init_data(str(tmpdir), "test")
- assert result.returncode == 0
- # The folder should be a git repo now
- result = data_mgmt.run_shell(['git', '-C', str(tmpdir), 'status', str(tmpdir)])
- assert result.returncode == 0
+def test_data_use_case(shared_dm, tmpdir):
+ tmp_dir_path = str(tmpdir)
+ assert git_status(tmp_dir_path).returncode == 128 # The folder should not be a git repo at first.
- # ...and a git-annex repo as well.
- result = data_mgmt.run_shell(['git', '-C', str(tmpdir), 'annex', 'status', str(tmpdir)])
+ result = shared_dm.init_data(tmp_dir_path, "test")
assert result.returncode == 0
+
+ assert git_status(tmp_dir_path).returncode == 0 # The folder should be a git repo now
+ assert git_status(tmp_dir_path, annex=True).returncode == 0 # ...and a git-annex repo as well.
+
+ copy_test_data(tmpdir)
+
+ with data_mgmt.cd(tmp_dir_path):
+ result = shared_dm.add_content(".")
+ assert result.returncode == 0
+
+ result = git_status()
+ assert result.returncode == 0
+ assert result.output == "A snb-data.zip\nA test.txt"
+
+ result = shared_dm.commit("Added data.")
+ assert result.returncode == 0
+
+ # The zip should be in the annex
+ result = data_mgmt.run_shell(['git', 'annex', 'info', 'snb-data.zip'])
+ present_p = result.output.split('\n')[-1]
+ assert present_p == 'present: true'
+
+ # The txt files should be in git normally
+ result = data_mgmt.run_shell(['git', 'annex', 'info', 'test.txt'])
+ present_p = result.output.split(' ')[-1]
+ assert present_p == 'failed'
+
+
+def copy_test_data(tmpdir):
+ # Put some (binary) content into our new repository
+ test_data_folder = os.path.join(os.path.dirname(__file__), '..', 'test-data')
+ test_data_bin_src = os.path.join(test_data_folder, "snb-data.zip")
+ test_data_bin_path = str(tmpdir.join(os.path.basename(test_data_bin_src)))
+ shutil.copyfile(test_data_bin_src, test_data_bin_path)
+
+ # Put some text content into our new repository
+ test_data_txt_src = os.path.join(test_data_folder, "test.txt")
+ test_data_txt_path = str(tmpdir.join(os.path.basename(test_data_txt_src)))
+ shutil.copyfile(test_data_txt_src, test_data_txt_path)
+
+ return test_data_bin_path, test_data_bin_path
diff --git a/src/python/OBis/obis/dm/git-annex-attributes b/src/python/OBis/obis/dm/git-annex-attributes
new file mode 100644
index 0000000000000000000000000000000000000000..b868a67d46ecf033c1511a0c20b8c3a070775a0f
--- /dev/null
+++ b/src/python/OBis/obis/dm/git-annex-attributes
@@ -0,0 +1,4 @@
+* annex.largefiles=(largerthan=100kb)
+*.zip annex.largefiles=anything
+*.py annex.largefiles=nothing
+*.r annex.largefiles=nothing
\ No newline at end of file
diff --git a/src/python/OBis/obis/test-data/snb-data.zip b/src/python/OBis/obis/test-data/snb-data.zip
new file mode 100644
index 0000000000000000000000000000000000000000..85dd312a6322fa7dce65aa6112499a04c24fd2f0
Binary files /dev/null and b/src/python/OBis/obis/test-data/snb-data.zip differ
diff --git a/src/python/OBis/obis/test-data/test.txt b/src/python/OBis/obis/test-data/test.txt
new file mode 100644
index 0000000000000000000000000000000000000000..959fc169b360231e0f1023a9a4b297866602506a
--- /dev/null
+++ b/src/python/OBis/obis/test-data/test.txt
@@ -0,0 +1 @@
+Move along, nothing to see here.
\ No newline at end of file