From 5d733ad58a94197330b38ad76263500d717538a9 Mon Sep 17 00:00:00 2001
From: Chandrasekhar Ramakrishnan <chandrasekhar.ramakrishnan@id.ethz.ch>
Date: Mon, 13 Mar 2017 16:25:44 +0100
Subject: [PATCH] SSDM-4670: Sync with openBIS server.
---
src/python/OBis/obis/conftest.py | 7 +-
src/python/OBis/obis/dm/data_mgmt.py | 120 ++++++++++++++++++++--
src/python/OBis/obis/dm/data_mgmt_test.py | 41 +++++++-
src/python/OBis/obis/dm/repo_test.py | 6 +-
src/python/OBis/obis/scripts/cli.py | 23 ++++-
5 files changed, 174 insertions(+), 23 deletions(-)
diff --git a/src/python/OBis/obis/conftest.py b/src/python/OBis/obis/conftest.py
index d581c429afd..26ef109812c 100644
--- a/src/python/OBis/obis/conftest.py
+++ b/src/python/OBis/obis/conftest.py
@@ -12,9 +12,4 @@ Copyright (c) 2017 Chandrasekhar Ramakrishnan. All rights reserved.
import pytest
from . import dm
-
-
-@pytest.fixture(scope="session")
-def shared_dm():
- git_config = {'find_git': True}
- return dm.DataMgmt(git_config=git_config)
+from unittest.mock import Mock
diff --git a/src/python/OBis/obis/dm/data_mgmt.py b/src/python/OBis/obis/dm/data_mgmt.py
index a0648350e7d..e5bc38334eb 100644
--- a/src/python/OBis/obis/dm/data_mgmt.py
+++ b/src/python/OBis/obis/dm/data_mgmt.py
@@ -16,6 +16,7 @@ import subprocess
from contextlib import contextmanager
from . import config as dm_config
import traceback
+import getpass
import pybis
@@ -225,12 +226,8 @@ class GitDataMgmt(AbstractDataMgmt):
return result
def sync(self):
- # TODO create a data set in openBIS
- # - check if openBIS has been setup. If not, prompt the user to set up openbis.
- # - write a file to the .git/obis folder containing the commit id. Filename includes a timestamp so they can be sorted.
- # - call openbis to create a data set, using the existing data set as a parent, if there is one
- # - save the data set id to .git/obis/datasetid.
- return CommandResult(returncode=0, output="")
+ cmd = OpenbisSync(self.openbis, self.git_wrapper, self.config_resolver)
+ return cmd.run()
def commit(self, msg, auto_add=True, sync=True):
if auto_add:
@@ -314,9 +311,112 @@ class GitWrapper(object):
class OpenbisSync(object):
"""A command object for synchronizing with openBIS."""
- def __init__(self, git_path=None, git_annex_path=None, find_git=None):
- self.git_path = git_path
- self.git_annex_path = git_annex_path
+ def __init__(self, openbis, git_wrapper, config_resolver):
+ self.openbis = openbis
+ self.git_wrapper = git_wrapper
+ self.config_resolver = config_resolver
+ self.config_dict = config_resolver.config_dict()
+
+ def user(self):
+ return self.config_dict.get('user')
+
+ def external_dms_id(self):
+ return self.config_dict.get('external_dms_id')
+
+ def data_set_type(self):
+ return self.config_dict.get('data_set_type')
+
+ def sample_id(self):
+ return self.config_dict.get('sample_id')
+
+ def check_configuration(self):
+ missing_config_settings = []
+ if self.openbis is None:
+ missing_config_settings.append('openbis_url')
+ if self.user() is None:
+ missing_config_settings.append('user')
+ if self.data_set_type() is None:
+ missing_config_settings.append('data_set_type')
+ if self.data_set_type() is None:
+ missing_config_settings.append('sample_id')
+ if len(missing_config_settings) > 0:
+ return CommandResult(returncode=-1,
+ output="Missing configuration settings for {}.".format(missing_config_settings))
+ return CommandResult(returncode=0, output="")
+
+ def login(self):
+ if self.openbis.is_session_active():
+ return CommandResult(returncode=0, output="")
+ user = self.user()
+ passwd = getpass.getpass("Password for {}".format(user))
+ try:
+ self.openbis.login(user, passwd)
+ except ValueError:
+ msg = "Could not log into openbis {}".format(self.config_dict['openbis_url'])
+ return CommandResult(returncode=-1, output=msg)
+
+ def get_external_data_management_system(self):
+ external_dms_id = self.external_dms_id()
+ if external_dms_id is None:
+ return None
+ external_dms = self.openbis.get_external_data_management_system(external_dms_id)
+ return external_dms
+
+ def create_external_data_management_system(self):
+ external_dms_id = self.external_dms_id()
+ user = self.user()
+ result = self.git_wrapper.get_top_level_path()
+ if result.failure():
+ return result
+ top_level_path = result.output
+ path_name = os.path.basename(top_level_path)
+ if external_dms_id is None:
+ external_dms_id = "{}-{}".format(user, path_name)
+ try:
+ edms = self.openbis.create_external_data_management_system(external_dms_id, external_dms_id,
+ "localhost:/{}".format(top_level_path))
+ return CommandResult(returncode=0, output=""), edms
+ except ValueError as e:
+ return CommandResult(returncode=-1, output=str(e)), None
+
+ def create_data_set(self, external_dms):
+ # Get the commit id
+ data_set_type = self.data_set_type()
+ result = self.git_wrapper.get_top_level_path()
+ if result.failure():
+ return result
+ top_level_path = result.output
+ # TODO Get commit id
+ commit_id = '12345'
+ sample_id = self.sample_id()
+ try:
+ data_set = self.openbis.new_git_data_set(data_set_type, top_level_path, commit_id, external_dms.code,
+ sample_id)
+ return CommandResult(returncode=0, output=""), data_set
+ except ValueError as e:
+ return CommandResult(returncode=-1, output=str(e)), None
def run(self):
- pass
+ # TODO create a data set in openBIS
+ # - write a file to the .git/obis folder containing the commit id. Filename includes a timestamp so they can be sorted.
+
+ result = self.check_configuration()
+ if result.failure():
+ return result
+ result = self.login()
+ if result.failure():
+ return result
+
+ # If there is no external data management system, create one.
+ external_dms = self.get_external_data_management_system()
+ if external_dms is None:
+ result, external_dms = self.create_external_data_management_system()
+ if result.failure():
+ return result
+
+ self.config_resolver.set_value_for_parameter('external_dms_id', external_dms.code, 'local')
+
+ # create a data set, using the existing data set as a parent, if there is one
+ result, data_set = self.create_data_set(external_dms)
+ # - save the data set id to .git/obis/datasetid.
+ return CommandResult(returncode=0, output="")
diff --git a/src/python/OBis/obis/dm/data_mgmt_test.py b/src/python/OBis/obis/dm/data_mgmt_test.py
index 42270778a78..1887570589c 100644
--- a/src/python/OBis/obis/dm/data_mgmt_test.py
+++ b/src/python/OBis/obis/dm/data_mgmt_test.py
@@ -12,6 +12,14 @@ import os
import shutil
from . import data_mgmt
+from unittest.mock import Mock, MagicMock
+from pybis.pybis import ExternalDMS, DataSet
+
+
+def shared_dm():
+ git_config = {'find_git': True}
+ dm = data_mgmt.DataMgmt(git_config=git_config)
+ return dm
def test_no_git(tmpdir):
@@ -43,11 +51,13 @@ def git_status(path=None, annex=False):
return data_mgmt.run_shell(cmd)
-def test_data_use_case(shared_dm, tmpdir):
+def test_data_use_case(tmpdir):
+ dm = shared_dm()
+
tmp_dir_path = str(tmpdir)
assert git_status(tmp_dir_path).returncode == 128 # The folder should not be a git repo at first.
- result = shared_dm.init_data(tmp_dir_path, "test")
+ result = dm.init_data(tmp_dir_path, "test")
assert result.returncode == 0
assert git_status(tmp_dir_path).returncode == 0 # The folder should be a git repo now
@@ -56,7 +66,11 @@ def test_data_use_case(shared_dm, tmpdir):
copy_test_data(tmpdir)
with data_mgmt.cd(tmp_dir_path):
- result = shared_dm.commit("Added data.")
+ dm = shared_dm()
+ prepare_registration_expectations(dm)
+ set_registration_configuration(dm)
+
+ result = dm.commit("Added data.")
assert result.returncode == 0
# The zip should be in the annex
@@ -81,6 +95,27 @@ def test_data_use_case(shared_dm, tmpdir):
assert stat.st_nlink == 1
+def set_registration_configuration(dm):
+ resolver = dm.config_resolver
+ resolver.set_value_for_parameter('openbis_url', "https://localhost:8443", 'local')
+ resolver.set_value_for_parameter('user', "auser", 'local')
+ resolver.set_value_for_parameter('data_set_type', "DS_TYPE", 'local')
+ resolver.set_value_for_parameter('sample_id', "/SAMPLE/ID", 'local')
+
+
+def prepare_registration_expectations(dm):
+ dm.openbis = Mock()
+ dm.openbis.is_session_active = MagicMock(return_value=True)
+ edms = ExternalDMS(dm.openbis, {'code': 'AUSER-PATH', 'label': 'AUSER-PATH'})
+ dm.openbis.create_external_data_management_system = MagicMock(return_value=edms)
+
+ data_set = DataSet(dm.openbis, None,
+ {'code': 'DS-1', 'properties': {},
+ "parents": [], "children": [], "samples": [], 'tags': [],
+ 'physicalData': None})
+ dm.openbis.new_git_data_set = MagicMock(data_set)
+
+
def copy_test_data(tmpdir):
# Put some (binary) content into our new repository
test_data_folder = os.path.join(os.path.dirname(__file__), '..', 'test-data')
diff --git a/src/python/OBis/obis/dm/repo_test.py b/src/python/OBis/obis/dm/repo_test.py
index fd5f25ad18c..a52644c6d4e 100644
--- a/src/python/OBis/obis/dm/repo_test.py
+++ b/src/python/OBis/obis/dm/repo_test.py
@@ -8,10 +8,12 @@ repo_test.py
Created by Chandrasekhar Ramakrishnan on 2017-03-03.
Copyright (c) 2017 Chandrasekhar Ramakrishnan. All rights reserved.
"""
+from unittest.mock import Mock, MagicMock
from . import repo as dm_repo
from . import data_mgmt
-from .data_mgmt_test import git_status, copy_test_data
+from .data_mgmt_test import git_status, copy_test_data, prepare_registration_expectations, \
+ set_registration_configuration
def test_data_use_case(tmpdir):
@@ -19,6 +21,8 @@ def test_data_use_case(tmpdir):
assert git_status(tmp_dir_path).returncode == 128 # The folder should not be a git repo at first.
repo = dm_repo.DataRepo(tmp_dir_path)
+ prepare_registration_expectations(repo.dm_api)
+ set_registration_configuration(repo.dm_api)
result = repo.init("test")
assert result.returncode == 0
diff --git a/src/python/OBis/obis/scripts/cli.py b/src/python/OBis/obis/scripts/cli.py
index 0c7e7abf85d..9eab84b807d 100644
--- a/src/python/OBis/obis/scripts/cli.py
+++ b/src/python/OBis/obis/scripts/cli.py
@@ -45,14 +45,31 @@ def cli(ctx, quiet):
@cli.command()
+@click.option('-g', '--is_global', default=False, is_flag=True, help='Configure global or local.')
+@click.argument('property', default="")
+@click.argument('value', default="")
@click.pass_context
-@click.option('-g', '--global', default=False, is_flag=True, help='Configure global or local.')
-def config(ctx):
+def config(ctx, is_global, property, value):
"""Configure the openBIS setup.
Configure the openBIS server url, the data set type, and the data set properties.
"""
- click_echo("config")
+ ctx.obj['global'] = is_global
+ resolver = shared_data_mgmt().config_resolver
+ is_global = ctx.obj['global']
+ if is_global:
+ resolver.location_search_order = ['global']
+ else:
+ resolver.location_search_order = ['local']
+
+ config_dict = resolver.config_dict()
+ if not property:
+ click.echo("{}".format(config_dict))
+ elif not value:
+ click.echo("{}".format(config_dict[property]))
+ else:
+ loc = 'global' if is_global else 'local'
+ resolver.set_value_for_parameter(property, value, loc)
@cli.group()
--
GitLab