diff --git a/src/python/OBis/integration_tests/00_get_config.sh b/src/python/OBis/integration_tests/00_get_config.sh
deleted file mode 100755
index 7df8639dbc1125509232b47b959c542cc4e409d8..0000000000000000000000000000000000000000
--- a/src/python/OBis/integration_tests/00_get_config.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-
-cd $1
-obis config
diff --git a/src/python/OBis/integration_tests/00_get_config_global.sh b/src/python/OBis/integration_tests/00_get_config_global.sh
deleted file mode 100755
index 13449f27ca4c4ba6fbee349bda810ad132ac7868..0000000000000000000000000000000000000000
--- a/src/python/OBis/integration_tests/00_get_config_global.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-
-obis config -g
diff --git a/src/python/OBis/integration_tests/01_global_config.sh b/src/python/OBis/integration_tests/01_global_config.sh
deleted file mode 100755
index ec95bed0b31bb2d1ff86c8aee5511018a8f0cbf0..0000000000000000000000000000000000000000
--- a/src/python/OBis/integration_tests/01_global_config.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-
-if [ -d "~/.obis" ]; then
- rm -r ~/.obis
-fi
-
-obis config -g openbis_url https://localhost:8443
-obis config -g user admin
-obis config -g data_set_type UNKNOWN
-obis config -g verify_certificates false
-obis config -g hostname `hostname`
diff --git a/src/python/OBis/integration_tests/02_first_commit_1_create_repository.sh b/src/python/OBis/integration_tests/02_first_commit_1_create_repository.sh
deleted file mode 100755
index f507e0123b1580b5e46103317170a8a7dbb4425a..0000000000000000000000000000000000000000
--- a/src/python/OBis/integration_tests/02_first_commit_1_create_repository.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-rm -rf $1/obis_data
-mkdir $1/obis_data && cd $1/obis_data
-obis init data1 && cd data1
-echo content >> file
-obis status
diff --git a/src/python/OBis/integration_tests/02_first_commit_2_commit.sh b/src/python/OBis/integration_tests/02_first_commit_2_commit.sh
deleted file mode 100755
index 5a62ded9a6139a9f41a69f0910b83972fa8b9455..0000000000000000000000000000000000000000
--- a/src/python/OBis/integration_tests/02_first_commit_2_commit.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/bash
-
-cd $1/obis_data/data1
-
-obis config object_id /DEFAULT/DEFAULT
-obis commit -m 'commit message'
diff --git a/src/python/OBis/integration_tests/03_second_commit_1_commit.sh b/src/python/OBis/integration_tests/03_second_commit_1_commit.sh
deleted file mode 100755
index 7488bbdcca86ffed934d76d12974e9e8315a556a..0000000000000000000000000000000000000000
--- a/src/python/OBis/integration_tests/03_second_commit_1_commit.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/bash
-
-cd $1/obis_data/data1
-
-dd if=/dev/zero of=big_file bs=1000000 count=1
-obis commit -m 'commit message'
diff --git a/src/python/OBis/integration_tests/03_second_commit_2_git_annex_info.sh b/src/python/OBis/integration_tests/03_second_commit_2_git_annex_info.sh
deleted file mode 100755
index f3e668bce882c1a334d1571a4518046d138af3b1..0000000000000000000000000000000000000000
--- a/src/python/OBis/integration_tests/03_second_commit_2_git_annex_info.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/bash
-
-cd $1/obis_data/data1
-
-git annex info big_file
diff --git a/src/python/OBis/integration_tests/04_second_repository.sh b/src/python/OBis/integration_tests/04_second_repository.sh
deleted file mode 100755
index 903e8a64b9edf464cb80b91176e20363fd5ff2e3..0000000000000000000000000000000000000000
--- a/src/python/OBis/integration_tests/04_second_repository.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-cd $1/obis_data
-obis init data2 && cd data2
-obis config object_id /DEFAULT/DEFAULT
-echo content >> file
-obis commit -m 'commit message'
diff --git a/src/python/OBis/integration_tests/05_second_external_dms.sh b/src/python/OBis/integration_tests/05_second_external_dms.sh
deleted file mode 100755
index c7c4ef87969a50b551bd632d4fcbcd2c3a713782..0000000000000000000000000000000000000000
--- a/src/python/OBis/integration_tests/05_second_external_dms.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-mkdir $1/obis_data_b && cd $1/obis_data_b
-obis init data3 && cd data3
-obis config object_id /DEFAULT/DEFAULT
-echo content >> file
-obis commit -m 'commit message'
diff --git a/src/python/OBis/integration_tests/06_error_on_first_commit_1_error.sh b/src/python/OBis/integration_tests/06_error_on_first_commit_1_error.sh
deleted file mode 100755
index 3c58d80edce46b66c55023226dbdcacba101fe9f..0000000000000000000000000000000000000000
--- a/src/python/OBis/integration_tests/06_error_on_first_commit_1_error.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/bash
-
-cd $1/obis_data
-obis init data4 && cd data4
-echo content >> file
-obis commit -m 'commit message'
diff --git a/src/python/OBis/integration_tests/06_error_on_first_commit_2_status.sh b/src/python/OBis/integration_tests/06_error_on_first_commit_2_status.sh
deleted file mode 100755
index 8815e2f5b74ac894b2b9705c27e44176752026f6..0000000000000000000000000000000000000000
--- a/src/python/OBis/integration_tests/06_error_on_first_commit_2_status.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-
-cd $1/obis_data/data4
-obis status
diff --git a/src/python/OBis/integration_tests/06_error_on_first_commit_3_commit.sh b/src/python/OBis/integration_tests/06_error_on_first_commit_3_commit.sh
deleted file mode 100755
index 6cb1ce4b2cd566ebdc8ca368aff6a8c757a346c6..0000000000000000000000000000000000000000
--- a/src/python/OBis/integration_tests/06_error_on_first_commit_3_commit.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/bash
-
-cd $1/obis_data/data4
-obis config object_id /DEFAULT/DEFAULT
-obis commit -m 'commit message'
diff --git a/src/python/OBis/integration_tests/07_attach_to_collection.sh b/src/python/OBis/integration_tests/07_attach_to_collection.sh
deleted file mode 100755
index 3ea844b98ecdd347c4adbecf0ec9cbfb361eb7cc..0000000000000000000000000000000000000000
--- a/src/python/OBis/integration_tests/07_attach_to_collection.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/bash
-
-cd $1/obis_data
-obis init data5 && cd data5
-echo content >> file
-obis config collection_id /DEFAULT/DEFAULT/DEFAULT
-obis commit -m 'msg'
-
diff --git a/src/python/OBis/integration_tests/08_addref_1_success.sh b/src/python/OBis/integration_tests/08_addref_1_success.sh
deleted file mode 100755
index f368719c5a216c3305eb2b72e71d53e619665cc5..0000000000000000000000000000000000000000
--- a/src/python/OBis/integration_tests/08_addref_1_success.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/bash
-
-cd $1/obis_data
-cp -r data1 data6
-obis addref data6
-
diff --git a/src/python/OBis/integration_tests/08_addref_2_duplicate.sh b/src/python/OBis/integration_tests/08_addref_2_duplicate.sh
deleted file mode 100755
index 0429bbf6221c3c75e7202b7fb862c4abbd3c0f68..0000000000000000000000000000000000000000
--- a/src/python/OBis/integration_tests/08_addref_2_duplicate.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/bash
-
-cd $1/obis_data
-obis addref data6
-
diff --git a/src/python/OBis/integration_tests/08_addref_3_non-existent.sh b/src/python/OBis/integration_tests/08_addref_3_non-existent.sh
deleted file mode 100755
index 363cc3d7027d1c2a456db0081595550a4d65cab2..0000000000000000000000000000000000000000
--- a/src/python/OBis/integration_tests/08_addref_3_non-existent.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/bash
-
-cd $1/obis_data
-obis addref data7
-
diff --git a/src/python/OBis/integration_tests/09_local_clone.sh b/src/python/OBis/integration_tests/09_local_clone.sh
deleted file mode 100755
index 5e2a031aa69de85754f86032b4aaed67717da288..0000000000000000000000000000000000000000
--- a/src/python/OBis/integration_tests/09_local_clone.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-data_set_id=$2
-
-cd $1/obis_data_b
-obis clone $data_set_id
-
diff --git a/src/python/OBis/integration_tests/11_init_analysis_1_external.sh b/src/python/OBis/integration_tests/11_init_analysis_1_external.sh
deleted file mode 100755
index d585576488124351ea0804d4881ffeea57a6375b..0000000000000000000000000000000000000000
--- a/src/python/OBis/integration_tests/11_init_analysis_1_external.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash
-
-cd $1/obis_data
-obis init_analysis -p data1 analysis1
-cd analysis1
-obis config object_id /DEFAULT/DEFAULT
-echo content >> file
-obis commit -m 'commit message'
-
diff --git a/src/python/OBis/integration_tests/11_init_analysis_2_internal.sh b/src/python/OBis/integration_tests/11_init_analysis_2_internal.sh
deleted file mode 100755
index 311d53e0579a468d5e97bff5f2b0bf92ba29cb57..0000000000000000000000000000000000000000
--- a/src/python/OBis/integration_tests/11_init_analysis_2_internal.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash
-
-cd $1/obis_data/data1
-obis init_analysis analysis2
-cd analysis2
-obis config object_id /DEFAULT/DEFAULT
-echo content >> file
-obis commit -m 'commit message'
-
diff --git a/src/python/OBis/integration_tests/11_init_analysis_3_git_check_ignore.sh b/src/python/OBis/integration_tests/11_init_analysis_3_git_check_ignore.sh
deleted file mode 100755
index bb597b8aae1c371242fb65c77948ed523f826008..0000000000000000000000000000000000000000
--- a/src/python/OBis/integration_tests/11_init_analysis_3_git_check_ignore.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/bash
-
-cd $1/obis_data/data1
-git check-ignore analysis2
-
diff --git a/src/python/OBis/integration_tests/12_metadata_only_1_commit.sh b/src/python/OBis/integration_tests/12_metadata_only_1_commit.sh
deleted file mode 100755
index 2a7d6560f3b989acde69e0b20a61b108f1eb7ffc..0000000000000000000000000000000000000000
--- a/src/python/OBis/integration_tests/12_metadata_only_1_commit.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/bash
-
-cd $1/obis_data
-obis init data7 && cd data7
-obis config object_id /DEFAULT/DEFAULT
-echo content >> file
-obis commit -m 'commit message'
-
diff --git a/src/python/OBis/integration_tests/12_metadata_only_2_metadata_commit.sh b/src/python/OBis/integration_tests/12_metadata_only_2_metadata_commit.sh
deleted file mode 100755
index 086383272c5210ed109876d98c346f59f5df0375..0000000000000000000000000000000000000000
--- a/src/python/OBis/integration_tests/12_metadata_only_2_metadata_commit.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/bash
-
-cd $1/obis_data/data7
-obis config collection_id /DEFAULT/DEFAULT/DEFAULT
-obis commit -m 'commit message'
-
diff --git a/src/python/OBis/integration_tests/13_sync_1_git_commit_and_sync.sh b/src/python/OBis/integration_tests/13_sync_1_git_commit_and_sync.sh
deleted file mode 100755
index 590f099d3a1be607a7f28a2b24fa3406f33d8e4d..0000000000000000000000000000000000000000
--- a/src/python/OBis/integration_tests/13_sync_1_git_commit_and_sync.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/bash
-
-cd $1/obis_data/data7
-echo content >> file2
-git add file2
-git commit -m 'msg'
-obis sync
-
diff --git a/src/python/OBis/integration_tests/13_sync_2_only_sync.sh b/src/python/OBis/integration_tests/13_sync_2_only_sync.sh
deleted file mode 100755
index 615a994c45922bba9ff223f2879c7ca8ad59416c..0000000000000000000000000000000000000000
--- a/src/python/OBis/integration_tests/13_sync_2_only_sync.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/bash
-
-cd $1/obis_data/data7
-obis sync
-
diff --git a/src/python/OBis/integration_tests/integration_tests.py b/src/python/OBis/integration_tests/integration_tests.py
index 3cfa87ae1479ecc60c6b8f7731c259908176eb33..4d9aaa390d78af71ff18c6626284ae2ccf52c70e 100644
--- a/src/python/OBis/integration_tests/integration_tests.py
+++ b/src/python/OBis/integration_tests/integration_tests.py
@@ -1,145 +1,372 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
+# can be run on vagrant like this:
+# vagrant ssh obisserver -c 'cd /vagrant_python/OBis/integration_tests && pytest ./integration_tests.py'
+
import json
-import subprocess
+import os
import socket
+import subprocess
+from subprocess import PIPE
+from subprocess import SubprocessError
+from contextlib import contextmanager
from pybis import Openbis
-def run(cmd, tmpdir="", params=[]):
- completed_process = subprocess.run([cmd, tmpdir] + params, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+output_buffer = ''
+
+def decorator_print(func):
+ def wrapper(tmpdir, *args, **kwargs):
+ try:
+ func(tmpdir, *args, **kwargs)
+ except Exception:
+ print(output_buffer)
+ raise
+ return wrapper
+
+@decorator_print
+def test_obis(tmpdir):
+ global output_buffer
+
+ o = Openbis('https://obisserver:8443', verify_certificates=False)
+ o.login('admin', 'admin', save_token=True)
+
+ output_buffer = '=================== 1. Global settings ===================\n'
+ if os.path.exists('~/.obis'):
+ os.rmdir('~/.obis')
+ cmd('obis config -g set openbis_url=https://obisserver:8443')
+ cmd('obis config -g set user=admin')
+ cmd('obis config -g set verify_certificates=false')
+ cmd('obis config -g set hostname=' + socket.gethostname())
+ cmd('obis data_set -g set type=UNKNOWN')
+ settings = get_settings_global()
+ assert settings['config']['openbis_url'] == 'https://obisserver:8443'
+ assert settings['config']['user'] == 'admin'
+ assert settings['config']['verify_certificates'] == False
+ assert settings['config']['hostname'] == socket.gethostname()
+ assert settings['data_set']['type'] == 'UNKNOWN'
+
+ with cd(tmpdir): cmd('mkdir obis_data')
+ with cd(tmpdir + '/obis_data'):
+
+ output_buffer = '=================== 2. First commit ===================\n'
+ cmd('obis init data1')
+ with cd('data1'):
+ cmd('touch file')
+ result = cmd('obis status')
+ assert '?? .obis/config.json' in result
+ assert '?? file' in result
+ cmd('obis object set id=/DEFAULT/DEFAULT')
+ result = cmd('obis commit -m \'commit-message\'')
+ settings = get_settings()
+ assert settings['repository']['external_dms_id'].startswith('ADMIN-' + socket.gethostname().upper())
+ assert len(settings['repository']['id']) == 36
+ assert "Created data set {}.".format(settings['repository']['data_set_id']) in result
+ data_set = o.get_dataset(settings['repository']['data_set_id']).data
+ assert_matching(settings, data_set, tmpdir, 'obis_data/data1')
+
+ output_buffer = '=================== 3. Second commit ===================\n'
+ with cd('data1'):
+ settings_before = get_settings()
+ cmd('dd if=/dev/zero of=big_file bs=1000000 count=1')
+ result = cmd('obis commit -m \'commit-message\'')
+ settings = get_settings()
+ assert settings['repository']['data_set_id'] != settings_before['repository']['data_set_id']
+ assert settings['repository']['external_dms_id'].startswith('ADMIN-' + socket.gethostname().upper())
+ assert settings['repository']['external_dms_id'] == settings_before['repository']['external_dms_id']
+ assert settings['repository']['id'] == settings_before['repository']['id']
+ assert "Created data set {}.".format(settings['repository']['data_set_id']) in result
+ result = cmd('git annex info big_file')
+ assert 'file: big_file' in result
+ assert 'key: SHA256E-s1000000--d29751f2649b32ff572b5e0a9f541ea660a50f94ff0beedfb0b692b924cc8025' in result
+ assert 'present: true' in result
+ data_set = o.get_dataset(settings['repository']['data_set_id']).data
+ assert_matching(settings, data_set, tmpdir, 'obis_data/data1')
+ assert data_set['parents'][0]['code'] == settings_before['repository']['data_set_id']
+
+ output_buffer = '=================== 4. Second repository ===================\n'
+ cmd('obis init data2')
+ with cd('data2'):
+ cmd('obis object set id=/DEFAULT/DEFAULT')
+ cmd('touch file')
+ result = cmd('obis commit -m \'commit-message\'')
+ with cd('../data1'): settings_data1 = get_settings()
+ settings = get_settings()
+ assert settings['repository']['external_dms_id'].startswith('ADMIN-' + socket.gethostname().upper())
+ assert settings['repository']['external_dms_id'] == settings_data1['repository']['external_dms_id']
+ assert len(settings['repository']['id']) == 36
+ assert settings['repository']['id'] != settings_data1['repository']['id']
+ assert "Created data set {}.".format(settings['repository']['data_set_id']) in result
+ data_set = o.get_dataset(settings['repository']['data_set_id']).data
+ assert_matching(settings, data_set, tmpdir, 'obis_data/data2')
+
+ output_buffer = '=================== 5. Second external dms ===================\n'
+ with cd(tmpdir): cmd('mkdir obis_data_b')
+ with cd(tmpdir + '/obis_data_b'):
+ cmd('obis init data3')
+ with cd('data3'):
+ cmd('obis object set id=/DEFAULT/DEFAULT')
+ cmd('touch file')
+ result = cmd('obis commit -m \'commit-message\'')
+ with cd('../../obis_data/data1'): settings_data1 = get_settings()
+ settings = get_settings()
+ assert settings['repository']['external_dms_id'].startswith('ADMIN-' + socket.gethostname().upper())
+ assert settings['repository']['external_dms_id'] != settings_data1['repository']['external_dms_id']
+ assert len(settings['repository']['id']) == 36
+ assert settings['repository']['id'] != settings_data1['repository']['id']
+ assert "Created data set {}.".format(settings['repository']['data_set_id']) in result
+ data_set = o.get_dataset(settings['repository']['data_set_id']).data
+ assert_matching(settings, data_set, tmpdir, 'obis_data_b/data3')
+
+ output_buffer = '=================== 6. Error on first commit ===================\n'
+ with cd(tmpdir + '/obis_data'):
+ cmd('obis init data4')
+ with cd('data4'):
+ cmd('touch file')
+ result = cmd('obis commit -m \'commit-message\'')
+ assert 'Missing configuration settings for [\'object id or collection id\'].' in result
+ result = cmd('obis status')
+ assert '?? file' in result
+ cmd('obis object set id=/DEFAULT/DEFAULT')
+ result = cmd('obis commit -m \'commit-message\'')
+ settings = get_settings()
+ assert "Created data set {}.".format(settings['repository']['data_set_id']) in result
+ data_set = o.get_dataset(settings['repository']['data_set_id']).data
+ assert_matching(settings, data_set, tmpdir, 'obis_data/data4')
+
+ output_buffer = '=================== 7. Attach data set to a collection ===================\n'
+ cmd('obis init data5')
+ with cd('data5'):
+ cmd('touch file')
+ cmd('obis collection set id=/DEFAULT/DEFAULT/DEFAULT')
+ result = cmd('obis commit -m \'commit-message\'')
+ settings = get_settings()
+ assert settings['repository']['external_dms_id'].startswith('ADMIN-' + socket.gethostname().upper())
+ assert len(settings['repository']['id']) == 36
+ assert "Created data set {}.".format(settings['repository']['data_set_id']) in result
+ data_set = o.get_dataset(settings['repository']['data_set_id']).data
+ assert_matching(settings, data_set, tmpdir, 'obis_data/data5')
+
+ output_buffer = '=================== 8. Addref ===================\n'
+ cmd('cp -r data1 data6')
+ cmd('obis addref data6')
+ with cd('data1'): settings_data1 = get_settings()
+ with cd('data6'): settings_data6 = get_settings()
+ assert settings_data6 == settings_data1
+ result = cmd('obis addref data6')
+ assert 'DataSet already exists in the database' in result
+ result = cmd('obis addref data7')
+ assert 'Invalid value' in result
+ data_set = o.get_dataset(settings_data6['repository']['data_set_id']).data
+ with cd('data6'): assert_matching(settings_data6, data_set, tmpdir, 'obis_data/data6')
+
+ output_buffer = '=================== 9. Local clone ===================\n'
+ with cd('data2'): settings_data2 = get_settings()
+ with cd('../obis_data_b'):
+ cmd('obis clone ' + settings_data2['repository']['data_set_id'])
+ with cd('data2'):
+ settings_data2_clone = get_settings()
+ assert settings_data2_clone['repository']['external_dms_id'].startswith('ADMIN-' + socket.gethostname().upper())
+ assert settings_data2_clone['repository']['external_dms_id'] != settings_data2['repository']['external_dms_id']
+ data_set = o.get_dataset(settings_data2_clone['repository']['data_set_id']).data
+ assert_matching(settings_data2_clone, data_set, tmpdir, 'obis_data_b/data2')
+ del settings_data2['repository']['external_dms_id']
+ del settings_data2_clone['repository']['external_dms_id']
+ assert settings_data2_clone == settings_data2
+
+ output_buffer = '=================== 11. Init analysis ===================\n'
+ cmd('obis init_analysis -p data1 analysis1')
+ with cd('analysis1'):
+ cmd('obis object set id=/DEFAULT/DEFAULT')
+ cmd('touch file')
+ result = cmd('obis commit -m \'commit-message\'')
+ with cd('data1'): settings_data1 = get_settings()
+ with cd('analysis1'):
+ settings_analysis1 = get_settings()
+ assert "Created data set {}.".format(settings_analysis1['repository']['data_set_id']) in result
+ assert len(settings_analysis1['repository']['id']) == 36
+ assert settings_analysis1['repository']['id'] != settings_data1['repository']['id']
+ assert settings_analysis1['repository']['data_set_id'] != settings_data1['repository']['data_set_id']
+ data_set = o.get_dataset(settings_analysis1['repository']['data_set_id']).data
+ assert_matching(settings_analysis1, data_set, tmpdir, 'obis_data/analysis1')
+ assert data_set['parents'][0]['code'] == settings_data1['repository']['data_set_id']
+ with cd('data1'):
+ cmd('obis init_analysis analysis2')
+ with cd('analysis2'):
+ cmd('obis object set id=/DEFAULT/DEFAULT')
+ cmd('touch file')
+ result = cmd('obis commit -m \'commit-message\'')
+ settings_analysis2 = get_settings()
+ assert "Created data set {}.".format(settings_analysis2['repository']['data_set_id']) in result
+ assert len(settings_analysis2['repository']['id']) == 36
+ assert settings_analysis2['repository']['id'] != settings_data1['repository']['id']
+ assert settings_analysis2['repository']['data_set_id'] != settings_data1['repository']['data_set_id']
+ data_set = o.get_dataset(settings_analysis2['repository']['data_set_id']).data
+ assert_matching(settings_analysis2, data_set, tmpdir, 'obis_data/data1/analysis2')
+ assert data_set['parents'][0]['code'] == settings_data1['repository']['data_set_id']
+ result = cmd('git check-ignore analysis2')
+ assert 'analysis2' in result
+
+ output_buffer = '=================== 12. Metadata only commit ===================\n'
+ cmd('obis init data7')
+ with cd('data7'):
+ cmd('obis object set id=/DEFAULT/DEFAULT')
+ cmd('touch file')
+ result = cmd('obis commit -m \'commit-message\'')
+ settings = get_settings()
+ assert "Created data set {}.".format(settings['repository']['data_set_id']) in result
+ data_set = o.get_dataset(settings['repository']['data_set_id']).data
+ assert_matching(settings, data_set, tmpdir, 'obis_data/data7')
+ cmd('obis collection set id=/DEFAULT/DEFAULT/DEFAULT')
+ result = cmd('obis commit -m \'commit-message\'')
+ settings = get_settings()
+ assert "Created data set {}.".format(settings['repository']['data_set_id']) in result
+ data_set = o.get_dataset(settings['repository']['data_set_id']).data
+ assert_matching(settings, data_set, tmpdir, 'obis_data/data7')
+
+ output_buffer = '=================== 13. obis sync ===================\n'
+ with cd('data7'):
+ cmd('touch file2')
+ cmd('git add file2')
+ cmd('git commit -m \'msg\'')
+ result = cmd('obis sync')
+ settings = get_settings()
+ assert "Created data set {}.".format(settings['repository']['data_set_id']) in result
+ data_set = o.get_dataset(settings['repository']['data_set_id']).data
+ assert_matching(settings, data_set, tmpdir, 'obis_data/data7')
+ result = cmd('obis sync')
+ assert 'Nothing to sync' in result
+
+ output_buffer = '=================== 14. Set data set properties ===================\n'
+ cmd('obis init data8')
+ with cd('data8'):
+ result = cmd('obis data_set -p set a=0')
+ settings = get_settings()
+ assert settings['data_set']['properties'] == { 'A': '0' }
+ cmd('obis data_set set properties={"a":"0","b":"1","c":"2"}')
+ cmd('obis data_set -p set c=3')
+ settings = get_settings()
+ assert settings['data_set']['properties'] == { 'A': '0', 'B': '1', 'C': '3' }
+ result = cmd('obis data_set set properties={"a":"0","A":"1"}')
+ assert 'Duplicate key after capitalizing JSON config: A' in result
+
+ output_buffer = '=================== 15. Removeref ===================\n'
+ with cd('data6'): settings = get_settings()
+ content_copies = get_data_set(o, settings)['linkedData']['contentCopies']
+ assert len(content_copies) == 2
+ cmd('obis removeref data6')
+ content_copies = get_data_set(o, settings)['linkedData']['contentCopies']
+ assert len(content_copies) == 1
+ assert content_copies[0]['path'].endswith('data1')
+ cmd('obis addref data6')
+ cmd('obis removeref data1')
+ content_copies = get_data_set(o, settings)['linkedData']['contentCopies']
+ assert len(content_copies) == 1
+ assert content_copies[0]['path'].endswith('data6')
+ result = cmd('obis removeref data1')
+ assert 'Matching content copy not fount in data set' in result
+ cmd('obis addref data1')
+
+ output_buffer = '=================== 18. Use git-annex hashes as checksums ===================\n'
+ cmd('obis init data10')
+ with cd('data10'):
+ cmd('touch file')
+ cmd('obis object set id=/DEFAULT/DEFAULT')
+ # use SHA256 form git annex by default
+ result = cmd('obis commit -m \'commit-message\'')
+ settings = get_settings()
+ search_result = o.search_files(settings['repository']['data_set_id'])
+ files = list(filter(lambda file: file['fileLength'] > 0, search_result['objects']))
+ assert len(files) == 5
+ for file in files:
+ assert file['checksumType'] == "SHA256"
+ assert len(file['checksum']) == 64
+ # don't use git annex hash - use default CRC32
+ cmd('obis config set git_annex_hash_as_checksum=false')
+ result = cmd('obis commit -m \'commit-message\'')
+ settings = get_settings()
+ search_result = o.search_files(settings['repository']['data_set_id'])
+ files = list(filter(lambda file: file['fileLength'] > 0, search_result['objects']))
+ assert len(files) == 5
+ for file in files:
+ assert file['checksumType'] is None
+ assert file['checksum'] is None
+ assert file['checksumCRC32'] != 0
+
+ output_buffer = '=================== 19. Clearing settings ===================\n'
+ cmd('obis init data11')
+ with cd('data11'):
+ assert get_settings()['repository'] == {'id': None, 'external_dms_id': None, 'data_set_id': None}
+ cmd('obis repository set id=0, external_dms_id=1, data_set_id=2')
+ assert get_settings()['repository'] == {'id': '0', 'external_dms_id': '1', 'data_set_id': '2'}
+ cmd('obis repository clear external_dms_id, data_set_id')
+ assert get_settings()['repository'] == {'id': '0', 'external_dms_id': None, 'data_set_id': None}
+ cmd('obis repository clear')
+ assert get_settings()['repository'] == {'id': None, 'external_dms_id': None, 'data_set_id': None}
+
+ output_buffer = '=================== 16. User switch ===================\n'
+ cmd('obis init data9')
+ with cd('data9'):
+ cmd('touch file')
+ cmd('obis object set id=/DEFAULT/DEFAULT')
+ result = cmd('obis commit -m \'commit-message\'')
+ settings = get_settings()
+ assert "Created data set {}.".format(settings['repository']['data_set_id']) in result
+ cmd('touch file2')
+ cmd('obis config set user=watney')
+ # expect timeout because obis is asking for the password of the new user
+ try:
+ timeout = False
+ result = cmd('obis commit -m \'commit-message\'', timeout=3)
+ except SubprocessError:
+ timeout = True
+ assert timeout == True
+
+
+def get_settings():
+ return json.loads(cmd('obis settings get'))
+
+def get_settings_global():
+ return json.loads(cmd('obis settings -g get'))
+
+def get_data_set(o, settings):
+ return o.get_dataset(settings['repository']['data_set_id']).data
+
+@contextmanager
+def cd(newdir):
+ """Safe cd -- return to original dir after execution, even if an exception is raised."""
+ prevdir = os.getcwd()
+ os.chdir(os.path.expanduser(newdir))
+ try:
+ yield
+ finally:
+ os.chdir(prevdir)
+
+def cmd(cmd, timeout=None):
+ global output_buffer
+ output_buffer += '==== running: ' + cmd + '\n'
+ completed_process = subprocess.run(cmd.split(' '), stdout=PIPE, stderr=PIPE, timeout=timeout)
+ result = get_cmd_result(completed_process)
+ output_buffer += result + '\n'
+ return result
+
+def get_cmd_result(completed_process, tmpdir=''):
result = ''
if completed_process.stderr:
result += completed_process.stderr.decode('utf-8').strip()
if completed_process.stdout:
result += completed_process.stdout.decode('utf-8').strip()
- print('-------------------' + cmd + '------------------- ' + str(tmpdir))
- print(result)
return result
-
-def test_obis(tmpdir):
- # 0. pybis login
- o = Openbis('https://localhost:8443', verify_certificates=False)
- o.login('admin', 'admin', save_token=True)
-
- # 1. Global configuration
- result = run('./01_global_config.sh', tmpdir)
- config = json.loads(run('./00_get_config_global.sh'))
- assert config['openbis_url'] == 'https://localhost:8443'
- assert config['user'] == 'admin'
- assert config['data_set_type'] == 'UNKNOWN'
- assert config['verify_certificates'] == False
-
- # 2. First commit
- result = run('./02_first_commit_1_create_repository.sh', tmpdir)
- assert '?? .obis/config.json' in result
- assert '?? file' in result
- result = run('./02_first_commit_2_commit.sh', tmpdir)
- config = json.loads(run('./00_get_config.sh', tmpdir + '/obis_data/data1'))
- assert config['external_dms_id'].startswith('ADMIN-' + socket.gethostname().upper())
- assert len(config['repository_id']) == 36
- assert "Created data set {}.".format(config['data_set_id']) in result
-
- # 3. Second commit
- config_before = json.loads(run('./00_get_config.sh', tmpdir + '/obis_data/data1'))
- result = run('./03_second_commit_1_commit.sh', tmpdir)
- config = json.loads(run('./00_get_config.sh', tmpdir + '/obis_data/data1'))
- assert config['data_set_id'] != config_before['data_set_id']
- assert config['external_dms_id'].startswith('ADMIN-' + socket.gethostname().upper())
- assert config['external_dms_id'] == config_before['external_dms_id']
- assert config['repository_id'] == config_before['repository_id']
- assert "Created data set {}.".format(config['data_set_id']) in result
- result = run('./03_second_commit_2_git_annex_info.sh', tmpdir)
- assert 'file: big_file' in result
- assert 'key: SHA256E-s1000000--d29751f2649b32ff572b5e0a9f541ea660a50f94ff0beedfb0b692b924cc8025' in result
- assert 'present: true' in result
-
- # 4. Second repository
- result = run('./04_second_repository.sh', tmpdir)
- config_data1 = json.loads(run('./00_get_config.sh', tmpdir + '/obis_data/data1'))
- config = json.loads(run('./00_get_config.sh', tmpdir + '/obis_data/data2'))
- assert config['external_dms_id'].startswith('ADMIN-' + socket.gethostname().upper())
- assert config['external_dms_id'] == config_data1['external_dms_id']
- assert len(config['repository_id']) == 36
- assert config['repository_id'] != config_data1['repository_id']
- assert "Created data set {}.".format(config['data_set_id']) in result
-
- # 5. Second external dms
- result = run('./05_second_external_dms.sh', tmpdir)
- config_data1 = json.loads(run('./00_get_config.sh', tmpdir + '/obis_data/data1'))
- config = json.loads(run('./00_get_config.sh', tmpdir + '/obis_data_b/data3'))
- assert config['external_dms_id'].startswith('ADMIN-' + socket.gethostname().upper())
- assert config['external_dms_id'] != config_data1['external_dms_id']
- assert len(config['repository_id']) == 36
- assert config['repository_id'] != config_data1['repository_id']
- assert "Created data set {}.".format(config['data_set_id']) in result
-
- # 6. Error on first commit
- result = run('./06_error_on_first_commit_1_error.sh', tmpdir)
- assert 'Missing configuration settings for [\'object_id\', \'collection_id\'].' in result
- result = run('./06_error_on_first_commit_2_status.sh', tmpdir)
- assert '?? file' in result
- result = run('./06_error_on_first_commit_3_commit.sh', tmpdir)
- config = json.loads(run('./00_get_config.sh', tmpdir + '/obis_data/data4'))
- assert "Created data set {}.".format(config['data_set_id']) in result
-
- # 7. Attach data set to a collection
- result = run('./07_attach_to_collection.sh', tmpdir)
- config = json.loads(run('./00_get_config.sh', tmpdir + '/obis_data/data5'))
- assert config['external_dms_id'].startswith('ADMIN-' + socket.gethostname().upper())
- assert len(config['repository_id']) == 36
- assert "Created data set {}.".format(config['data_set_id']) in result
-
- # 8. Addref
- result = run('./08_addref_1_success.sh', tmpdir)
- config_data1 = json.loads(run('./00_get_config.sh', tmpdir + '/obis_data/data1'))
- config_data6 = json.loads(run('./00_get_config.sh', tmpdir + '/obis_data/data6'))
- assert config_data6 == config_data1
- result = run('./08_addref_2_duplicate.sh', tmpdir)
- assert 'DataSet already exists in the database' in result
- result = run('./08_addref_3_non-existent.sh', tmpdir)
- assert 'Invalid value' in result
-
- # 9. Local clone
- config_data2 = json.loads(run('./00_get_config.sh', tmpdir + '/obis_data/data2'))
- result = run('./09_local_clone.sh', tmpdir, [config_data2['data_set_id']])
- config_data2_clone = json.loads(run('./00_get_config.sh', tmpdir + '/obis_data_b/data2'))
- assert config_data2_clone['external_dms_id'].startswith('ADMIN-' + socket.gethostname().upper())
- assert config_data2_clone['external_dms_id'] != config_data2['external_dms_id']
- del config_data2['external_dms_id']
- del config_data2_clone['external_dms_id']
- assert config_data2_clone == config_data2
-
- # 11. Init analysis
- result = run('./11_init_analysis_1_external.sh', tmpdir, [config_data2['data_set_id']])
- config_data1 = json.loads(run('./00_get_config.sh', tmpdir + '/obis_data/data1'))
- config_analysis1 = json.loads(run('./00_get_config.sh', tmpdir + '/obis_data/analysis1'))
- assert "Created data set {}.".format(config_analysis1['data_set_id']) in result
- assert len(config_analysis1['repository_id']) == 36
- assert config_analysis1['repository_id'] != config_data1['repository_id']
- assert config_analysis1['data_set_id'] != config_data1['data_set_id']
- result = run('./11_init_analysis_2_internal.sh', tmpdir)
- config_analysis2 = json.loads(run('./00_get_config.sh', tmpdir + '/obis_data/data1/analysis2'))
- assert "Created data set {}.".format(config_analysis2['data_set_id']) in result
- assert len(config_analysis2['repository_id']) == 36
- assert config_analysis2['repository_id'] != config_data1['repository_id']
- assert config_analysis2['data_set_id'] != config_data1['data_set_id']
- result = run('./11_init_analysis_3_git_check_ignore.sh', tmpdir)
- assert 'analysis2' in result
-
- # 12. Metadata only commit
- result = run('./12_metadata_only_1_commit.sh', tmpdir)
- config = json.loads(run('./00_get_config.sh', tmpdir + '/obis_data/data7'))
- assert "Created data set {}.".format(config['data_set_id']) in result
- result = run('./12_metadata_only_2_metadata_commit.sh', tmpdir)
- config = json.loads(run('./00_get_config.sh', tmpdir + '/obis_data/data7'))
- assert "Created data set {}.".format(config['data_set_id']) in result
-
- # 13. obis sync
- result = run('./13_sync_1_git_commit_and_sync.sh', tmpdir)
- config = json.loads(run('./00_get_config.sh', tmpdir + '/obis_data/data7'))
- assert "Created data set {}.".format(config['data_set_id']) in result
- result = run('./13_sync_2_only_sync.sh', tmpdir)
- assert 'Nothing to sync' in result
+def assert_matching(settings, data_set, tmpdir, path):
+ content_copies = data_set['linkedData']['contentCopies']
+ content_copy = list(filter(lambda cc: cc['path'].endswith(path) == 1, content_copies))[0]
+ assert data_set['type']['code'] == settings['data_set']['type']
+ assert content_copy['externalDms']['code'] == settings['repository']['external_dms_id']
+ assert content_copy['gitCommitHash'] == cmd('git rev-parse --short HEAD')
+ assert content_copy['gitRepositoryId'] == settings['repository']['id']
+ if settings['object']['id'] is not None:
+ assert data_set['sample']['identifier']['identifier'] == settings['object']['id']
+ if settings['collection']['id'] is not None:
+ assert data_set['experiment']['identifier']['identifier'] == settings['collection']['id']
diff --git a/src/python/OBis/obis/dm/__init__.py b/src/python/OBis/obis/dm/__init__.py
index 0c8e9028cb3fa8304cec636f3e19e837757d20c4..239ae62d551f49f6abeb36d44ab6dc9891bdd529 100644
--- a/src/python/OBis/obis/dm/__init__.py
+++ b/src/python/OBis/obis/dm/__init__.py
@@ -10,4 +10,4 @@ Copyright (c) 2017 Chandrasekhar Ramakrishnan. All rights reserved.
"""
from .data_mgmt import *
-from .config import ConfigResolver
+from .config import SettingsResolver
diff --git a/src/python/OBis/obis/dm/command_result.py b/src/python/OBis/obis/dm/command_result.py
index c53a1c00d68cf7793486810cae0bcb4de6fb8ee3..37bd5794323e02d6c50e410b103299b270290097 100644
--- a/src/python/OBis/obis/dm/command_result.py
+++ b/src/python/OBis/obis/dm/command_result.py
@@ -1,14 +1,16 @@
class CommandResult(object):
"""Encapsulate result from a subprocess call."""
- def __init__(self, completed_process=None, returncode=None, output=None):
+ def __init__(self, completed_process=None, returncode=None, output=None, strip_leading_whitespace=True):
"""Convert a completed_process object into a ShellResult."""
if completed_process:
self.returncode = completed_process.returncode
if completed_process.stderr:
- self.output = completed_process.stderr.decode('utf-8').strip()
+ self.output = completed_process.stderr.decode('utf-8').rstrip()
else:
- self.output = completed_process.stdout.decode('utf-8').strip()
+ self.output = completed_process.stdout.decode('utf-8').rstrip()
+ if strip_leading_whitespace:
+ self.output = self.output.strip()
else:
self.returncode = returncode
self.output = output
@@ -23,4 +25,10 @@ class CommandResult(object):
return self.returncode == 0
def failure(self):
- return not self.success()
\ No newline at end of file
+ return not self.success()
+
+
+class CommandException(Exception):
+
+ def __init__(self, command_result):
+ self.command_result = command_result
diff --git a/src/python/OBis/obis/dm/commands/addref.py b/src/python/OBis/obis/dm/commands/addref.py
index 9d88a8537009916716f5cc13d48aa1b4c0814a43..1480ffaa2a88f0adc77d35b7fe0a39f04a9eab52 100644
--- a/src/python/OBis/obis/dm/commands/addref.py
+++ b/src/python/OBis/obis/dm/commands/addref.py
@@ -24,7 +24,7 @@ class Addref(OpenbisCommand):
def update_external_dms_id(self):
- self.config_dict['external_dms_id'] = None
+ self.set_external_dms_id(None)
self.prepare_external_dms()
@@ -35,13 +35,6 @@ class Addref(OpenbisCommand):
return CommandResult(returncode=-1, output="This is not an obis repository.")
- def path(self):
- result = self.git_wrapper.git_top_level_path()
- if result.failure():
- return result
- return result.output
-
-
def commit_id(self):
result = self.git_wrapper.git_commit_hash()
if result.failure():
diff --git a/src/python/OBis/obis/dm/commands/clone.py b/src/python/OBis/obis/dm/commands/clone.py
index 77df6316bf3af934c5358aaafd74e231cb964e55..103a3f5a88ebe5a6e5a2a429bf36443618cd91ae 100644
--- a/src/python/OBis/obis/dm/commands/clone.py
+++ b/src/python/OBis/obis/dm/commands/clone.py
@@ -1,13 +1,11 @@
import socket
import os
import pybis
-from .openbis_command import OpenbisCommand
+from .openbis_command import OpenbisCommand, ContentCopySelector
from ..command_result import CommandResult
from ..utils import cd
from ..utils import run_shell
from ..utils import complete_openbis_config
-from .. import config as dm_config
-from ...scripts.cli import shared_data_mgmt
from ... import dm
@@ -25,16 +23,6 @@ class Clone(OpenbisCommand):
super(Clone, self).__init__(dm)
- def load_global_config(self, dm):
- """
- Use global config only.
- """
- resolver = dm_config.ConfigResolver()
- config = {}
- complete_openbis_config(config, resolver, False)
- dm.openbis_config = config
-
-
def check_configuration(self):
missing_config_settings = []
if self.openbis is None:
@@ -55,7 +43,7 @@ class Clone(OpenbisCommand):
data_set = self.openbis.get_dataset(self.data_set_id)
- content_copy = self.get_content_copy(data_set)
+ content_copy = ContentCopySelector(data_set, self.content_copy_index).select()
host = content_copy['externalDms']['address'].split(':')[0]
path = content_copy['path']
repository_folder = path.split('/')[-1]
@@ -69,41 +57,6 @@ class Clone(OpenbisCommand):
return self.add_content_copy_to_openbis(repository_folder)
- def get_content_copy(self, data_set):
- if data_set.data['kind'] != 'LINK':
- raise ValueError('Data set is of type ' + data_set.data['kind'] + ' but should be LINK.')
- content_copies = data_set.data['linkedData']['contentCopies']
- if len(content_copies) == 0:
- raise ValueError("Data set has no content copies.")
- elif len(content_copies) == 1:
- return content_copies[0]
- else:
- return self.select_content_copy(content_copies)
-
-
- def select_content_copy(self, content_copies):
- if self.content_copy_index is not None:
- # use provided content_copy_index
- if self.content_copy_index > 0 and self.content_copy_index <= len(content_copies):
- return content_copies[self.content_copy_index-1]
- else:
- raise ValueError("Invalid content copy index.")
- else:
- # ask user
- while True:
- print('From which content copy do you want to clone?')
- for i, content_copy in enumerate(content_copies):
- host = content_copy['externalDms']['address'].split(":")[0]
- path = content_copy['path']
- print(" {}) {}:{}".format(i+1, host, path))
-
- copy_index_string = input('> ')
- if copy_index_string.isdigit():
- copy_index_int = int(copy_index_string)
- if copy_index_int > 0 and copy_index_int <= len(content_copies):
- return content_copies[copy_index_int-1]
-
-
def copy_repository(self, ssh_user, host, path):
# abort if local folder already exists
repository_folder = path.split('/')[-1]
diff --git a/src/python/OBis/obis/dm/commands/download.py b/src/python/OBis/obis/dm/commands/download.py
new file mode 100644
index 0000000000000000000000000000000000000000..3aa1e55482e9baf5c0aeeea7b936f03dc9b59e24
--- /dev/null
+++ b/src/python/OBis/obis/dm/commands/download.py
@@ -0,0 +1,31 @@
+import pybis
+from .openbis_command import OpenbisCommand, ContentCopySelector
+from ..command_result import CommandResult
+
+
+class Download(OpenbisCommand):
+ """
+ Command to download files of a data set. Uses the microservice server to access the files.
+ As opposed to the clone, the user does not need to be able to access the files via ssh
+ and no new content copy is created in openBIS.
+ """
+
+
+ def __init__(self, dm, data_set_id, content_copy_index, file):
+ self.data_set_id = data_set_id
+ self.content_copy_index = content_copy_index
+ self.file = file
+ self.load_global_config(dm)
+ super(Download, self).__init__(dm)
+
+
+ def run(self):
+
+ if self.fileservice_url() is None:
+ return CommandResult(returncode=-1, output="Configuration fileservice_url needs to be set for download.")
+
+ data_set = self.openbis.get_dataset(self.data_set_id)
+ content_copy_index = ContentCopySelector(data_set, self.content_copy_index, get_index=True).select()
+ files = [self.file] if self.file is not None else None
+ output = data_set.download(files, linked_dataset_fileservice_url=self.fileservice_url(), content_copy_index=content_copy_index)
+ return CommandResult(returncode=0, output=output)
diff --git a/src/python/OBis/obis/dm/commands/openbis_command.py b/src/python/OBis/obis/dm/commands/openbis_command.py
index 81057f1dc3c9bddd910db3b8f3ecd8a25338e041..c37e1cc1c43c0e41a0327981d588711dd7a53e3d 100644
--- a/src/python/OBis/obis/dm/commands/openbis_command.py
+++ b/src/python/OBis/obis/dm/commands/openbis_command.py
@@ -4,6 +4,9 @@ import os
import socket
import pybis
from ..command_result import CommandResult
+from ..command_result import CommandException
+from .. import config as dm_config
+from ..utils import complete_openbis_config
from ...scripts import cli
@@ -13,38 +16,88 @@ class OpenbisCommand(object):
self.data_mgmt = dm
self.openbis = dm.openbis
self.git_wrapper = dm.git_wrapper
- self.config_resolver = dm.config_resolver
- self.config_dict = dm.config_resolver.config_dict()
+ self.settings_resolver = dm.settings_resolver
+ self.config_dict = dm.settings_resolver.config_dict()
if self.openbis is None and dm.openbis_config.get('url') is not None:
self.openbis = pybis.Openbis(**dm.openbis_config)
+ if self.user() is not None:
+ result = self.login()
+ if result.failure():
+ raise CommandException(result)
+
def external_dms_id(self):
- return self.config_dict.get('external_dms_id')
+ return self.config_dict['repository']['external_dms_id']
+
+ def set_external_dms_id(self, value):
+ self.config_dict['repository']['external_dms_id'] = value
def repository_id(self):
- return self.config_dict.get('repository_id')
+ return self.config_dict['repository']['id']
- def data_set_type(self):
- return self.config_dict.get('data_set_type')
+ def set_repository_id(self, value):
+ self.config_dict['repository']['id'] = value
def data_set_id(self):
- return self.config_dict.get('data_set_id')
+ return self.config_dict['repository']['data_set_id']
+
+ def set_data_set_id(self, value):
+ self.config_dict['repository']['data_set_id'] = value
+
+ def data_set_type(self):
+ return self.config_dict['data_set']['type']
+
+ def set_data_set_type(self, value):
+ self.config_dict['data_set']['type'] = value
def data_set_properties(self):
- return self.config_dict.get('data_set_properties')
+ return self.config_dict['data_set']['properties']
+
+ def set_data_set_properties(self, value):
+ self.config_dict['data_set']['properties'] = value
def object_id(self):
- return self.config_dict.get('object_id')
+ return self.config_dict['object']['id']
+
+ def set_object_id(self, value):
+ self.config_dict['object']['id'] = value
def collection_id(self):
- return self.config_dict.get('collection_id')
+ return self.config_dict['collection']['id']
+
+ def set_collection_id(self, value):
+ self.config_dict['collection']['id'] = value
def user(self):
- return self.config_dict.get('user')
+ return self.config_dict['config']['user']
+
+ def set_user(self, value):
+ self.config_dict['config']['user'] = value
def hostname(self):
- return self.config_dict.get('hostname')
+ return self.config_dict['config']['hostname']
+
+ def set_hostname(self, value):
+ self.config_dict['config']['hostname'] = value
+
+ def fileservice_url(self):
+ return self.config_dict['config']['fileservice_url']
+
+ def set_fileservice_url(self, value):
+ self.config_dict['config']['fileservice_url'] = value
+
+ def git_annex_hash_as_checksum(self):
+ return self.config_dict['config']['git_annex_hash_as_checksum']
+
+ def set_git_annex_hash_as_checksum(self, value):
+ self.config_dict['config']['git_annex_hash_as_checksum'] = value
+
+ def openbis_url(self):
+ return self.config_dict['config']['openbis_url']
+
+ def set_openbis_url(self, value):
+ self.config_dict['config']['openbis_url'] = value
def prepare_run(self):
result = self.check_configuration()
@@ -62,14 +115,17 @@ class OpenbisCommand(object):
def login(self):
- if self.openbis.is_session_active():
- return CommandResult(returncode=0, output="")
user = self.user()
+ if self.openbis.is_session_active():
+ if self.openbis.token.startswith(user):
+ return CommandResult(returncode=0, output="")
+ else:
+ self.openbis.logout()
passwd = getpass.getpass("Password for {}:".format(user))
try:
self.openbis.login(user, passwd, save_token=True)
except ValueError:
- msg = "Could not log into openbis {}".format(self.config_dict['openbis_url'])
+ msg = "Could not log into openbis {}".format(self.openbis_url())
return CommandResult(returncode=-1, output=msg)
return CommandResult(returncode=0, output='')
@@ -79,8 +135,8 @@ class OpenbisCommand(object):
if result.failure():
return result
external_dms = result.output
- self.config_resolver.set_value_for_parameter('external_dms_id', external_dms.code, 'local')
- self.config_dict['external_dms_id'] = external_dms.code
+ self.settings_resolver.repository.set_value_for_parameter('external_dms_id', external_dms.code, 'local')
+ self.set_external_dms_id(external_dms.code)
return result
def generate_external_data_management_system_code(self, user, hostname, edms_path):
@@ -119,7 +175,8 @@ class OpenbisCommand(object):
# ask user
hostname = self.ask_for_hostname(socket.gethostname())
# store
- cli.config_internal(self.data_mgmt, True, 'hostname', hostname)
+ resolver = self.data_mgmt.settings_resolver.config
+ cli.config_internal(self.data_mgmt, resolver, True, False, 'hostname', hostname)
return hostname
def ask_for_hostname(self, hostname):
@@ -129,3 +186,70 @@ class OpenbisCommand(object):
return hostname_input
else:
return hostname
+
+ def path(self):
+ result = self.git_wrapper.git_top_level_path()
+ if result.failure():
+ return result
+ return result.output
+
+ def load_global_config(self, dm):
+ """
+ Use global config only.
+ """
+ resolver = dm_config.SettingsResolver()
+ config = {}
+ complete_openbis_config(config, resolver, False)
+ dm.openbis_config = config
+
+
+class ContentCopySelector(object):
+
+
+ def __init__(self, data_set, content_copy_index, get_index=False):
+ self.data_set = data_set
+ self.content_copy_index = content_copy_index
+ self.get_index = get_index
+
+
+ def select(self):
+ content_copy_index = self.select_index()
+ if self.get_index == True:
+ return content_copy_index
+ else:
+ return self.data_set.data['linkedData']['contentCopies'][content_copy_index]
+
+
+ def select_index(self):
+ if self.data_set.data['kind'] != 'LINK':
+ raise ValueError('Data set is of type ' + self.data_set.data['kind'] + ' but should be LINK.')
+ content_copies = self.data_set.data['linkedData']['contentCopies']
+ if len(content_copies) == 0:
+ raise ValueError("Data set has no content copies.")
+ elif len(content_copies) == 1:
+ return 0
+ else:
+ return self.select_content_copy_index(content_copies)
+
+
+ def select_content_copy_index(self, content_copies):
+ if self.content_copy_index is not None:
+ # use provided content_copy_index
+ if self.content_copy_index >= 0 and self.content_copy_index < len(content_copies):
+ return self.content_copy_index
+ else:
+ raise ValueError("Invalid content copy index.")
+ else:
+ # ask user
+ while True:
+ print('From which location should the files be copied?')
+ for i, content_copy in enumerate(content_copies):
+ host = content_copy['externalDms']['address'].split(":")[0]
+ path = content_copy['path']
+ print(" {}) {}:{}".format(i, host, path))
+
+ copy_index_string = input('> ')
+ if copy_index_string.isdigit():
+ copy_index_int = int(copy_index_string)
+ if copy_index_int >= 0 and copy_index_int < len(content_copies):
+ return copy_index_int
diff --git a/src/python/OBis/obis/dm/commands/openbis_sync.py b/src/python/OBis/obis/dm/commands/openbis_sync.py
index 5a928fe36d14b6cfa5112ad65d45d292b6b4385f..e6e1476a2ec4cdaad5075a5cd252d009be5853f7 100644
--- a/src/python/OBis/obis/dm/commands/openbis_sync.py
+++ b/src/python/OBis/obis/dm/commands/openbis_sync.py
@@ -9,6 +9,12 @@ from .openbis_command import OpenbisCommand
class OpenbisSync(OpenbisCommand):
"""A command object for synchronizing with openBIS."""
+
+ def __init__(self, dm, ignore_missing_parent=False):
+ self.ignore_missing_parent = ignore_missing_parent
+ super(OpenbisSync, self).__init__(dm)
+
+
def check_configuration(self):
missing_config_settings = []
if self.openbis is None:
@@ -16,10 +22,9 @@ class OpenbisSync(OpenbisCommand):
if self.user() is None:
missing_config_settings.append('user')
if self.data_set_type() is None:
- missing_config_settings.append('data_set_type')
+ missing_config_settings.append('data_set type')
if self.object_id() is None and self.collection_id() is None:
- missing_config_settings.append('object_id')
- missing_config_settings.append('collection_id')
+ missing_config_settings.append('object id or collection id')
if len(missing_config_settings) > 0:
return CommandResult(returncode=-1,
output="Missing configuration settings for {}.".format(missing_config_settings))
@@ -47,7 +52,7 @@ class OpenbisSync(OpenbisCommand):
commit_id = result.output
sample_id = self.object_id()
experiment_id = self.collection_id()
- contents = GitRepoFileInfo(self.git_wrapper).contents()
+ contents = GitRepoFileInfo(self.git_wrapper).contents(git_annex_hash_as_checksum=self.git_annex_hash_as_checksum())
try:
data_set = self.openbis.new_git_data_set(data_set_type, top_level_path, commit_id, repository_id, external_dms.code,
sample=sample_id, experiment=experiment_id, properties=properties, parents=parent_data_set_id,
@@ -65,7 +70,7 @@ class OpenbisSync(OpenbisCommand):
repository_id = self.repository_id()
if self.repository_id() is None:
repository_id = str(uuid.uuid4())
- self.config_resolver.set_value_for_parameter('repository_id', repository_id, 'local')
+ self.settings_resolver.repository.set_value_for_parameter('id', repository_id, 'local')
return CommandResult(returncode=0, output=repository_id)
@@ -90,6 +95,10 @@ class OpenbisSync(OpenbisCommand):
return False
def continue_without_parent_data_set(self):
+
+ if self.ignore_missing_parent:
+ return True
+
while True:
print("The data set {} not found in openBIS".format(self.data_set_id()))
print("Create new data set without parent? (y/n)")
@@ -100,7 +109,7 @@ class OpenbisSync(OpenbisCommand):
return False
- def run(self):
+ def run(self, info_only=False):
ignore_parent = False
@@ -111,15 +120,20 @@ class OpenbisSync(OpenbisCommand):
return CommandResult(returncode=0, output="Nothing to sync.")
except ValueError as e:
if 'no such dataset' in str(e):
+ if info_only:
+ return CommandResult(returncode=-1, output="Parent data set not found in openBIS.")
ignore_parent = self.continue_without_parent_data_set()
if not ignore_parent:
return CommandResult(returncode=-1, output="Parent data set not found in openBIS.")
- elif 'Your session expired' in str(e):
- self.login()
- return self.run()
else:
raise e
+ if info_only:
+ if self.data_set_id() is None:
+ return CommandResult(returncode=-1, output="Not yet synchronized with openBIS.")
+ else:
+ return CommandResult(returncode=-1, output="There are git commits which have not been synchronized.")
+
# TODO Write mementos in case openBIS is unreachable
# - write a file to the .git/obis folder containing the commit id. Filename includes a timestamp so they can be sorted.
@@ -144,7 +158,7 @@ class OpenbisSync(OpenbisCommand):
self.commit_metadata_updates()
# Update data set id as last commit so we can easily revert it on failure
- self.config_resolver.set_value_for_parameter('data_set_id', data_set_code, 'local')
+ self.settings_resolver.repository.set_value_for_parameter('data_set_id', data_set_code, 'local')
self.commit_metadata_updates("data set id")
# create a data set, using the existing data set as a parent, if there is one
diff --git a/src/python/OBis/obis/dm/commands/removeref.py b/src/python/OBis/obis/dm/commands/removeref.py
new file mode 100644
index 0000000000000000000000000000000000000000..dd5edadfab17286b7246bc883ae4539ce436343f
--- /dev/null
+++ b/src/python/OBis/obis/dm/commands/removeref.py
@@ -0,0 +1,62 @@
+import json
+import os
+from .openbis_command import OpenbisCommand
+from ..command_result import CommandResult
+from ..utils import complete_openbis_config
+
+
+class Removeref(OpenbisCommand):
+ """
+ Command to add the current folder, which is supposed to be an obis repository, as
+ a new content copy to openBIS.
+ """
+
+ def __init__(self, dm):
+ super(Removeref, self).__init__(dm)
+
+
+ def run(self):
+ result = self.check_obis_repository()
+ if result.failure():
+ return result
+
+ data_set = self.openbis.get_dataset(self.data_set_id()).data
+
+ if data_set['linkedData'] is None:
+ return CommandResult(returncode=-1, output="Data set has no linked data: " + self.data_set_id())
+ if data_set['linkedData']['contentCopies'] is None:
+ return CommandResult(returncode=-1, output="Data set has no content copies: " + self.data_set_id())
+
+ content_copies = data_set['linkedData']['contentCopies']
+ matching_content_copies = list(filter(lambda cc:
+ cc['externalDms']['code'] == self.external_dms_id() and cc['path'] == self.path()
+ , content_copies))
+
+ if len(matching_content_copies) == 0:
+ return CommandResult(returncode=-1, output="Matching content copy not fount in data set: " + self.data_set_id())
+
+ for content_copy in matching_content_copies:
+ self.openbis.delete_content_copy(self.data_set_id(), content_copy)
+
+ return CommandResult(returncode=0, output="")
+
+
+ def check_obis_repository(self):
+ if os.path.exists('.obis'):
+ return CommandResult(returncode=0, output="")
+ else:
+ return CommandResult(returncode=-1, output="This is not an obis repository.")
+
+
+ def path(self):
+ result = self.git_wrapper.git_top_level_path()
+ if result.failure():
+ return result
+ return result.output
+
+
+ def commit_id(self):
+ result = self.git_wrapper.git_commit_hash()
+ if result.failure():
+ return result
+ return result.output
diff --git a/src/python/OBis/obis/dm/config.py b/src/python/OBis/obis/dm/config.py
index 18135858d4d903c217df26464622f952c4970bc2..d1d55c8059830f623d18d09febe8d54a7bc76ffe 100644
--- a/src/python/OBis/obis/dm/config.py
+++ b/src/python/OBis/obis/dm/config.py
@@ -30,7 +30,7 @@ class ConfigLocation(object):
class ConfigParam(object):
"""Class for configuration parameters."""
- def __init__(self, name, private, is_json=False, ignore_global=False):
+ def __init__(self, name, private, is_json=False, ignore_global=False, default_value=None):
"""
:param name: Name of the parameter.
:param private: Should the parameter be private to the repo or visible in the data set?
@@ -40,6 +40,7 @@ class ConfigParam(object):
self.private = private
self.is_json = is_json
self.ignore_global = ignore_global
+ self.default_value = default_value
def location_path(self, loc):
if loc == 'global':
@@ -53,9 +54,23 @@ class ConfigParam(object):
if not self.is_json:
return value
if isinstance(value, str):
- return json.loads(value)
+ return self.parse_json_value(value)
return value
+ def parse_json_value(self, value):
+ value_dict = json.loads(value, object_hook=self.json_upper)
+ return value_dict
+
+ def json_upper(self, obj):
+ for key in obj.keys():
+ new_key = key.upper()
+ if new_key != key:
+ if new_key in obj:
+ raise ValueError("Duplicate key after capitalizing JSON config: " + new_key)
+ obj[new_key] = obj[key]
+ del obj[key]
+ return obj
+
class ConfigEnv(object):
"""The environment in which configurations are constructed."""
@@ -85,13 +100,11 @@ class ConfigEnv(object):
def initialize_params(self):
self.add_param(ConfigParam(name='openbis_url', private=False))
+ self.add_param(ConfigParam(name='fileservice_url', private=False))
self.add_param(ConfigParam(name='user', private=True))
- self.add_param(ConfigParam(name='verify_certificates', private=True, is_json=True))
- self.add_param(ConfigParam(name='object_id', private=False, ignore_global=True))
- self.add_param(ConfigParam(name='collection_id', private=False, ignore_global=True))
- self.add_param(ConfigParam(name='data_set_type', private=False))
- self.add_param(ConfigParam(name='data_set_properties', private=False, is_json=True))
+ self.add_param(ConfigParam(name='verify_certificates', private=True, is_json=True, default_value=True))
self.add_param(ConfigParam(name='hostname', private=False))
+ self.add_param(ConfigParam(name='git_annex_hash_as_checksum', private=False, is_json=True, default_value=True))
def add_param(self, param):
self.params[param.name] = param
@@ -106,12 +119,31 @@ class ConfigEnv(object):
return True
-class PropertiesEnv(ConfigEnv):
+class CollectionEnv(ConfigEnv):
+
+ def initialize_params(self):
+ self.add_param(ConfigParam(name='id', private=False, ignore_global=True))
+
+
+class ObjectEnv(ConfigEnv):
+
+ def initialize_params(self):
+ self.add_param(ConfigParam(name='id', private=False, ignore_global=True))
+
+
+class DataSetEnv(ConfigEnv):
+
+ def initialize_params(self):
+ self.add_param(ConfigParam(name='type', private=False))
+ self.add_param(ConfigParam(name='properties', private=False, is_json=True))
+
+
+class RepositoryEnv(ConfigEnv):
""" These are properties which are not configured by the user but set by obis. """
def initialize_params(self):
+ self.add_param(ConfigParam(name='id', private=True))
self.add_param(ConfigParam(name='external_dms_id', private=True))
- self.add_param(ConfigParam(name='repository_id', private=True))
self.add_param(ConfigParam(name='data_set_id', private=False))
def is_usersetting(self):
@@ -130,16 +162,22 @@ class LocationResolver(object):
return os.path.join(root, location.basename)
-class ConfigResolverImpl(object):
+class ConfigResolver(object):
"""Construct a config dictionary."""
- def __init__(self, env=None, location_resolver=None, config_file='config.json'):
+ def __init__(self, env=None, location_resolver=None, categoty='config'):
self.env = env if env is not None else ConfigEnv()
self.location_resolver = location_resolver if location_resolver is not None else LocationResolver()
self.location_search_order = ['global', 'local']
self.location_cache = {}
self.is_initialized = False
- self.config_file = config_file
+ self.categoty = categoty
+
+ def set_location_search_order(self, order):
+ self.location_search_order = order
+
+ def set_resolver_location_roots(self, key, value):
+ self.location_resolver.location_roots[key] = value
def initialize_location_cache(self):
env = self.env
@@ -153,7 +191,7 @@ class ConfigResolverImpl(object):
self.initialize_location(k, v, cache[key])
else:
root_path = self.location_resolver.resolve_location(loc)
- config_path = os.path.join(root_path, self.config_file)
+ config_path = os.path.join(root_path, self.categoty + '.json')
if os.path.exists(config_path):
with open(config_path) as f:
config = json.load(f)
@@ -186,6 +224,9 @@ class ConfigResolverImpl(object):
:param loc: Either 'local' or 'global'
:return:
"""
+ if not name in self.env.params:
+ raise ValueError("Unknown setting {} for {}.".format(name, self.categoty))
+
if not self.is_initialized:
self.initialize_location_cache()
@@ -197,10 +238,33 @@ class ConfigResolverImpl(object):
location_dir_path = self.location_resolver.resolve_location(location)
if not os.path.exists(location_dir_path):
os.makedirs(location_dir_path)
- config_path = os.path.join(location_dir_path, self.config_file)
+ config_path = os.path.join(location_dir_path, self.categoty + '.json')
with open(config_path, "w") as f:
json.dump(location_config_dict, f, sort_keys=True)
+ def set_value_for_json_parameter(self, json_param_name, name, value, loc):
+ """Set one field for the json parameter
+ :param json_param_name: Name of the json parameter
+ :param name: Name of the field
+ :param loc: Either 'local' or 'global'
+ :return:
+ """
+ if not self.is_initialized:
+ self.initialize_location_cache()
+
+ param = self.env.params[json_param_name]
+
+ if not param.is_json:
+ raise ValueError('Can not set json value for non-json parameter: ' + json_param_name)
+
+ json_value = self.value_for_parameter(param, loc)
+ if json_value is None:
+ json_value = {}
+ json_value[name.upper()] = value
+
+ self.set_value_for_parameter(json_param_name, json.dumps(json_value), loc)
+
+
def value_for_parameter(self, param, loc):
config = self.location_cache[loc]
if loc != 'global':
@@ -208,7 +272,10 @@ class ConfigResolverImpl(object):
config = config['private']
else:
config = config['public']
- return config.get(param.name)
+ value = config.get(param.name)
+ if loc == 'global' and value is None:
+ value = param.default_value
+ return value
def set_cache_value_for_parameter(self, param, value, loc):
config = self.location_cache[loc]
@@ -222,7 +289,7 @@ class ConfigResolverImpl(object):
def local_public_properties_path(self):
loc = self.env.location_at_path(['local', 'public'])
- return self.location_resolver.resolve_location(loc) + '/' + self.config_file
+ return self.location_resolver.resolve_location(loc) + '/' + self.categoty + '.json'
def copy_global_to_local(self):
config = self.config_dict(False)
@@ -241,29 +308,33 @@ class ConfigResolverImpl(object):
return self.env.is_usersetting()
-class ConfigResolver(object):
+class SettingsResolver(object):
""" This class functions as a wrapper since we have multiple config resolvers. """
-
def __init__(self, location_resolver=None):
+ self.repository = ConfigResolver(location_resolver=location_resolver, env=RepositoryEnv(), categoty='repository')
+ self.data_set = ConfigResolver(location_resolver=location_resolver, env=DataSetEnv(), categoty='data_set')
+ self.object = ConfigResolver(location_resolver=location_resolver, env=ObjectEnv(), categoty='object')
+ self.collection = ConfigResolver(location_resolver=location_resolver, env=CollectionEnv(), categoty='collection')
+ self.config = ConfigResolver(location_resolver=location_resolver, env=ConfigEnv())
self.resolvers = []
- self.resolvers.append(ConfigResolverImpl(env=ConfigEnv()))
- self.resolvers.append(ConfigResolverImpl(env=PropertiesEnv(), config_file='properties.json'))
+ self.resolvers.append(self.repository)
+ self.resolvers.append(self.data_set)
+ self.resolvers.append(self.object)
+ self.resolvers.append(self.collection)
+ self.resolvers.append(self.config)
def config_dict(self, local_only=False):
combined_dict = {}
for resolver in self.resolvers:
- combined_dict.update(resolver.config_dict(local_only=local_only))
+ combined_dict[resolver.categoty] = resolver.config_dict(local_only=local_only)
return combined_dict
- def set_value_for_parameter(self, name, value, loc):
+ def local_public_properties_paths(self, get_usersettings=False):
+ paths = []
for resolver in self.resolvers:
- if name in resolver.env.params:
- return resolver.set_value_for_parameter(name, value, loc)
-
- def local_public_properties_path(self):
- for resolver in self.resolvers:
- if not resolver.is_usersetting():
- return resolver.local_public_properties_path()
+ if get_usersettings == resolver.is_usersetting():
+ paths.append(resolver.local_public_properties_path())
+ return paths
def copy_global_to_local(self):
for resolver in self.resolvers:
@@ -276,8 +347,3 @@ class ConfigResolver(object):
def set_location_search_order(self, order):
for resolver in self.resolvers:
resolver.location_search_order = order
-
- def is_usersetting(self, name):
- for resolver in self.resolvers:
- if name in resolver.env.params:
- return resolver.is_usersetting()
diff --git a/src/python/OBis/obis/dm/config_test.py b/src/python/OBis/obis/dm/config_test.py
index e5f5dc4cc65ef41df1935f8b825bb14c1d6fab2e..dfd1c79c38858935fc695f0bd68b991217fe74ca 100644
--- a/src/python/OBis/obis/dm/config_test.py
+++ b/src/python/OBis/obis/dm/config_test.py
@@ -38,20 +38,19 @@ def configure_resolver_for_test(resolver, tmpdir):
def test_read_config(tmpdir):
copy_user_config_test_data(tmpdir)
- resolver = config.ConfigResolver()
+ resolver = config.SettingsResolver().config
configure_resolver_for_test(resolver, tmpdir)
config_dict = resolver.config_dict()
assert config_dict is not None
with open(os.path.join(user_config_test_data_path(), ".obis", "config.json")) as f:
expected_dict = json.load(f)
assert config_dict['user'] == expected_dict['user']
-
- assert './.obis/properties.json' == resolver.local_public_properties_path()
+ assert './.obis/config.json' == resolver.local_public_properties_path()
def test_write_config(tmpdir):
copy_user_config_test_data(tmpdir)
- resolver = config.ConfigResolver()
+ resolver = config.SettingsResolver().config
configure_resolver_for_test(resolver, tmpdir)
config_dict = resolver.config_dict()
assert config_dict is not None
diff --git a/src/python/OBis/obis/dm/data_mgmt.py b/src/python/OBis/obis/dm/data_mgmt.py
index 0be42bf40750a8be00d5171bcf96d5ec9c6c2379..e9462f232fb1db0675fd3c3d5ff11a445ab54b50 100644
--- a/src/python/OBis/obis/dm/data_mgmt.py
+++ b/src/python/OBis/obis/dm/data_mgmt.py
@@ -14,11 +14,15 @@ import os
import shutil
import traceback
import pybis
+import requests
from . import config as dm_config
from .commands.addref import Addref
+from .commands.removeref import Removeref
from .commands.clone import Clone
from .commands.openbis_sync import OpenbisSync
+from .commands.download import Download
from .command_result import CommandResult
+from .command_result import CommandException
from .git import GitWrapper
from .utils import default_echo
from .utils import complete_git_config
@@ -28,7 +32,7 @@ from ..scripts import cli
# noinspection PyPep8Naming
-def DataMgmt(echo_func=None, config_resolver=None, openbis_config={}, git_config={}, openbis=None):
+def DataMgmt(echo_func=None, settings_resolver=None, openbis_config={}, git_config={}, openbis=None):
"""Factory method for DataMgmt instances"""
echo_func = echo_func if echo_func is not None else default_echo
@@ -36,16 +40,16 @@ def DataMgmt(echo_func=None, config_resolver=None, openbis_config={}, git_config
complete_git_config(git_config)
git_wrapper = GitWrapper(**git_config)
if not git_wrapper.can_run():
- return NoGitDataMgmt(config_resolver, None, git_wrapper, openbis)
+ return NoGitDataMgmt(settings_resolver, None, git_wrapper, openbis)
- if config_resolver is None:
- config_resolver = dm_config.ConfigResolver()
+ if settings_resolver is None:
+ settings_resolver = dm_config.SettingsResolver()
result = git_wrapper.git_top_level_path()
if result.success():
- config_resolver.set_resolver_location_roots('data_set', result.output)
- complete_openbis_config(openbis_config, config_resolver)
+ settings_resolver.set_resolver_location_roots('data_set', result.output)
+ complete_openbis_config(openbis_config, settings_resolver)
- return GitDataMgmt(config_resolver, openbis_config, git_wrapper, openbis)
+ return GitDataMgmt(settings_resolver, openbis_config, git_wrapper, openbis)
class AbstractDataMgmt(metaclass=abc.ABCMeta):
@@ -54,8 +58,8 @@ class AbstractDataMgmt(metaclass=abc.ABCMeta):
All operations throw an exepction if they fail.
"""
- def __init__(self, config_resolver, openbis_config, git_wrapper, openbis):
- self.config_resolver = config_resolver
+ def __init__(self, settings_resolver, openbis_config, git_wrapper, openbis):
+ self.settings_resolver = settings_resolver
self.openbis_config = openbis_config
self.git_wrapper = git_wrapper
self.openbis = openbis
@@ -85,7 +89,7 @@ class AbstractDataMgmt(metaclass=abc.ABCMeta):
return
@abc.abstractmethod
- def commit(self, msg, auto_add=True, sync=True):
+ def commit(self, msg, auto_add=True, ignore_missing_parent=False, sync=True):
"""Commit the current repo.
This issues a git commit and connects to openBIS and creates a data set in openBIS.
@@ -97,7 +101,7 @@ class AbstractDataMgmt(metaclass=abc.ABCMeta):
return
@abc.abstractmethod
- def sync(self):
+ def sync(self, ignore_missing_parent=False):
"""Sync the current repo.
This connects to openBIS and creates a data set in openBIS.
@@ -122,12 +126,24 @@ class AbstractDataMgmt(metaclass=abc.ABCMeta):
"""
return
+ @abc.abstractmethod
def addref(self):
"""Add the current folder as an obis repository to openBIS.
:return: A CommandResult.
"""
return
+ @abc.abstractmethod
+ def removeref(self):
+ """Remove the current folder / repository from openBIS.
+ :return: A CommandResult.
+ """
+ return
+
+ @abc.abstractmethod
+ def download(self, data_set_id, content_copy_index, file):
+ return
+
class NoGitDataMgmt(AbstractDataMgmt):
"""DataMgmt operations when git is not available -- show error messages."""
@@ -138,10 +154,10 @@ class NoGitDataMgmt(AbstractDataMgmt):
def init_analysis(self, path, parent, desc=None, create=True, apply_config=False):
self.error_raise("init analysis", "No git command found.")
- def commit(self, msg, auto_add=True, sync=True):
+ def commit(self, msg, auto_add=True, ignore_missing_parent=False, sync=True):
self.error_raise("commit", "No git command found.")
- def sync(self):
+ def sync(self, ignore_missing_parent=False):
self.error_raise("sync", "No git command found.")
def status(self):
@@ -153,15 +169,37 @@ class NoGitDataMgmt(AbstractDataMgmt):
def addref(self):
self.error_raise("addref", "No git command found.")
+ def removeref(self):
+ self.error_raise("removeref", "No git command found.")
+
+ def download(self, data_set_id, content_copy_index, file):
+ self.error_raise("download", "No git command found.")
+
+
+def with_restore(f):
+ def f_with_restore(self, *args):
+ self.set_restorepoint()
+ try:
+ result = f(self, *args)
+ if result.failure():
+ self.restore()
+ return result
+ except Exception as e:
+ self.restore()
+ return CommandResult(returncode=-1, output="Error: " + str(e))
+ return f_with_restore
+
class GitDataMgmt(AbstractDataMgmt):
"""DataMgmt operations in normal state."""
- def setup_local_config(self, config, path):
+ def setup_local_settings(self, all_settings, path):
with cd(path):
- self.config_resolver.set_resolver_location_roots('data_set', '.')
- for key, value in config.items():
- self.config_resolver.set_value_for_parameter(key, value, 'local')
+ self.settings_resolver.set_resolver_location_roots('data_set', '.')
+ for resolver_type, settings in all_settings.items():
+ resolver = getattr(self.settings_resolver, resolver_type)
+ for key, value in settings.items():
+ resolver.set_value_for_parameter(key, value, 'local')
def check_repository_state(self, path):
@@ -177,11 +215,11 @@ class GitDataMgmt(AbstractDataMgmt):
def get_data_set_id(self, path):
with cd(path):
- return self.config_resolver.config_dict().get('data_set_id')
+ return self.settings_resolver.repository.config_dict().get('data_set_id')
- def get_config(self, path, key):
+ def get_repository_id(self, path):
with cd(path):
- return self.config_resolver.config_dict().get(key)
+ return self.settings_resolver.repository.config_dict().get('id')
def init_data(self, path, desc=None, create=True, apply_config=False):
if not os.path.exists(path) and create:
@@ -194,8 +232,8 @@ class GitDataMgmt(AbstractDataMgmt):
return result
with cd(path):
# Update the resolvers location
- self.config_resolver.set_resolver_location_roots('data_set', '.')
- self.config_resolver.copy_global_to_local()
+ self.settings_resolver.set_resolver_location_roots('data_set', '.')
+ self.settings_resolver.copy_global_to_local()
self.commit_metadata_updates('local with global')
return result
@@ -206,7 +244,7 @@ class GitDataMgmt(AbstractDataMgmt):
parent_folder = parent if parent is not None and len(parent) > 0 else "."
parent_data_set_id = self.get_data_set_id(parent_folder)
# check that parent repository has been added to openBIS
- if self.get_config(parent_folder, 'repository_id') is None:
+ if self.get_repository_id(parent_folder) is None:
return CommandResult(returncode=-1, output="Parent data set must be committed to openBIS before creating an analysis data set.")
# check that analysis repository does not already exist
if os.path.exists(path):
@@ -222,37 +260,30 @@ class GitDataMgmt(AbstractDataMgmt):
return CommandResult(returncode=-1, output="Not within a repository and no parent set.")
# set data_set_id to analysis repository so it will be used as parent when committing
with cd(path):
- cli.set_property(self, "data_set_id", parent_data_set_id, False)
+ cli.set_property(self, self.settings_resolver.repository, "data_set_id", parent_data_set_id, False, False)
return result
- def sync(self):
- self.set_restorepoint()
- result = self._sync()
- if result.failure():
- self.restore()
- return result
+ @with_restore
+ def sync(self, ignore_missing_parent=False):
+ return self._sync(ignore_missing_parent)
- def _sync(self):
- try:
- cmd = OpenbisSync(self)
- return cmd.run()
- except Exception:
- traceback.print_exc()
- return CommandResult(returncode=-1, output="Could not synchronize with openBIS.")
+ def _sync(self, ignore_missing_parent=False):
+ cmd = OpenbisSync(self, ignore_missing_parent)
+ return cmd.run()
- def commit(self, msg, auto_add=True, sync=True, path=None):
+ def commit(self, msg, auto_add=True, ignore_missing_parent=False, sync=True, path=None):
if path is not None:
with cd(path):
- return self._commit(msg, auto_add, sync);
+ return self._commit(msg, auto_add, ignore_missing_parent, sync);
else:
- return self._commit(msg, auto_add, sync);
+ return self._commit(msg, auto_add, ignore_missing_parent, sync);
- def _commit(self, msg, auto_add=True, sync=True):
- self.set_restorepoint()
+ @with_restore
+ def _commit(self, msg, auto_add=True, ignore_missing_parent=False, sync=True):
if auto_add:
result = self.git_wrapper.git_top_level_path()
if result.failure():
@@ -265,21 +296,34 @@ class GitDataMgmt(AbstractDataMgmt):
# TODO If no changes were made check if the data set is in openbis. If not, just sync.
return result
if sync:
- result = self._sync()
- if result.failure():
- self.restore()
+ result = self._sync(ignore_missing_parent)
return result
+
def status(self):
- return self.git_wrapper.git_status()
+ git_status = self.git_wrapper.git_status()
+ try:
+ sync_status = OpenbisSync(self).run(info_only=True)
+ except requests.exceptions.ConnectionError:
+ sync_status = CommandResult(returncode=-1, output="Could not connect to openBIS.")
+ output = git_status.output
+ if sync_status.failure():
+ if len(output) > 0:
+ output += '\n'
+ output += sync_status.output
+ return CommandResult(returncode=0, output=output)
def commit_metadata_updates(self, msg_fragment=None):
- properties_path = self.config_resolver.local_public_properties_path()
- status = self.git_wrapper.git_status(properties_path)
- if len(status.output.strip()) < 1:
+ properties_paths = self.settings_resolver.local_public_properties_paths()
+ total_status = ''
+ for properties_path in properties_paths:
+ status = self.git_wrapper.git_status(properties_path).output.strip()
+ total_status += status
+ if len(status) > 0:
+ self.git_wrapper.git_add(properties_path)
+ if len(total_status) < 1:
# Nothing to commit
return CommandResult(returncode=0, output="")
- self.git_wrapper.git_add(properties_path)
if msg_fragment is None:
msg = "OBIS: Update openBIS metadata cache."
else:
@@ -291,19 +335,23 @@ class GitDataMgmt(AbstractDataMgmt):
def restore(self):
self.git_wrapper.git_reset_to(self.previous_git_commit_hash)
- properties_path = self.config_resolver.local_public_properties_path()
- self.git_wrapper.git_checkout(properties_path)
+ properties_paths = self.settings_resolver.local_public_properties_paths()
+ for properties_path in properties_paths:
+ self.git_wrapper.git_checkout(properties_path)
+ self.git_wrapper.git_delete_if_untracked(properties_path)
def clone(self, data_set_id, ssh_user, content_copy_index):
- try:
- cmd = Clone(self, data_set_id, ssh_user, content_copy_index)
- return cmd.run()
- except Exception as e:
- return CommandResult(returncode=-1, output="Error: " + str(e))
+ cmd = Clone(self, data_set_id, ssh_user, content_copy_index)
+ return cmd.run()
def addref(self):
- try:
- cmd = Addref(self)
- return cmd.run()
- except Exception as e:
- return CommandResult(returncode=-1, output="Error: " + str(e))
+ cmd = Addref(self)
+ return cmd.run()
+
+ def removeref(self):
+ cmd = Removeref(self)
+ return cmd.run()
+
+ def download(self, data_set_id, content_copy_index, file):
+ cmd = Download(self, data_set_id, content_copy_index, file)
+ return cmd.run()
diff --git a/src/python/OBis/obis/dm/data_mgmt_test.py b/src/python/OBis/obis/dm/data_mgmt_test.py
index 729811057662fd20f90a47b31bbe1985e24b7faf..acb0ea77e8fcf1920464cd28e16f98d4a7cb7866 100644
--- a/src/python/OBis/obis/dm/data_mgmt_test.py
+++ b/src/python/OBis/obis/dm/data_mgmt_test.py
@@ -59,14 +59,14 @@ def git_status(path=None, annex=False):
def check_correct_config_semantics():
# This how things should work
- with open('.obis/properties.json') as f:
+ with open('.obis/repository.json') as f:
config_local = json.load(f)
assert config_local.get('data_set_id') is not None
def check_workaround_config_semantics():
# This how things should work
- with open('.obis/properties.json') as f:
+ with open('.obis/repository.json') as f:
config_local = json.load(f)
assert config_local.get('data_set_id') is None
@@ -93,7 +93,7 @@ def test_data_use_case(tmpdir):
raw_status = git_status()
status = dm.status()
assert raw_status.returncode == status.returncode
- assert raw_status.output == status.output
+ assert raw_status.output + '\nNot yet synchronized with openBIS.' == status.output
assert len(status.output) > 0
result = dm.commit("Added data.")
@@ -120,7 +120,7 @@ def test_data_use_case(tmpdir):
assert stat.st_nlink == 1
status = dm.status()
- assert len(status.output) == 0
+ assert status.output == 'There are git commits which have not been synchronized.'
check_correct_config_semantics()
@@ -142,7 +142,7 @@ def test_child_data_set(tmpdir):
result = dm.commit("Added data.")
assert result.returncode == 0
- parent_ds_code = dm.config_resolver.config_dict()['data_set_id']
+ parent_ds_code = dm.settings_resolver.config_dict()['repository']['data_set_id']
update_test_data(tmpdir)
properties = {'DESCRIPTION': 'Updated content.'}
@@ -150,13 +150,13 @@ def test_child_data_set(tmpdir):
prepare_new_data_set_expectations(dm, properties)
result = dm.commit("Updated data.")
assert result.returncode == 0
- child_ds_code = dm.config_resolver.config_dict()['data_set_id']
+ child_ds_code = dm.settings_resolver.config_dict()['repository']['data_set_id']
assert parent_ds_code != child_ds_code
commit_id = dm.git_wrapper.git_commit_hash().output
- repository_id = dm.config_resolver.config_dict()['repository_id']
+ repository_id = dm.settings_resolver.config_dict()['repository']['id']
assert repository_id is not None
- contents = git.GitRepoFileInfo(dm.git_wrapper).contents()
+ contents = git.GitRepoFileInfo(dm.git_wrapper).contents(git_annex_hash_as_checksum=True)
check_new_data_set_expectations(dm, tmp_dir_path, commit_id, repository_id, ANY, child_ds_code, parent_ds_code,
properties, contents)
@@ -190,7 +190,7 @@ def test_undo_commit_when_sync_fails(tmpdir):
dm.git_wrapper.git_top_level_path = MagicMock(return_value = CommandResult(returncode=0, output=None))
dm.git_wrapper.git_add = MagicMock(return_value = CommandResult(returncode=0, output=None))
dm.git_wrapper.git_commit = MagicMock(return_value = CommandResult(returncode=0, output=None))
- dm.sync = lambda: CommandResult(returncode=-1, output="dummy error")
+ dm._sync = lambda: CommandResult(returncode=-1, output="dummy error")
# when
result = dm.commit("Added data.")
# then
@@ -215,7 +215,7 @@ def test_init_analysis(tmpdir):
result = dm.commit("Added data.")
assert result.returncode == 0
- parent_ds_code = dm.config_resolver.config_dict()['data_set_id']
+ parent_ds_code = dm.settings_resolver.config_dict()['repository']['data_set_id']
analysis_repo = "analysis"
result = dm.init_analysis(analysis_repo, None)
@@ -227,13 +227,13 @@ def test_init_analysis(tmpdir):
prepare_new_data_set_expectations(dm)
result = dm.commit("Analysis.")
assert result.returncode == 0
- child_ds_code = dm.config_resolver.config_dict()['data_set_id']
+ child_ds_code = dm.settings_resolver.config_dict()['repository']['data_set_id']
assert parent_ds_code != child_ds_code
commit_id = dm.git_wrapper.git_commit_hash().output
- repository_id = dm.config_resolver.config_dict()['repository_id']
+ repository_id = dm.settings_resolver.config_dict()['repository']['id']
assert repository_id is not None
- contents = git.GitRepoFileInfo(dm.git_wrapper).contents()
+ contents = git.GitRepoFileInfo(dm.git_wrapper).contents(git_annex_hash_as_checksum=True)
check_new_data_set_expectations(dm, tmp_dir_path + '/' + analysis_repo, commit_id, repository_id, ANY, child_ds_code, parent_ds_code,
None, contents)
@@ -241,13 +241,13 @@ def test_init_analysis(tmpdir):
# TODO Test that if the data set registration fails, the data_set_id is reverted
def set_registration_configuration(dm, properties=None):
- resolver = dm.config_resolver
- resolver.set_value_for_parameter('openbis_url', "http://localhost:8888", 'local')
- resolver.set_value_for_parameter('user', "auser", 'local')
- resolver.set_value_for_parameter('data_set_type', "DS_TYPE", 'local')
- resolver.set_value_for_parameter('object_id', "/SAMPLE/ID", 'local')
+ resolver = dm.settings_resolver
+ resolver.config.set_value_for_parameter('openbis_url', "http://localhost:8888", 'local')
+ resolver.config.set_value_for_parameter('user', "auser", 'local')
+ resolver.data_set.set_value_for_parameter('type', "DS_TYPE", 'local')
+ resolver.object.set_value_for_parameter('id', "/SAMPLE/ID", 'local')
if properties is not None:
- resolver.set_value_for_parameter('data_set_properties', properties, 'local')
+ resolver.data_set.set_value_for_parameter('properties', properties, 'local')
def prepare_registration_expectations(dm):
diff --git a/src/python/OBis/obis/dm/git-annex-attributes b/src/python/OBis/obis/dm/git-annex-attributes
index c6cf20e09e3c117cb5b1579fc47018e237e6c0a2..8164b72c5f61d12fd50340acc8028bbef4859d5c 100644
--- a/src/python/OBis/obis/dm/git-annex-attributes
+++ b/src/python/OBis/obis/dm/git-annex-attributes
@@ -1,3 +1,4 @@
+* annex.backend=SHA256E
* annex.largefiles=(largerthan=100kb)
*.zip annex.largefiles=anything
*.gz annex.largefiles=anything
diff --git a/src/python/OBis/obis/dm/git.py b/src/python/OBis/obis/dm/git.py
index cdfea134597969b313553558271f58c6ed70a021..ec5f573c1caf32ec151724ebbac76dcfdc75ea54 100644
--- a/src/python/OBis/obis/dm/git.py
+++ b/src/python/OBis/obis/dm/git.py
@@ -1,6 +1,10 @@
+from abc import ABC, abstractmethod
+import hashlib
+import json
import shutil
import os
from .utils import run_shell
+from .command_result import CommandException
class GitWrapper(object):
@@ -29,9 +33,9 @@ class GitWrapper(object):
def git_status(self, path=None):
if path is None:
- return run_shell([self.git_path, "status", "--porcelain"])
+ return run_shell([self.git_path, "status", "--porcelain"], strip_leading_whitespace=False)
else:
- return run_shell([self.git_path, "status", "--porcelain", path])
+ return run_shell([self.git_path, "status", "--porcelain", path], strip_leading_whitespace=False)
def git_annex_init(self, path, desc):
cmd = [self.git_path, "-C", path, "annex", "init", "--version=6"]
@@ -86,6 +90,10 @@ class GitWrapper(object):
gitignore.write(path)
gitignore.write("\n")
+ def git_delete_if_untracked(self, file):
+ result = run_shell([self.git_path, 'ls-files', '--error-unmatch', file])
+ if 'did not match' in result.output:
+ run_shell(['rm', file])
class GitRepoFileInfo(object):
"""Class that gathers checksums and file lengths for all files in the repo."""
@@ -93,16 +101,18 @@ class GitRepoFileInfo(object):
def __init__(self, git_wrapper):
self.git_wrapper = git_wrapper
- def contents(self):
+ def contents(self, git_annex_hash_as_checksum=False):
"""Return a list of dicts describing the contents of the repo.
:return: A list of dictionaries
{'crc32': checksum,
+ 'checksum': checksum other than crc32
+ 'checksumType': type of checksum
'fileLength': size of the file,
'path': path relative to repo root.
'directory': False
}"""
files = self.file_list()
- cksum = self.cksum(files)
+ cksum = self.cksum(files, git_annex_hash_as_checksum)
return cksum
def file_list(self):
@@ -113,20 +123,135 @@ class GitRepoFileInfo(object):
files = [line.split("\t")[-1].strip() for line in lines]
return files
- def cksum(self, files):
- cmd = ['cksum']
- cmd.extend(files)
- result = run_shell(cmd)
- if result.failure():
- return []
- lines = result.output.split("\n")
- return [self.checksum_line_to_dict(line) for line in lines]
+ def cksum(self, files, git_annex_hash_as_checksum=False):
+
+ if git_annex_hash_as_checksum == False:
+ checksum_generator = ChecksumGeneratorCrc32()
+ else:
+ checksum_generator = ChecksumGeneratorGitAnnex()
+
+ checksums = []
+
+ for file in files:
+ checksum = checksum_generator.get_checksum(file)
+ checksums.append(checksum)
+
+ return checksums
- @staticmethod
- def checksum_line_to_dict(line):
- fields = line.split(" ")
+
+class ChecksumGeneratorCrc32(object):
+ def get_checksum(self, file):
+ result = run_shell(['cksum', file])
+ if result.failure():
+ raise CommandException(result)
+ fields = result.output.split(" ")
return {
'crc32': int(fields[0]),
'fileLength': int(fields[1]),
- 'path': fields[2]
- }
\ No newline at end of file
+ 'path': file
+ }
+
+
+class ChecksumGeneratorHashlib(ABC):
+ @abstractmethod
+ def hash_function(self):
+ pass
+ @abstractmethod
+ def hash_type(self):
+ pass
+
+ def get_checksum(self, file):
+ return {
+ 'checksum': self._checksum(file),
+ 'checksumType': self.hash_type(),
+ 'fileLength': os.path.getsize(file),
+ 'path': file
+ }
+
+ def _checksum(self, file):
+ hash_function = self.hash_function()
+ with open(file, "rb") as f:
+ for chunk in iter(lambda: f.read(4096), b""):
+ hash_function.update(chunk)
+ return hash_function.hexdigest()
+
+
+class ChecksumGeneratorSha256(ChecksumGeneratorHashlib):
+ def hash_function(self):
+ return hashlib.sha256()
+ def hash_type(self):
+ return 'SHA256'
+
+
+class ChecksumGeneratorMd5(ChecksumGeneratorHashlib):
+ def hash_function(self):
+ return hashlib.md5()
+ def hash_type(self):
+ return "MD5"
+
+
+class ChecksumGeneratorWORM(object):
+ def get_checksum(self, file):
+ return {
+ 'checksum': self.worm(file),
+ 'checksumType': 'WORM',
+ 'fileLength': os.path.getsize(file),
+ 'path': file
+ }
+ def worm(self, file):
+ modification_time = int(os.path.getmtime(file))
+ size = os.path.getsize(file)
+ return "WORM-s{}-m{}--{}".format(size, modification_time, file)
+
+
+class ChecksumGeneratorGitAnnex(object):
+
+ def __init__(self):
+ self.backend = self._get_annex_backend()
+ self.checksum_generator_replacement = ChecksumGeneratorCrc32() if self.backend is None else None
+ # define which generator to use for files which are not handled by annex
+ if self.backend == 'SHA256':
+ self.checksum_generator_supplement = ChecksumGeneratorSha256()
+ elif self.backend == 'MD5':
+ self.checksum_generator_supplement = ChecksumGeneratorMd5()
+ elif self.backend == 'WORM':
+ self.checksum_generator_supplement = ChecksumGeneratorWORM()
+ else:
+ self.checksum_generator_supplement = ChecksumGeneratorCrc32()
+
+ def get_checksum(self, file):
+ if self.checksum_generator_replacement is not None:
+ return self.checksum_generator_replacement.get_checksum(file)
+ return self._get_checksum(file)
+
+ def _get_checksum(self, file):
+ annex_result = run_shell(['git', 'annex', 'info', '-j', file], raise_exception_on_failure=True)
+ if 'Not a valid object name' in annex_result.output:
+ return self.checksum_generator_supplement.get_checksum(file)
+ annex_info = json.loads(annex_result.output)
+ if annex_info['present'] != True:
+ return self.checksum_generator_supplement.get_checksum(file)
+ return {
+ 'checksum': self._get_checksum_from_annex_info(annex_info),
+ 'checksumType': self.backend,
+ 'fileLength': os.path.getsize(file),
+ 'path': file
+ }
+
+ def _get_checksum_from_annex_info(self, annex_info):
+ if self.backend in ['MD5', 'SHA256']:
+ return annex_info['key'].split('--')[1]
+ elif self.backend == 'WORM':
+ return annex_info['key'][5:]
+ else:
+ raise ValueError("Git annex backend not supported: " + self.backend)
+
+ def _get_annex_backend(self):
+ with open('.gitattributes') as gitattributes:
+ for line in gitattributes.readlines():
+ if 'annex.backend' in line:
+ backend = line.split('=')[1].strip()
+ if backend == 'SHA256E':
+ backend = 'SHA256'
+ return backend
+ return None
diff --git a/src/python/OBis/obis/dm/repo.py b/src/python/OBis/obis/dm/repo.py
index 9a5060e136b4e61f85471b5563336168606bd61d..9e6599744bbc3a3c589cc5c07404679998513ee9 100644
--- a/src/python/OBis/obis/dm/repo.py
+++ b/src/python/OBis/obis/dm/repo.py
@@ -20,7 +20,7 @@ class DataRepo(object):
"""
self.root = root
self.dm_api = data_mgmt.DataMgmt(git_config={'find_git': True})
- self.dm_api.config_resolver.set_resolver_location_roots('data_set', self.root)
+ self.dm_api.settings_resolver.set_resolver_location_roots('data_set', self.root)
def init(self, desc=None):
return self.dm_api.init_data(self.root, desc)
diff --git a/src/python/OBis/obis/dm/utils.py b/src/python/OBis/obis/dm/utils.py
index 0a4a52e0cde621a2591ef51ecd8366453ad9d479..751722d2968b216981a1cf7100e5f2b2b6d29637 100644
--- a/src/python/OBis/obis/dm/utils.py
+++ b/src/python/OBis/obis/dm/utils.py
@@ -1,19 +1,16 @@
import subprocess
import os
from contextlib import contextmanager
-from .command_result import CommandResult
+from .command_result import CommandResult, CommandException
def complete_openbis_config(config, resolver, local_only=True):
"""Add default values for empty entries in the config."""
- config_dict = resolver.config_dict(local_only)
+ config_dict = resolver.config.config_dict(local_only)
if config.get('url') is None:
config['url'] = config_dict['openbis_url']
if config.get('verify_certificates') is None:
- if config_dict.get('verify_certificates') is not None:
- config['verify_certificates'] = config_dict['verify_certificates']
- else:
- config['verify_certificates'] = True
+ config['verify_certificates'] = config_dict['verify_certificates']
if config.get('token') is None:
config['token'] = None
@@ -37,8 +34,11 @@ def default_echo(details):
print(details['message'])
-def run_shell(args, shell=False):
- return CommandResult(subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell))
+def run_shell(args, shell=False, strip_leading_whitespace=True, raise_exception_on_failure=False):
+ result = CommandResult(subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell), strip_leading_whitespace=strip_leading_whitespace)
+ if raise_exception_on_failure == True and result.failure():
+ raise CommandException(result)
+ return result
def locate_command(command):
diff --git a/src/python/OBis/obis/scripts/cli.py b/src/python/OBis/obis/scripts/cli.py
index 848eb3d318b7099286d7eeff8670d72359993805..f851972656015d7650c86ced48f547f8f98d87f4 100644
--- a/src/python/OBis/obis/scripts/cli.py
+++ b/src/python/OBis/obis/scripts/cli.py
@@ -17,6 +17,7 @@ import click
from .. import dm
from ..dm.command_result import CommandResult
+from ..dm.command_result import CommandException
from ..dm.utils import cd
@@ -35,6 +36,14 @@ def click_progress_no_ts(progress_data):
click.echo("{}".format(progress_data['message']))
+def add_params(params):
+ def _add_params(func):
+ for param in reversed(params):
+ func = param(func)
+ return func
+ return _add_params
+
+
def shared_data_mgmt(context={}):
git_config = {'find_git': True}
openbis_config = {}
@@ -51,6 +60,15 @@ def check_result(command, result):
return result.returncode
+def run(function):
+ try:
+ return function()
+ except CommandException as e:
+ return e.command_result
+ except Exception as e:
+ return CommandResult(returncode=-1, output="Error: " + str(e))
+
+
@click.group()
@click.option('-q', '--quiet', default=False, is_flag=True, help='Suppress status reporting.')
@click.option('-s', '--skip_verification', default=False, is_flag=True, help='Do not verify cerficiates')
@@ -61,56 +79,147 @@ def cli(ctx, quiet, skip_verification):
ctx.obj['verify_certificates'] = False
-@cli.command()
-@click.pass_context
-@click.argument('repository', type=click.Path(exists=True))
-def addref(ctx, repository):
- """Add a reference to the other repository in this repository.
- """
- with cd(repository):
- data_mgmt = shared_data_mgmt(ctx.obj)
- return check_result("addref", data_mgmt.addref())
-
-
-@cli.command()
-@click.pass_context
-@click.option('-u', '--ssh_user', default=None, help='User to connect to remote systems via ssh')
-@click.option('-c', '--content_copy_index', type=int, default=None, help='Index of the content copy to clone from in case there are multiple copies')
-@click.argument('data_set_id')
-def clone(ctx, ssh_user, content_copy_index, data_set_id):
- """Clone the repository found in the given data set id.
- """
- data_mgmt = shared_data_mgmt(ctx.obj)
- return check_result("clone", data_mgmt.clone(data_set_id, ssh_user, content_copy_index))
+def set_property(data_mgmt, resolver, prop, value, is_global, is_data_set_property=False):
+ """Helper function to implement the property setting semantics."""
+ loc = 'global' if is_global else 'local'
+ try:
+ if is_data_set_property:
+ resolver.set_value_for_json_parameter('properties', prop, value, loc)
+ else:
+ resolver.set_value_for_parameter(prop, value, loc)
+ except ValueError as e:
+ return CommandResult(returncode=-1, output="Error: " + str(e))
+ if not is_global:
+ return data_mgmt.commit_metadata_updates(prop)
+ else:
+ return CommandResult(returncode=0, output="")
-@cli.command()
-@click.pass_context
-@click.option('-m', '--msg', prompt=True, help='A message explaining what was done.')
-@click.option('-a', '--auto_add', default=True, is_flag=True, help='Automatically add all untracked files.')
-def commit(ctx, msg, auto_add):
- """Commit the repository to git and inform openBIS.
- """
+def init_data_impl(ctx, object_id, collection_id, repository, desc):
+ """Shared implementation for the init_data command."""
+ if repository is None:
+ repository = "."
+ click_echo("init_data {}".format(repository))
data_mgmt = shared_data_mgmt(ctx.obj)
- return check_result("commit", data_mgmt.commit(msg, auto_add))
+ desc = desc if desc != "" else None
+ result = run(lambda: data_mgmt.init_data(repository, desc, create=True))
+ init_handle_cleanup(result, object_id, collection_id, repository, data_mgmt)
-@cli.command()
-@click.option('-g', '--is_global', default=False, is_flag=True, help='Configure global or local.')
-@click.argument('prop', default="")
-@click.argument('value', default="")
-@click.pass_context
-def config(ctx, is_global, prop, value):
- """Configure the openBIS setup.
-
- Configure the openBIS server url, the data set type, and the data set properties.
- """
+def init_analysis_impl(ctx, parent, object_id, collection_id, repository, description):
+ click_echo("init_analysis {}".format(repository))
data_mgmt = shared_data_mgmt(ctx.obj)
- config_internal(data_mgmt, is_global, prop, value)
+ description = description if description != "" else None
+ result = run(lambda: data_mgmt.init_analysis(repository, parent, description, create=True))
+ init_handle_cleanup(result, object_id, collection_id, repository, data_mgmt)
-def config_internal(data_mgmt, is_global, prop, value):
- resolver = data_mgmt.config_resolver
+def init_handle_cleanup(result, object_id, collection_id, repository, data_mgmt):
+ if (not object_id and not collection_id) or result.failure():
+ return check_result("init_data", result)
+ with dm.cd(repository):
+ if object_id:
+ resolver = data_mgmt.object
+ return check_result("init_data", set_property(data_mgmt, resolver, 'id', object_id, False, False))
+ if collection_id:
+ resolver = data_mgmt.collection
+ return check_result("init_data", set_property(data_mgmt, resolver, 'id', collection_id, False, False))
+
+
+# settings commands
+
+
+class SettingsGet(click.ParamType):
+ name = 'settings_get'
+
+ def convert(self, value, param, ctx):
+ try:
+ split = list(filter(lambda term: len(term) > 0, value.split(',')))
+ return split
+ except:
+ self._fail(param)
+
+ def _fail(self, param):
+ self.fail(param=param, message='Settings must be in the format: key1, key2, ...')
+
+
+class SettingsClear(SettingsGet):
+ pass
+
+
+class SettingsSet(click.ParamType):
+ name = 'settings_set'
+
+ def convert(self, value, param, ctx):
+ try:
+ value = self._encode_json(value)
+ settings = {}
+ split = list(filter(lambda term: len(term) > 0, value.split(',')))
+ for setting in split:
+ setting_split = setting.split('=')
+ if len(setting_split) != 2:
+ self._fail(param)
+ key = setting_split[0]
+ value = setting_split[1]
+ settings[key] = self._decode_json(value)
+ return settings
+ except:
+ self._fail(param)
+
+ def _encode_json(self, value):
+ encoded = ''
+ SEEK = 0
+ ENCODE = 1
+ mode = SEEK
+ for char in value:
+ if char == '{':
+ mode = ENCODE
+ elif char == '}':
+ mode = SEEK
+ if mode == SEEK:
+ encoded += char
+ elif mode == ENCODE:
+ encoded += char.replace(',', '|')
+ return encoded
+
+ def _decode_json(self, value):
+ return value.replace('|', ',')
+
+ def _fail(self, param):
+ self.fail(param=param, message='Settings must be in the format: key1=value1, key2=value2, ...')
+
+
+def _join_settings_set(setting_dicts):
+ joined = {}
+ for setting_dict in setting_dicts:
+ for key, value in setting_dict.items():
+ joined[key] = value
+ return joined
+
+
+def _join_settings_get(setting_lists):
+ joined = []
+ for setting_list in setting_lists:
+ joined += setting_list
+ return joined
+
+
+def config_internal(data_mgmt, resolver, is_global, is_data_set_property, prop=None, value=None, set=False, get=False, clear=False):
+ if set == True:
+ assert get == False
+ assert clear == False
+ assert prop is not None
+ assert value is not None
+ elif get == True:
+ assert set == False
+ assert clear == False
+ assert value is None
+ elif clear == True:
+ assert get == False
+ assert set == False
+ assert value is None
+
+ assert set == True or get == True or clear == True
if is_global:
resolver.set_location_search_order(['global'])
else:
@@ -122,115 +231,509 @@ def config_internal(data_mgmt, is_global, prop, value):
resolver.set_location_search_order(['global'])
config_dict = resolver.config_dict()
- if not prop:
- config_str = json.dumps(config_dict, indent=4, sort_keys=True)
- click.echo("{}".format(config_str))
- elif not value:
- little_dict = {prop: config_dict[prop]}
- config_str = json.dumps(little_dict, indent=4, sort_keys=True)
- click.echo("{}".format(config_str))
- else:
- return check_result("config", set_property(data_mgmt, prop, value, is_global))
+ if is_data_set_property:
+ config_dict = config_dict['properties']
+ if get == True:
+ if prop is None:
+ config_str = json.dumps(config_dict, indent=4, sort_keys=True)
+ click.echo("{}".format(config_str))
+ else:
+ if not prop in config_dict:
+ raise ValueError("Unknown setting {} for {}.".format(prop, resolver.categoty))
+ little_dict = {prop: config_dict[prop]}
+ config_str = json.dumps(little_dict, indent=4, sort_keys=True)
+ click.echo("{}".format(config_str))
+ elif set == True:
+ return check_result("config", set_property(data_mgmt, resolver, prop, value, is_global, is_data_set_property))
+ elif clear == True:
+ if prop is None:
+ returncode = 0
+ for prop in config_dict.keys():
+ returncode += check_result("config", set_property(data_mgmt, resolver, prop, None, is_global, is_data_set_property))
+ return returncode
+ else:
+ return check_result("config", set_property(data_mgmt, resolver, prop, None, is_global, is_data_set_property))
-def set_property(data_mgmt, prop, value, is_global):
- """Helper function to implement the property setting semantics."""
- loc = 'global' if is_global else 'local'
- resolver = data_mgmt.config_resolver
- resolver.set_value_for_parameter(prop, value, loc)
- if not is_global:
- return data_mgmt.commit_metadata_updates(prop)
- else:
- return CommandResult(returncode=0, output="")
+def _access_settings(ctx, prop=None, value=None, set=False, get=False, clear=False):
+ is_global = ctx.obj['is_global']
+ data_mgmt = ctx.obj['data_mgmt']
+ resolver = ctx.obj['resolver']
+ is_data_set_property = False
+ if 'is_data_set_property' in ctx.obj:
+ is_data_set_property = ctx.obj['is_data_set_property']
+ config_internal(data_mgmt, resolver, is_global, is_data_set_property, prop=prop, value=value, set=set, get=get, clear=clear)
-def init_data_impl(ctx, object_id, collection_id, folder, desc):
- """Shared implementation for the init_data command."""
- click_echo("init_data {}".format(folder))
- data_mgmt = shared_data_mgmt(ctx.obj)
- desc = desc if desc != "" else None
- result = data_mgmt.init_data(folder, desc, create=True)
- init_handle_cleanup(result, object_id, collection_id)
+def _set(ctx, settings):
+ settings_dict = _join_settings_set(settings)
+ for prop, value in settings_dict.items():
+ _access_settings(ctx, prop=prop, value=value, set=True)
+ return CommandResult(returncode=0, output='')
+
+
+def _get(ctx, settings):
+ settings_list = _join_settings_get(settings)
+ if len(settings_list) == 0:
+ settings_list = [None]
+ for prop in settings_list:
+ _access_settings(ctx, prop=prop, get=True)
+ return CommandResult(returncode=0, output='')
+
+def _clear(ctx, settings):
+ settings_list = _join_settings_get(settings)
+ if len(settings_list) == 0:
+ settings_list = [None]
+ for prop in settings_list:
+ _access_settings(ctx, prop=prop, clear=True)
+ return CommandResult(returncode=0, output='')
-def init_analysis_impl(ctx, parent, object_id, collection_id, folder, description):
- click_echo("init_analysis {}".format(folder))
+
+## get all settings
+
+@cli.group()
+@click.option('-g', '--is_global', default=False, is_flag=True, help='Get global or local.')
+@click.pass_context
+def settings(ctx, is_global):
+ """ Get all settings.
+ """
+ ctx.obj['is_global'] = is_global
+
+
+@settings.command('get')
+@click.pass_context
+def settings_get(ctx):
data_mgmt = shared_data_mgmt(ctx.obj)
- description = description if description != "" else None
- result = data_mgmt.init_analysis(folder, parent, description, create=True)
- init_handle_cleanup(result, object_id, collection_id)
+ settings = data_mgmt.settings_resolver.config_dict()
+ settings_str = json.dumps(settings, indent=4, sort_keys=True)
+ click.echo("{}".format(settings_str))
-def init_handle_cleanup(result, object_id, collection_id):
- if (not object_id and not collection_id) or result.failure():
- return check_result("init_data", result)
- with dm.cd(folder):
- if object_id:
- return check_result("init_data", set_property(data_mgmt, 'object_id', object_id, False))
- if collection_id:
- return check_result("init_data", set_property(data_mgmt, 'collection_id', collection_id, False))
+## repository: repository_id, external_dms_id, data_set_id
+
+@cli.group()
+@click.option('-g', '--is_global', default=False, is_flag=True, help='Set/get global or local.')
+@click.pass_context
+def repository(ctx, is_global):
+ """ Get/set settings related to the repository.
+ """
+ ctx.obj['is_global'] = is_global
+ ctx.obj['data_mgmt'] = shared_data_mgmt(ctx.obj)
+ ctx.obj['resolver'] = ctx.obj['data_mgmt'].settings_resolver.repository
+
+
+@repository.command('set')
+@click.argument('settings', type=SettingsSet(), nargs=-1)
+@click.pass_context
+def repository_set(ctx, settings):
+ return check_result("repository_set", run(lambda: _set(ctx, settings)))
+
+
+@repository.command('get')
+@click.argument('settings', type=SettingsGet(), nargs=-1)
+@click.pass_context
+def repository_get(ctx, settings):
+ return check_result("repository_get", run(lambda: _get(ctx, settings)))
+
+
+@repository.command('clear')
+@click.argument('settings', type=SettingsClear(), nargs=-1)
+@click.pass_context
+def repository_clear(ctx, settings):
+ return check_result("repository_clear", run(lambda: _clear(ctx, settings)))
+
+
+## data_set: type, properties
+
+
+@cli.group()
+@click.option('-g', '--is_global', default=False, is_flag=True, help='Set/get global or local.')
+@click.option('-p', '--is_data_set_property', default=False, is_flag=True, help='Configure data set property.')
+@click.pass_context
+def data_set(ctx, is_global, is_data_set_property):
+ """ Get/set settings related to the data set.
+ """
+ ctx.obj['is_global'] = is_global
+ ctx.obj['is_data_set_property'] = is_data_set_property
+ ctx.obj['data_mgmt'] = shared_data_mgmt(ctx.obj)
+ ctx.obj['resolver'] = ctx.obj['data_mgmt'].settings_resolver.data_set
+
+
+@data_set.command('set')
+@click.argument('settings', type=SettingsSet(), nargs=-1)
+@click.pass_context
+def data_set_set(ctx, settings):
+ return check_result("data_set_set", run(lambda: _set(ctx, settings)))
+
+
+@data_set.command('get')
+@click.argument('settings', type=SettingsGet(), nargs=-1)
+@click.pass_context
+def data_set_get(ctx, settings):
+ return check_result("data_set_get", run(lambda: _get(ctx, settings)))
+
+
+@data_set.command('clear')
+@click.argument('settings', type=SettingsClear(), nargs=-1)
+@click.pass_context
+def data_set_clear(ctx, settings):
+ return check_result("data_set_clear", run(lambda: _clear(ctx, settings)))
+
+
+## object: object_id
+
+
+@cli.group()
+@click.option('-g', '--is_global', default=False, is_flag=True, help='Set/get global or local.')
+@click.pass_context
+def object(ctx, is_global):
+ """ Get/set settings related to the object.
+ """
+ ctx.obj['is_global'] = is_global
+ ctx.obj['data_mgmt'] = shared_data_mgmt(ctx.obj)
+ ctx.obj['resolver'] = ctx.obj['data_mgmt'].settings_resolver.object
+
+
+@object.command('set')
+@click.argument('settings', type=SettingsSet(), nargs=-1)
+@click.pass_context
+def object_set(ctx, settings):
+ return check_result("object_set", run(lambda: _set(ctx, settings)))
+
+
+@object.command('get')
+@click.argument('settings', type=SettingsGet(), nargs=-1)
+@click.pass_context
+def object_get(ctx, settings):
+ return check_result("object_get", run(lambda: _get(ctx, settings)))
+
+
+@object.command('clear')
+@click.argument('settings', type=SettingsClear(), nargs=-1)
+@click.pass_context
+def object_clear(ctx, settings):
+ return check_result("object_clear", run(lambda: _clear(ctx, settings)))
+## collection: collection_id
+
+
+@cli.group()
+@click.option('-g', '--is_global', default=False, is_flag=True, help='Set/get global or local.')
+@click.pass_context
+def collection(ctx, is_global):
+ """ Get/set settings related to the collection.
+ """
+ ctx.obj['is_global'] = is_global
+ ctx.obj['data_mgmt'] = shared_data_mgmt(ctx.obj)
+ ctx.obj['resolver'] = ctx.obj['data_mgmt'].settings_resolver.collection
+
+
+@collection.command('set')
+@click.argument('settings', type=SettingsSet(), nargs=-1)
+@click.pass_context
+def collection_set(ctx, settings):
+ return check_result("collection_set", run(lambda: _set(ctx, settings)))
+
+
+@collection.command('get')
+@click.argument('settings', type=SettingsGet(), nargs=-1)
+@click.pass_context
+def collection_get(ctx, settings):
+ return check_result("collection_get", run(lambda: _get(ctx, settings)))
+
+
+@collection.command('clear')
+@click.argument('settings', type=SettingsClear(), nargs=-1)
+@click.pass_context
+def collection_clear(ctx, settings):
+ return check_result("collection_clear", run(lambda: _clear(ctx, settings)))
+
+
+## config: fileservice_url, git_annex_hash_as_checksum, hostname, openbis_url, user, verify_certificates
+
+
+@cli.group()
+@click.option('-g', '--is_global', default=False, is_flag=True, help='Set/get global or local.')
+@click.pass_context
+def config(ctx, is_global):
+ """ Get/set configurations.
+ """
+ ctx.obj['is_global'] = is_global
+ ctx.obj['data_mgmt'] = shared_data_mgmt(ctx.obj)
+ ctx.obj['resolver'] = ctx.obj['data_mgmt'].settings_resolver.config
+
+
+@config.command('set')
+@click.argument('settings', type=SettingsSet(), nargs=-1)
+@click.pass_context
+def config_set(ctx, settings):
+ return check_result("config_set", run(lambda: _set(ctx, settings)))
+
+
+@config.command('get')
+@click.argument('settings', type=SettingsGet(), nargs=-1)
+@click.pass_context
+def config_get(ctx, settings):
+ return check_result("config_get", run(lambda: _get(ctx, settings)))
+
+
+@config.command('clear')
+@click.argument('settings', type=SettingsClear(), nargs=-1)
+@click.pass_context
+def config_clear(ctx, settings):
+ return check_result("config_clear", run(lambda: _clear(ctx, settings)))
+
+
+# repository commands: status, sync, commit, init, addref, removeref, init_analysis
+
+## commit
+
+_commit_params = [
+ click.option('-m', '--msg', prompt=True, help='A message explaining what was done.'),
+ click.option('-a', '--auto_add', default=True, is_flag=True, help='Automatically add all untracked files.'),
+ click.option('-i', '--ignore_missing_parent', default=True, is_flag=True, help='If parent data set is missing, ignore it.'),
+ click.argument('repository', type=click.Path(exists=True, file_okay=False), required=False),
+]
+
+def _repository_commit(ctx, msg, auto_add, ignore_missing_parent):
+ data_mgmt = shared_data_mgmt(ctx.obj)
+ return check_result("commit", run(lambda: data_mgmt.commit(msg, auto_add, ignore_missing_parent)))
+
+@repository.command("commit")
+@click.pass_context
+@add_params(_commit_params)
+def repository_commit(ctx, msg, auto_add, ignore_missing_parent, repository):
+ """Commit the repository to git and inform openBIS.
+ """
+ if repository is None:
+ return _repository_commit(ctx, msg, auto_add, ignore_missing_parent)
+ with cd(repository):
+ return _repository_commit(ctx, msg, auto_add, ignore_missing_parent)
+
@cli.command()
@click.pass_context
-@click.option('-oi', '--object_id', help='Set the id of the owning sample.')
-@click.option('-ci', '--collection_id', help='Set the id of the owning experiment.')
-@click.argument('folder', type=click.Path(exists=False, file_okay=False))
-@click.argument('description', default="")
-def init(ctx, object_id, collection_id, folder, description):
- """Initialize the folder as a data folder (alias for init_data)."""
- return init_data_impl(ctx, object_id, collection_id, folder, description)
+@add_params(_commit_params)
+def commit(ctx, msg, auto_add, ignore_missing_parent, repository):
+ """Commit the repository to git and inform openBIS.
+ """
+ ctx.invoke(repository_commit, msg=msg, auto_add=auto_add, ignore_missing_parent=ignore_missing_parent, repository=repository)
+
+## init
+_init_params = [
+ click.option('-oi', '--object_id', help='Set the id of the owning sample.'),
+ click.option('-ci', '--collection_id', help='Set the id of the owning experiment.'),
+ click.argument('repository', type=click.Path(exists=False, file_okay=False), required=False),
+ click.argument('description', default=""),
+]
+
+@repository.command("init")
+@click.pass_context
+@add_params(_init_params)
+def repository_init(ctx, object_id, collection_id, repository, description):
+ """Initialize the folder as a data repository."""
+ return init_data_impl(ctx, object_id, collection_id, repository, description)
@cli.command()
@click.pass_context
-@click.option('-oi', '--object_id', help='Set the id of the owning sample.')
-@click.option('-ci', '--collection_id', help='Set the id of the owning experiment.')
-@click.argument('folder', type=click.Path(exists=False, file_okay=False))
-@click.argument('description', default="")
-def init_data(ctx, object_id, collection_id, folder, description):
- """Initialize the folder as a data folder."""
- return init_data_impl(ctx, object_id, collection_id, folder, description)
+@add_params(_init_params)
+def init(ctx, object_id, collection_id, repository, description):
+ """Initialize the folder as a data repository."""
+ ctx.invoke(repository_init, object_id=object_id, collection_id=collection_id, repository=repository, description=description)
+## init analysis
+
+_init_analysis_params = [
+ click.option('-p', '--parent', type=click.Path(exists=False, file_okay=False)),
+]
+_init_analysis_params += _init_params
+
+@repository.command("init_analysis")
+@click.pass_context
+@add_params(_init_analysis_params)
+def repository_init_analysis(ctx, parent, object_id, collection_id, repository, description):
+ """Initialize the folder as an analysis folder."""
+ return init_analysis_impl(ctx, parent, object_id, collection_id, repository, description)
@cli.command()
@click.pass_context
-@click.option('-p', '--parent', type=click.Path(exists=False, file_okay=False))
-@click.option('-oi', '--object_id', help='Set the id of the owning sample.')
-@click.option('-ci', '--collection_id', help='Set the id of the owning experiment.')
-@click.argument('folder', type=click.Path(exists=False, file_okay=False))
-@click.argument('description', default="")
-def init_analysis(ctx, parent, object_id, collection_id, folder, description):
+@add_params(_init_analysis_params)
+def init_analysis(ctx, parent, object_id, collection_id, repository, description):
"""Initialize the folder as an analysis folder."""
- return init_analysis_impl(ctx, parent, object_id, collection_id, folder, description)
+ ctx.invoke(repository_init_analysis, parent=parent, object_id=object_id, collection_id=collection_id, repository=repository, description=description)
+
+## status
+
+_status_params = [
+ click.argument('repository', type=click.Path(exists=True, file_okay=False), required=False),
+]
+def _repository_status(ctx):
+ data_mgmt = shared_data_mgmt(ctx.obj)
+ result = run(data_mgmt.status)
+ click.echo(result.output)
+
+@repository.command("status")
+@click.pass_context
+@add_params(_status_params)
+def repository_status(ctx, repository):
+ """Show the state of the obis repository.
+ """
+ if repository is None:
+ return _repository_status(ctx)
+ with cd(repository):
+ return _repository_status(ctx)
@cli.command()
@click.pass_context
-@click.argument('file')
-def get(ctx, f):
- """Get one or more files from a clone of this repository.
+@add_params(_status_params)
+def status(ctx, repository):
+ """Show the state of the obis repository.
"""
- click_echo("get {}".format(f))
+ ctx.invoke(repository_status, repository=repository)
+
+## sync
+
+_sync_params = [
+ click.option('-i', '--ignore_missing_parent', default=True, is_flag=True, help='If parent data set is missing, ignore it.'),
+ click.argument('repository', type=click.Path(exists=True, file_okay=False), required=False),
+]
+def _repository_sync(ctx, ignore_missing_parent):
+ data_mgmt = shared_data_mgmt(ctx.obj)
+ return check_result("sync", run(lambda: data_mgmt.sync(ignore_missing_parent)))
+
+@repository.command("sync")
+@click.pass_context
+@add_params(_sync_params)
+def repository_sync(ctx, ignore_missing_parent, repository):
+ """Sync the repository with openBIS.
+ """
+ if repository is None:
+ return _repository_sync(ctx, ignore_missing_parent)
+ with cd(repository):
+ return _repository_sync(ctx, ignore_missing_parent)
@cli.command()
@click.pass_context
-def status(ctx):
+@add_params(_sync_params)
+def sync(ctx, ignore_missing_parent, repository):
"""Sync the repository with openBIS.
"""
+ ctx.invoke(repository_sync, ignore_missing_parent=ignore_missing_parent, repository=repository)
+
+## addref
+
+_addref_params = [
+ click.argument('repository', type=click.Path(exists=True, file_okay=False), required=False),
+]
+
+def _repository_addref(ctx):
data_mgmt = shared_data_mgmt(ctx.obj)
- result = data_mgmt.status()
- click.echo(result.output)
+ return check_result("addref", run(data_mgmt.addref))
+@repository.command("addref")
+@click.pass_context
+@add_params(_addref_params)
+def repository_addref(ctx, repository):
+ """Add the given repository as a reference to openBIS.
+ """
+ if repository is None:
+ return _repository_addref(ctx)
+ with cd(repository):
+ return _repository_addref(ctx)
@cli.command()
@click.pass_context
-def sync(ctx):
- """Sync the repository with openBIS.
+@add_params(_addref_params)
+def addref(ctx, repository):
+ """Add the given repository as a reference to openBIS.
"""
+ ctx.invoke(repository_addref, repository=repository)
+
+# removeref
+
+_removeref_params = [
+ click.argument('repository', type=click.Path(exists=True, file_okay=False), required=False),
+]
+
+def _repository_removeref(ctx):
data_mgmt = shared_data_mgmt(ctx.obj)
- return check_result("sync", data_mgmt.sync())
+ return check_result("addref", run(data_mgmt.removeref))
+
+@repository.command("removeref")
+@click.pass_context
+@add_params(_removeref_params)
+def repository_removeref(ctx, repository):
+ """Remove the reference to the given repository from openBIS.
+ """
+ if repository is None:
+ return _repository_removeref(ctx)
+ with cd(repository):
+ return _repository_removeref(ctx)
+
+@cli.command()
+@click.pass_context
+@add_params(_removeref_params)
+def removeref(ctx, repository):
+ """Remove the reference to the given repository from openBIS.
+ """
+ ctx.invoke(repository_removeref, repository=repository)
+
+
+# data set commands: download / clone
+
+## download
+
+_download_params = [
+ click.option('-c', '--content_copy_index', type=int, default=None, help='Index of the content copy to download from.'),
+ click.option('-f', '--file', help='File in the data set to download - downloading all if not given.'),
+ click.argument('data_set_id'),
+]
+
+@data_set.command("download")
+@add_params(_download_params)
+@click.pass_context
+def data_set_download(ctx, content_copy_index, file, data_set_id):
+ """ Download files of a linked data set.
+ """
+ data_mgmt = shared_data_mgmt(ctx.obj)
+ return check_result("download", run(lambda: data_mgmt.download(data_set_id, content_copy_index, file)))
+
+@cli.command()
+@add_params(_download_params)
+@click.pass_context
+def download(ctx, content_copy_index, file, data_set_id):
+ """ Download files of a linked data set.
+ """
+ ctx.invoke(download, content_copy_index=content_copy_index, file=file, data_set_id=data_set_id)
+
+## clone
+
+_clone_params = [
+ click.option('-u', '--ssh_user', default=None, help='User to connect to remote systems via ssh'),
+ click.option('-c', '--content_copy_index', type=int, default=None, help='Index of the content copy to clone from in case there are multiple copies'),
+ click.argument('data_set_id'),
+]
+
+@data_set.command("clone")
+@click.pass_context
+@add_params(_clone_params)
+def data_set_clone(ctx, ssh_user, content_copy_index, data_set_id):
+ """Clone the repository found in the given data set id.
+ """
+ data_mgmt = shared_data_mgmt(ctx.obj)
+ return check_result("clone", run(lambda: data_mgmt.clone(data_set_id, ssh_user, content_copy_index)))
+
+@cli.command()
+@click.pass_context
+@add_params(_clone_params)
+def clone(ctx, ssh_user, content_copy_index, data_set_id):
+ """Clone the repository found in the given data set id.
+ """
+ ctx.invoke(data_set_clone, ssh_user=ssh_user, content_copy_index=content_copy_index, data_set_id=data_set_id)
def main():
diff --git a/src/python/PyBis/pybis/data_set.py b/src/python/PyBis/pybis/data_set.py
index 709c1ca79fc920dc3661111f8be707ff664d5619..f61dc9296fc151a0469424e9b15a38eae5be370a 100644
--- a/src/python/PyBis/pybis/data_set.py
+++ b/src/python/PyBis/pybis/data_set.py
@@ -38,6 +38,8 @@ class GitDataSetCreation(object):
:param contents: A list of dicts that describe the contents:
{'fileLength': [file length],
'crc32': [crc32 checksum],
+ 'checksum': [checksum other than crc32],
+ 'checksumType': [checksum type if fiels checksum is used],
'directory': [is path a directory?]
'path': [the relative path string]}
@@ -172,6 +174,8 @@ class GitDataSetCreation(object):
result = {}
transfer_to_file_creation(content, result, 'fileLength')
transfer_to_file_creation(content, result, 'crc32', 'checksumCRC32')
+ transfer_to_file_creation(content, result, 'checksum', 'checksum')
+ transfer_to_file_creation(content, result, 'checksumType', 'checksumType')
transfer_to_file_creation(content, result, 'directory')
transfer_to_file_creation(content, result, 'path')
return result
@@ -179,30 +183,34 @@ class GitDataSetCreation(object):
class GitDataSetUpdate(object):
- def __init__(self, openbis, path, commit_id, repository_id, edms_id, data_set_id):
+ def __init__(self, openbis, data_set_id):
"""Initialize the command object with the necessary parameters.
:param openbis: The openBIS API object.
- :param path: The path to the git repository
- :param commit_id: The git commit id
- :param repository_id: The git repository id - same for copies
- :param edms_id: If of the external data managment system
:param data_set_id: Id of the data set to be updated
"""
self.openbis = openbis
- self.path = path
- self.commit_id = commit_id
- self.repository_id = repository_id
- self.edms_id =edms_id
self.data_set_id = data_set_id
-
- def new_content_copy(self):
+ def new_content_copy(self, path, commit_id, repository_id, edms_id):
""" Create a data set update for adding a content copy.
:return: A DataSetUpdate object
"""
- data_set_update = self.get_data_set_update()
+ self.path = path
+ self.commit_id = commit_id
+ self.repository_id = repository_id
+ self.edms_id =edms_id
+
+ content_copy_actions = self.get_actions_add_content_copy()
+ data_set_update = self.get_data_set_update(content_copy_actions)
self.send_request(data_set_update)
+ def delete_content_copy(self, content_copy):
+ """ Deletes the given content_copy from openBIS.
+ :param content_copy: Content copy to be deleted.
+ """
+ content_copy_actions = self.get_actions_remove_content_copy(content_copy)
+ data_set_update = self.get_data_set_update(content_copy_actions)
+ self.send_request(data_set_update)
def send_request(self, data_set_update):
request = {
@@ -215,11 +223,11 @@ class GitDataSetUpdate(object):
self.openbis._post_request(self.openbis.as_v3, request)
- def get_data_set_update(self):
+ def get_data_set_update(self, content_copy_actions=[]):
return {
"@type": "as.dto.dataset.update.DataSetUpdate",
"dataSetId": self.get_data_set_id(),
- "linkedData": self.get_linked_data()
+ "linkedData": self.get_linked_data(content_copy_actions)
}
@@ -230,7 +238,7 @@ class GitDataSetUpdate(object):
}
- def get_linked_data(self):
+ def get_linked_data(self, actions):
return {
"@type": "as.dto.common.update.FieldUpdateValue",
"isModified": True,
@@ -238,15 +246,24 @@ class GitDataSetUpdate(object):
"@type": "as.dto.dataset.update.LinkedDataUpdate",
"contentCopies": {
"@type": "as.dto.dataset.update.ContentCopyListUpdateValue",
- "actions": [ {
- "@type": "as.dto.common.update.ListUpdateActionAdd",
- "items": [ self.get_content_copy_creation() ]
- } ]
+ "actions": actions,
}
}
}
+ def get_actions_add_content_copy(self):
+ return [{
+ "@type": "as.dto.common.update.ListUpdateActionAdd",
+ "items": [ self.get_content_copy_creation() ]
+ }]
+
+ def get_actions_remove_content_copy(self, content_copy):
+ return [{
+ "@type": "as.dto.common.update.ListUpdateActionRemove",
+ "items": [ content_copy["id"] ]
+ }]
+
def get_content_copy_creation(self):
return {
"@type": "as.dto.dataset.create.ContentCopyCreation",
@@ -258,3 +275,64 @@ class GitDataSetUpdate(object):
"gitCommitHash": self.commit_id,
"gitRepositoryId" : self.repository_id,
}
+
+
+class GitDataSetFileSearch(object):
+
+ def __init__(self, openbis, data_set_id, dss_code=None):
+ """Initialize the command object with the necessary parameters.
+ :param openbis: The openBIS API object.
+ :param data_set_id: Id of the data set to be updated
+ :param dss_code: Code for the DSS -- defaults to the first dss if none is supplied.
+ """
+ self.openbis = openbis
+ self.data_set_id = data_set_id
+ self.dss_code = dss_code
+
+ def search_files(self):
+ request = {
+ "method": "searchFiles",
+ "params": [
+ self.openbis.token,
+ self.get_data_set_file_search_criteria(),
+ self.get_data_set_file_fetch_options(),
+ ]
+ }
+ server_url = self.data_store_url()
+ return self.openbis._post_request_full_url(server_url, request)
+
+ def get_data_set_file_search_criteria(self):
+ return {
+ "@type": "dss.dto.datasetfile.search.DataSetFileSearchCriteria",
+ "operator": "AND",
+ "criteria": [
+ {
+ "@type": "as.dto.dataset.search.DataSetSearchCriteria",
+ "relation": "DATASET",
+ "operator": "OR",
+ "criteria": [
+ {
+ "fieldName": "code",
+ "fieldType": "ATTRIBUTE",
+ "fieldValue": {
+ "value": self.data_set_id,
+ "@type": "as.dto.common.search.StringEqualToValue"
+ },
+ "@type": "as.dto.common.search.CodeSearchCriteria"
+ }
+ ],
+ }
+ ],
+ }
+
+ def get_data_set_file_fetch_options(self):
+ return {
+ "@type": "dss.dto.datasetfile.fetchoptions.DataSetFileFetchOptions",
+ }
+
+ def data_store_url(self):
+ data_stores = self.openbis.get_datastores()
+ if self.dss_code is None:
+ self.dss_code = self.openbis.get_datastores()['code'][0]
+ data_store = data_stores[data_stores['code'] == self.dss_code]
+ return "{}/datastore_server/rmi-data-store-server-v3.json".format(data_store['hostUrl'][0])
diff --git a/src/python/PyBis/pybis/pybis.py b/src/python/PyBis/pybis/pybis.py
index 979071b985dd6e18135ce5eef7705518d7c0c7e9..a78deb31a389256b4eade66b2016e07e39cc49c4 100644
--- a/src/python/PyBis/pybis/pybis.py
+++ b/src/python/PyBis/pybis/pybis.py
@@ -2497,7 +2497,18 @@ class Openbis:
"param edms_id: Id of the external data managment system of the content copy
"param data_set_id: Id of the data set to which the new content copy belongs
"""
- return pbds.GitDataSetUpdate(self, path, commit_id, repository_id, edms_id, data_set_id).new_content_copy()
+ return pbds.GitDataSetUpdate(self, data_set_id).new_content_copy(path, commit_id, repository_id, edms_id)
+
+ def search_files(self, data_set_id, dss_code=None):
+ return pbds.GitDataSetFileSearch(self, data_set_id).search_files()
+
+ def delete_content_copy(self, data_set_id, content_copy):
+ """
+ Deletes a content copy from a data set.
+ :param data_set_id: Id of the data set containing the content copy
+ :param content_copy: The content copy to be deleted
+ """
+ return pbds.GitDataSetUpdate(self, data_set_id).delete_content_copy(content_copy)
@staticmethod
def sample_to_sample_id(sample):
@@ -2669,7 +2680,6 @@ class PhysicalData():
return html
def __repr__(self):
-
headers = ['attribute', 'value']
lines = []
for attr in self.attrs:
diff --git a/src/vagrant/obis/Vagrantfile b/src/vagrant/obis/Vagrantfile
index 4c5963f595af780856280bf6c3698c7dfa09f32e..539638b7ce70ee6dc997306dac553a88782a8143 100644
--- a/src/vagrant/obis/Vagrantfile
+++ b/src/vagrant/obis/Vagrantfile
@@ -1,8 +1,6 @@
# -*- mode: ruby -*-
# vi: set ft=ruby :
-Vagrant::DEFAULT_SERVER_URL.replace('https://vagrantcloud.com')
-
# All Vagrant configuration is done below. The "2" in Vagrant.configure
# configures the configuration version (we support older styles for
# backwards compatibility). Please don't change it unless you know what
diff --git a/src/vagrant/obis/initialize/install_openbis.sh b/src/vagrant/obis/initialize/install_openbis.sh
index 7df0a7802ea5d7b0d018686280113043bee27605..ab07be5bc759c65974708c194e805fd69c4e7d87 100755
--- a/src/vagrant/obis/initialize/install_openbis.sh
+++ b/src/vagrant/obis/initialize/install_openbis.sh
@@ -12,7 +12,12 @@ if [ ! -L /openbis_installed ]; then
sudo -u openbis cd ~openbis/
sudo su openbis -c "export ADMIN_PASSWORD=admin && export ETLSERVER_PASSWORD=etlserver && $ob_dir/run-console.sh"
+ sudo su openbis -c "sed -i '/host-address = /c\host-address = https://obisserver' /home/openbis/servers/datastore_server/etc/service.properties"
+
+ sudo su openbis -c "~/servers/openBIS-server/jetty/bin/passwd.sh add obis -p obis"
+
sudo touch /openbis_installed
+ sudo chmod 777 /openbis_installed
popd $@ > /dev/null