diff --git a/src/python/PyBis/pybis/data_set.py b/src/python/PyBis/pybis/data_set.py
index c9d3a89bab1e779176620920352773a5c8e59c2f..72ffb3772cf7c42b3fd79265172d16a818e25743 100644
--- a/src/python/PyBis/pybis/data_set.py
+++ b/src/python/PyBis/pybis/data_set.py
@@ -12,9 +12,16 @@ Copyright (c) 2017 Chandrasekhar Ramakrishnan. All rights reserved.
"""
+def transfer_to_file_creation(content, file_creation, key, file_creation_key=None):
+ if file_creation_key is None:
+ file_creation_key = key
+ if content.get(key):
+ file_creation[file_creation_key] = content[key]
+
+
class GitDataSetCreation(object):
def __init__(self, openbis, data_set_type, path, commit_id, dms, sample=None, properties={},
- dss_code=None, parents=None, data_set_code=None):
+ dss_code=None, parents=None, data_set_code=None, contents=[]):
"""Initialize the command object with the necessary parameters.
:param openbis: The openBIS API object.
:param data_set_type: The type of the data set
@@ -27,6 +34,12 @@ class GitDataSetCreation(object):
:param dss_code: Code for the DSS -- defaults to the first dss if none is supplied.
:param parents: Parents for the data set.
:param data_set_code: A data set code -- used if provided, otherwise generated on the server
+ :param contents: A list of dicts that describe the contents:
+ {'fileLength': [file length],
+ 'crc32': [crc32 checksum],
+ 'directory': [is path a directory?]
+ 'path': [the relative path string]}
+
"""
self.openbis = openbis
self.data_set_type = data_set_type
@@ -38,12 +51,62 @@ class GitDataSetCreation(object):
self.dss_code = dss_code
self.parents = parents
self.data_set_code = data_set_code
+ self.contents = contents
def new_git_data_set(self):
""" Create a link data set.
:return: A DataSet object
"""
+ data_set_creation = self.data_set_metadata_creation()
+ file_metadata = self.data_set_file_metadata()
+ if not file_metadata:
+ return self.create_pure_metadata_data_set(data_set_creation)
+ return self.create_mixed_data_set(data_set_creation, file_metadata)
+
+ def create_pure_metadata_data_set(self, data_set_creation):
+ # register the files in openBIS
+ request = {
+ "method": "createDataSets",
+ "params": [
+ self.openbis.token,
+ [data_set_creation]
+ ]
+ }
+
+ # noinspection PyProtectedMember
+ resp = self.openbis._post_request(self.openbis.as_v3, request)
+ return self.openbis.get_dataset(resp[0]['permId'])
+
+ def create_mixed_data_set(self, metadata_creation, file_metadata):
+ data_set_creation = {
+ "fileMetadata": file_metadata,
+ "metadataCreation": metadata_creation
+ # The DSS objects do not use the type annotation
+ # "@type": "dss.dto.dataset.create.FullDataSetCreation"
+ }
+
+ # register the files in openBIS
+ request = {
+ "method": "createDataSets",
+ "params": [
+ self.openbis.token,
+ [data_set_creation]
+ ]
+ }
+
+ server_url = self.data_store_url(metadata_creation['dataStoreId']['permId'])
+ # noinspection PyProtectedMember
+ resp = self.openbis._post_request_full_url(server_url, request)
+ return self.openbis.get_dataset(resp[0]['permId'])
+
+ def data_store_url(self, dss_code):
+ data_stores = self.openbis.get_datastores()
+ data_store = data_stores[data_stores['code'] == dss_code]
+ return "{}/datastore_server/rmi-data-store-server-v3.json".format(data_store['hostUrl'][0])
+
+ def data_set_metadata_creation(self):
+ """Create the respresentation of the data set metadata."""
dss_code = self.dss_code
if dss_code is None:
dss_code = self.openbis.get_datastores()['code'][0]
@@ -52,14 +115,12 @@ class GitDataSetCreation(object):
# if a sample object was given, take its identifier
sample_id = self.openbis.sample_to_sample_id(self.sample)
dms_id = self.openbis.external_data_managment_system_to_dms_id(self.dms)
-
parents = self.parents
parentIds = []
if parents is not None:
if not isinstance(parents, list):
parents = [parents]
parentIds = [self.openbis.data_set_to_data_set_id(parent) for parent in parents]
-
data_set_creation = {
"linkedData": {
"@type": "as.dto.dataset.create.LinkedDataCreation",
@@ -92,15 +153,18 @@ class GitDataSetCreation(object):
else:
data_set_creation["autoGeneratedCode"] = True
- # register the files in openBIS
- request = {
- "method": "createDataSets",
- "params": [
- self.openbis.token,
- [data_set_creation]
- ]
- }
+ return data_set_creation
- # noinspection PyProtectedMember
- resp = self.openbis._post_request(self.openbis.as_v3, request)
- return self.openbis.get_dataset(resp[0]['permId'])
+ def data_set_file_metadata(self):
+ """Create a representation of the file metadata"""
+ return [self.as_file_metadata(c) for c in self.contents]
+
+ def as_file_metadata(self, content):
+ # The DSS objects do not use type
+ # result = {"@type": "dss.dto.datasetfile.DataSetFileCreation"}
+ result = {}
+ transfer_to_file_creation(content, result, 'fileLength')
+ transfer_to_file_creation(content, result, 'crc32', 'checksumCRC32')
+ transfer_to_file_creation(content, result, 'directory')
+ transfer_to_file_creation(content, result, 'path')
+ return result
diff --git a/src/python/PyBis/pybis/pybis.py b/src/python/PyBis/pybis/pybis.py
index 632a9c596dec42a04942eb5c4ca2095615bcf859..4b44fa9fadfcbd4871fbe540860a747b3fb34d7e 100644
--- a/src/python/PyBis/pybis/pybis.py
+++ b/src/python/PyBis/pybis/pybis.py
@@ -658,6 +658,12 @@ class Openbis:
os.remove(token_path)
def _post_request(self, resource, request):
+ """ internal method, used to handle all post requests and serializing / deserializing
+ data
+ """
+ return self._post_request_full_url(self.url + resource, request)
+
+ def _post_request_full_url(self, full_url, request):
""" internal method, used to handle all post requests and serializing / deserializing
data
"""
@@ -668,7 +674,7 @@ class Openbis:
if request["params"][0] is None:
raise ValueError("Your session expired, please log in again")
resp = requests.post(
- self.url + resource,
+ full_url,
json.dumps(request),
verify=self.verify_certificates
)
@@ -1753,7 +1759,7 @@ class Openbis:
return resp
def new_git_data_set(self, data_set_type, path, commit_id, dms, sample=None, properties={},
- dss_code=None, parents=None, data_set_code=None):
+ dss_code=None, parents=None, data_set_code=None, contents=[]):
""" Create a link data set.
:param data_set_type: The type of the data set
:param data_set_type: The type of the data set
@@ -1765,10 +1771,15 @@ class Openbis:
:param properties: Properties for the data set.
:param parents: Parents for the data set.
:param data_set_code: A data set code -- used if provided, otherwise generated on the server
+ :param contents: A list of dicts that describe the contents:
+ {'file_length': [file length],
+ 'crc32': [crc32 checksum],
+ 'directory': [is path a directory?]
+ 'path': [the relative path string]}
:return: A DataSet object
"""
return pbds.GitDataSetCreation(self, data_set_type, path, commit_id, dms, sample,
- properties, dss_code, parents, data_set_code).new_git_data_set()
+ properties, dss_code, parents, data_set_code, contents).new_git_data_set()
@staticmethod
def sample_to_sample_id(sample):
diff --git a/src/python/PyBis/tests/test_openbis.py b/src/python/PyBis/tests/test_openbis.py
index 71d116750800718f5ae16c6fd80fe20279924e56..af49b96e14b7f0fe30317a4a0d7a382e1ee6ee8c 100644
--- a/src/python/PyBis/tests/test_openbis.py
+++ b/src/python/PyBis/tests/test_openbis.py
@@ -207,6 +207,21 @@ def test_new_git_data_set_with_property(openbis_instance):
openbis_instance.delete_entity('DataSet', result.code, 'Testing.', capitalize=False)
+def test_new_git_data_set_with_contents(openbis_instance):
+ dms_code, dms = create_external_data_management_system(openbis_instance)
+ data_set_code = openbis_instance.create_perm_id()
+ contents = [{'crc32': 1234, 'directory': False, 'fileLength': 4321, 'path': 'path/to/the/file.txt'},
+ {'directory': True, 'path': 'path/to/empty/directory'}]
+ result = openbis_instance.new_git_data_set("GIT_REPO", "./", '12345', dms_code, "/DEFAULT/DEFAULT",
+ data_set_code=data_set_code,
+ properties={"DESCRIPTION": 'This is a description'},
+ contents=contents)
+ assert result is not None
+ # TODO Turn this on again once the bug in registration is fixed.
+ # assert result.code == data_set_code
+ openbis_instance.delete_entity('DataSet', result.code, 'Testing.', capitalize=False)
+
+
def test_create_perm_id(openbis_instance):
perm_id = openbis_instance.create_perm_id()
assert perm_id is not None