diff --git a/pybis/src/python/CHANGELOG.md b/pybis/src/python/CHANGELOG.md
index abfaa5ab7964d5cdf92ebe0ccb19f3366475e662..dcf59acc3a2fea71b53da40664362234f51dbe40 100644
--- a/pybis/src/python/CHANGELOG.md
+++ b/pybis/src/python/CHANGELOG.md
@@ -1,3 +1,7 @@
+## Changes with pybis-1.10.4
+
+* better error messages when downloading files from datastore server
+
## Changes with pybis-1.10.3
* print warning message when downloaded file-size does not match with promised file-size. Do not die.
diff --git a/pybis/src/python/pybis/__init__.py b/pybis/src/python/pybis/__init__.py
index 2d588f6a74cf8efc6015fd6d00e51ce3843c9b8f..57e35f83d5ad77c00560ccaef1fda1f3dd9ef304 100644
--- a/pybis/src/python/pybis/__init__.py
+++ b/pybis/src/python/pybis/__init__.py
@@ -1,7 +1,7 @@
name = 'pybis'
__author__ = 'Swen Vermeul'
__email__ = 'swen@ethz.ch'
-__version__ = '1.10.3'
+__version__ = '1.10.4'
from . import pybis
from .pybis import Openbis
diff --git a/pybis/src/python/pybis/dataset.py b/pybis/src/python/pybis/dataset.py
index fa99d564e6a6b736ea90f7099abab1c95b0c62ad..79962ccc7f3a7044ae4b225a5f5f0267b4957cc3 100644
--- a/pybis/src/python/pybis/dataset.py
+++ b/pybis/src/python/pybis/dataset.py
@@ -18,6 +18,7 @@ import time
# needed for Data upload
PYBIS_PLUGIN = "dataset-uploader-api"
dataset_definitions = openbis_definitions('dataSet')
+dss_endpoint = '/datastore_server/rmi-data-store-server-v3.json'
class DataSet(
@@ -202,6 +203,33 @@ class DataSet(
set_props = set_properties
+ def get_dataset_files(self, **properties):
+
+
+ search_criteria = get_search_type_for_entity('datasetFiles')
+ search_criteria['criteria'] = sub_criteria
+ search_criteria['operator'] = 'AND'
+
+
+ request = {
+ "method": "searchFiles",
+ "params": [
+ self.token,
+ search_criteria,
+ fetchopts,
+ ],
+ }
+ resp = self._post_request(datastore.url, dss_endpoint, request)
+
+ return self._dataset_list_for_response(
+ response=resp['objects'],
+ props=props,
+ start_with=start_with,
+ count=count,
+ totalCount=resp['totalCount'],
+ )
+
+
def download(self, files=None, destination=None, wait_until_finished=True, workers=10,
linked_dataset_fileservice_url=None, content_copy_index=0):
""" download the actual files and put them by default in the following folder:
@@ -247,6 +275,7 @@ class DataSet(
file_info = self.get_file_list(start_folder=filename)
file_size = file_info[0]['fileSize']
download_url = base_url + filename + '?sessionID=' + self.openbis.token
+ #print(download_url)
filename_dest = os.path.join(destination, self.permId, filename)
queue.put([download_url, filename, filename_dest, file_size, self.openbis.verify_certificates, 'wb'])
@@ -771,11 +800,18 @@ class DataSetDownloadQueue():
if r.ok == False:
raise ValueError("Could not download from {}: HTTP {}. Reason: {}".format(url, r.status_code, r.reason))
- with open(filename_dest, write_mode) as f:
- for chunk in r.iter_content(chunk_size=1024):
+ with open(filename_dest, write_mode) as fh:
+ for chunk in r.iter_content(chunk_size=1024*1024):
+ #size += len(chunk)
+ #print("WRITE ", datetime.now(), len(chunk))
if chunk: # filter out keep-alive new chunks
- f.write(chunk)
+ fh.write(chunk)
+ #print("DONE WRITE", datetime.now())
+ #print("DONE", datetime.now())
+
+ r.raise_for_status()
+ #print("{} bytes written".format(size))
actual_file_size = os.path.getsize(filename_dest)
if actual_file_size != int(file_size):
if self.collect_files_with_wrong_length:
@@ -785,6 +821,12 @@ class DataSetDownloadQueue():
"WARNING! File {} has the wrong length: Expected: {} Actual size: {}".format(
filename_dest, int(file_size), actual_file_size)
)
+ print (
+ "REASON: The connection has been silently dropped upstreams.",
+ "Please check the http timeout settings of the openBIS datastore server"
+ )
+ except Exception as err:
+ print("ERROR while writing file {}: {}".format(filename_dest, err))
finally:
self.download_queue.task_done()
diff --git a/pybis/src/python/pybis/experiment.py b/pybis/src/python/pybis/experiment.py
index b003643f29e094b800e96a61b045ecccfbba26e2..a1d9839187f7e758da0be870ac5495714000e675 100644
--- a/pybis/src/python/pybis/experiment.py
+++ b/pybis/src/python/pybis/experiment.py
@@ -35,6 +35,10 @@ class Experiment(
'save()'
] + super().__dir__()
+ @property
+ def props(self):
+ return self.__dict__['p']
+
@property
def type(self):
return self.__dict__['type']
diff --git a/pybis/src/python/pybis/project.py b/pybis/src/python/pybis/project.py
index 8860385b7931a9fa8012bde28c55db541d31d75c..ab893ff92cec84124f5b255fe065c26b685e1fc3 100644
--- a/pybis/src/python/pybis/project.py
+++ b/pybis/src/python/pybis/project.py
@@ -24,6 +24,10 @@ class Project(
'save()', 'delete()'
] + super().__dir__()
+ @property
+ def props(self):
+ return self.__dict__['p']
+
def get_samples(self, **kwargs):
return self.openbis.get_samples(project=self.permId, **kwargs)
get_objects = get_samples # Alias
diff --git a/pybis/src/python/setup.py b/pybis/src/python/setup.py
index a6507d023c035f02cbacc67a380f291980077d3b..38b7ca5d4be0edf31e457f9ee6469081bdc19143 100644
--- a/pybis/src/python/setup.py
+++ b/pybis/src/python/setup.py
@@ -11,7 +11,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
setup(
name='PyBIS',
- version= '1.10.3',
+ version= '1.10.4',
author='Swen Vermeul • ID SIS • ETH Zürich',
author_email='swen@ethz.ch',
description='openBIS connection and interaction, optimized for using with Jupyter',