From 05f6fd1f24a8adf5081aa012e4876e83c3994874 Mon Sep 17 00:00:00 2001
From: Swen Vermeul <swen@ethz.ch>
Date: Wed, 20 Jul 2016 15:58:50 +0200
Subject: [PATCH] complete working uploader
---
.../dataset-uploader-api.py | 112 ++++++++----------
1 file changed, 47 insertions(+), 65 deletions(-)
diff --git a/src/core-plugins/jupyter-api/1/dss/reporting-plugins/jupyter-uploader-api/dataset-uploader-api.py b/src/core-plugins/jupyter-api/1/dss/reporting-plugins/jupyter-uploader-api/dataset-uploader-api.py
index 8d5c42b09fb..d04cb14f689 100644
--- a/src/core-plugins/jupyter-api/1/dss/reporting-plugins/jupyter-uploader-api/dataset-uploader-api.py
+++ b/src/core-plugins/jupyter-api/1/dss/reporting-plugins/jupyter-uploader-api/dataset-uploader-api.py
@@ -35,7 +35,6 @@ import shutil
import errno
-
def getSampleByIdentifier(transaction, identifier):
space = identifier.split("/")[1]
code = identifier.split("/")[2]
@@ -58,10 +57,11 @@ def get_dataset_for_name(transaction, dataset_name):
search_service = transaction.getSearchService()
criteria = SearchCriteria()
criteria.addMatchClause(MatchClause.createPropertyMatch('NAME', dataset_name))
-
found = list(search_service.searchForDataSets(criteria))
if len(found) == 1:
- return found[0]
+ print("DataSetCode of found dataset = " + found[0].getDataSetCode())
+ return transaction.getDataSetForUpdate(found[0].getDataSetCode())
+ #return found[0]
else:
return None
@@ -110,7 +110,7 @@ def process(transaction, parameters, tableBuilder):
print('userSessionToken: ' + userSessionToken)
# get mandatory sample to connect the container to
- sampleIdentifier = parameters.get("sample").get("identifier")
+ sampleIdentifier = parameters.get("sample").get("identifier").get("identifier")
print('looking for sample with identifier: ' + sampleIdentifier)
if sampleIdentifier is None:
raise UserFailureException('mandatory parameter sample["identifier"] is missing')
@@ -133,29 +133,23 @@ def process(transaction, parameters, tableBuilder):
)
dataset_codes.append(dataset_code)
-# if parameters.get("result").get("fileNames") is not None:
-# dataset_code = register_dataset(
-# transaction,
-# "JUPYTER_RESULT",
-# sample,
-# 'container_permId',
-#
-# parameters.get("result").get("fileNames")
-# )
-# dataset_codes.append(dataset_code)
-
-# if parameters.get("containers") is not None:
-# for container in parameters.get("containers"):
-# container_code = register_container(
-# transaction,
-# container.get("dataSetType"),
-# sample,
-# container.get("properties"),
-# dataset_codes
-# )
-
-
- #
+
+ if parameters.get("containers") is not None:
+ print("...creating container...")
+ for container in parameters.get("containers"):
+ #print("transaction = " + str(transaction))
+ #print("dataSetType = " + container.get("dataSetType"))
+ #print("sample = " + str(sample))
+ #print("properties = " + str(container.get("properties")) )
+ #print("dataset_codes: " + str(dataset_codes))
+ container_code = register_container(
+ transaction,
+ container.get("dataSetType"),
+ sample,
+ container.get("properties"),
+ dataset_codes
+ )
+
# create the dataset
if everything_ok:
# Success message
@@ -178,16 +172,21 @@ def process(transaction, parameters, tableBuilder):
def register_container(transaction, dataset_type, sample, properties, contained_dataset_codes ):
+ container_name = properties.get("NAME")
+ print("check if the JUPYTER_CONTAINER already exists with name: "+ container_name)
# make sure container dataset doesn't exist yet
- container = get_dataset_for_name(transaction, properties.get("name"))
+ container = get_dataset_for_name(transaction, container_name)
if container is None:
- print("creating new JUPYTER_CONTAINER dataset...")
+ print("creating new JUPYTER_CONTAINER dataset... with name: " + container_name)
# Create new container (a dataset of type "JUPYTER_CONTAINER")
container = transaction.createNewDataSet(dataset_type)
container.setSample(sample)
container.setRegistrator(userId)
-
+ else:
+ print("JUPYTER_CONTAINER already exists: " + container_name)
+
+ print("setting properties...")
for key in properties.keySet():
propertyValue = unicode(properties[key])
print("container: setting "+key+"="+propertyValue)
@@ -203,19 +202,18 @@ def register_container(transaction, dataset_type, sample, properties, contained_
def register_dataset(transaction, dataset_type, sample, properties, ws_folder, file_names):
- """ creates a new dataset of type JUPYTER_RESULT.
- the parent dataset is the JUPYTER_CONTAINER we just created
+ """ creates a new dataset of a given type.
- the result files are copied from the session workspace
to a temp dir close to the DSS: prepareFilesForRegistration()
- from there, the files are moved to the DSS: transaction.moveFile()
- finally, the remaining files are deleted from the session workspace
"""
- print("dataset_type = " +dataset_type)
- print("creating " + dataset_type + " dataset...")
+ print("creating dataset of type: " + dataset_type)
dataset = transaction.createNewDataSet(dataset_type)
dataset.setSample(sample)
+ # setting any given properties
for key in properties.keySet():
propertyValue = unicode(properties[key]);
print("setting propertyValue: "+key + " = " + propertyValue)
@@ -223,39 +221,40 @@ def register_dataset(transaction, dataset_type, sample, properties, ws_folder, f
propertyValue = None;
dataset.setPropertyValue(key,propertyValue);
- print("JUPYTER_RESULT permId: " + dataset.getDataSetCode())
-
+ print("dataset created with permId: " + dataset.getDataSetCode())
+ print("workspace folder is: " + ws_folder)
# create temporary folder in incoming-dir ( openbis/servers/datastore_server/data/incoming )
threadProperties = getThreadProperties(transaction)
- print(threadProperties)
- dst_dir = os.path.join( threadProperties[u'incoming-dir'], str(time.time()) )
- print("incoming folder: " + dst_dir)
+ incoming_dir = os.path.join( threadProperties[u'incoming-dir'], str(time.time()) )
+ print("incoming folder is: " + incoming_dir)
dss_service = ServiceProvider.getDssServiceRpcGeneric().getService()
+ # copy all files from session workspace to (temporary) incoming directory.
for file_name in file_names:
- print("copying file: " + file_name)
- file_path = os.path.join(dst_dir, file_name)
- print("to: "+file_path)
+ ws_file_path = os.path.join(ws_folder, file_name)
+ print("copying file from session workspace: " + ws_file_path)
+ incoming_file_path = os.path.join(incoming_dir, file_name)
+ print("to incoming: "+incoming_file_path)
# ensure that all necessary folders exist
try:
- os.makedirs(os.path.dirname(file_path))
- print("subdir created: " + os.path.dirname(file_path))
+ os.makedirs(os.path.dirname(incoming_file_path))
+ print("subdir created: " + os.path.dirname(incoming_file_path))
except:
pass
# create input and output stream
- inputStream = dss_service.getFileFromSessionWorkspace(userSessionToken, file_name)
- outputStream = FileOutputStream(File(file_path))
+ inputStream = dss_service.getFileFromSessionWorkspace(userSessionToken, ws_file_path)
+ outputStream = FileOutputStream(File(incoming_file_path))
IOUtils.copyLarge(inputStream, outputStream)
IOUtils.closeQuietly(inputStream)
IOUtils.closeQuietly(outputStream)
- dst_dir2 = os.path.join(dst_dir, ws_folder)
- transaction.moveFile(File(dst_dir2).getAbsolutePath(), dataset);
+ print("transaction.move from incoming folder: " + incoming_dir)
+ transaction.moveFile(File(incoming_dir).getAbsolutePath(), dataset);
# temp_dir = prepareFilesForRegistration(transaction, file_names)
# ...and delete all files from the session workspace
@@ -279,23 +278,6 @@ def getThreadProperties(transaction):
return threadPropertyDict
-
-def registerNotebook(transaction, sample, container, parameters):
- """ creates a new dataset of type JUPYTER_NOTEBOOK.
- the parent dataset is the container.
- """
- dataSet = transaction.createNewDataSet("JUPYTER_NOTEBOOK")
- dataSet.setSample(sample)
- parent_codes = [container.getDataSetCode()]
- dataSet.setParentDatasets(parent_codes)
-
- files = parameters.get("notebook").get("fileNames")
- for file in files:
- pass
-
- return dataSet
-
-
def getThreadProperties(transaction):
threadPropertyDict = {}
threadProperties = transaction.getGlobalState().getThreadParameters().getThreadProperties()
--
GitLab