container and result registration working

src/core-plugins/jupyter-api/1/dss/reporting-plugins/jupyter-uploader-api/dataset-uploader-api.py

@@ -31,6 +31,8 @@ import subprocess
 import os
 import re
 import sys
+import shutil
+import errno
 
 
 
@@ -48,7 +50,7 @@ def getSampleByIdentifier(transaction, identifier):
     if len(found) == 1:
         return transaction.makeSampleMutable(found[0]);
     else:
-        raise UserFailureException(identifier + "Not found by search service.");
+        return None
 
 
 def get_dataset_for_name(transaction, dataset_name):
@@ -63,6 +65,7 @@ def get_dataset_for_name(transaction, dataset_name):
     else:
         return None
 
+
 def get_dataset_for_permid(transaction, permid):
 
     search_service = transaction.getSearchService()
@@ -75,13 +78,6 @@ def get_dataset_for_permid(transaction, permid):
     else:
         return None
 
-def get_username_sessionid(sessionToken):
-    """ divides a session-token into username and sessionId. Username may contain a dash (-)
-    """
-    m = re.compile('(.*)-([^-]*)').match(sessionToken)
-    if m:
-        return m.group(1), m.group(2)
-
 
 def process(transaction, parameters, tableBuilder):
     ''' 
@@ -114,34 +110,39 @@ def process(transaction, parameters, tableBuilder):
 
     # get mandatory sample to connect the container to
     sampleIdentifier = parameters.get("sample").get("identifier")
-    print('sampleIdentifier: ' + sampleIdentifier)
+    print('looking for sample with identifier: ' + sampleIdentifier)
     if sampleIdentifier is None:
         raise UserFailureException('mandatory parameter sample["identifier"] is missing')
     sample = getSampleByIdentifier(transaction, sampleIdentifier)
     if sample == None: 
-        raise UserFailureException("no sample found with this identifier: " + sampleIdentifier)
+        raise Exception("no sample found with this identifier: " + sampleIdentifier)
+
+    everything_ok = True
 
-    #container = registerContainer(transaction, sample, parameters)
-    #everything_ok = register_files(transaction, sample, container, parameters)
+    dataset_codes= []
+   # #everything_ok = register_files(transaction, sample, container, parameters)
     if parameters.get("result").get("fileNames") is not None:
-        everything_ok = register_files(
+        dataset_code = register_files(
             transaction, 
             "JUPYTER_RESULT",
             sample, 
-            'container', 
+            'container_permId', 
+            
             parameters.get("result").get("fileNames")
         )
-
-    if parameters.get("notebook").get("fileNames") is not None:
-        everything_ok = register_files(
-            transaction, 
-            "JUPYTER_NOTEBOOK",
-            sample, 
-            'container',
-            parameters.get("notebook").get("fileNames")
-        )
-
-    
+        dataset_codes.append(dataset_code)
+
+    container = registerContainer(transaction, sample, parameters, dataset_codes)
+   # if parameters.get("notebook").get("fileNames") is not None:
+   #     everything_ok = register_files(
+   #         transaction, 
+   #         "JUPYTER_NOTEBOOK",
+   #         sample, 
+   #         'container',
+   #         parameters.get("notebook").get("fileNames")
+   #     )
+
+   # 
     # create the dataset
     if everything_ok:
         # Success message
@@ -162,45 +163,6 @@ def process(transaction, parameters, tableBuilder):
         row.setCell("MESSAGE", "Dataset registration failed")
 
 
-def getThreadProperties(transaction):
-  threadPropertyDict = {}
-  threadProperties = transaction.getGlobalState().getThreadParameters().getThreadProperties()
-  for key in threadProperties:
-    try:
-      threadPropertyDict[key] = threadProperties.getProperty(key)
-    except:
-      pass
-  return threadPropertyDict
-
-def registerContainer(transaction, sample, parameters):
-
-    container_name = parameters.get("container").get("name")
-    container_description = parameters.get("container").get("description")
-
-    # make sure container dataset doesn't exist yet
-    container = get_dataset_for_name(transaction, container_name)
-    if container is None:
-        print("creating JUPYTER_CONTAINER dataset...")
-        # Create new container (a dataset of type "JUPYTER_CONTAINER")
-        container = transaction.createNewDataSet("JUPYTER_CONTAINER")
-        container.setSample(sample)
-        container.setPropertyValue("NAME", container_name)
-        container.setPropertyValue("DESCRIPTION", container_description)
-        #container.setParentDatasets(parameters.get("parents"))
-        print("JUPYTER_CONTAINER permId: " + container.getDataSetCode())
-
-    # Assign Data Set properties
-    # set name and description
-    for key in parameters.get("container").keySet():
-        propertyValue = unicode(parameters.get(key))
-        if propertyValue == "":
-            propertyValue = None
-        container.setPropertyValue(key,propertyValue)
-
-    # TODO: container registrieren, aber wie???
-    #transaction.moveFile(None, container);
-    return container
-
 def register_files(transaction, dataset_type, sample, container, file_names):
     """ creates a new dataset of type JUPYTER_RESULT.
     the parent dataset is the JUPYTER_CONTAINER we just created
@@ -210,62 +172,87 @@ def register_files(transaction, dataset_type, sample, container, file_names):
     - finally, the remaining files are deleted from the session workspace
     """
     print("creating " + dataset_type + " dataset...")
-    result_ds = transaction.createNewDataSet(dataset_type)
-    result_ds.setSample(sample)
-    #result_ds.setParentDatasets([container.getDataSetCode()])
-    #print("JUPYTER RESULT permId: " + container.getDataSetCode())
-    
-    # copy the files to the temp dir
-    temp_dir = prepareFilesForRegistration(transaction, file_names)
-    transaction.moveFile(File(temp_dir).getAbsolutePath(), result_ds);
-
-    # ...and delete all files from the session workspace
-    # TODO: delete it later
-    #dss_service = ServiceProvider.getDssServiceRpcGeneric().getService()
-    #for file_name in file_names:
-    #    file_path = os.path.join(temp_dir, file_name)
-    #    dss_service.deleteSessionWorkspaceFile(userSessionToken, file_name)
-
-    return True
+    new_dataset = transaction.createNewDataSet(dataset_type)
+    new_dataset.setSample(sample)
+    # TODO: set the container just created as the parent dataset
+    #new_dataset.setParentDatasets([container.getDataSetCode()])
+    print("JUPYTER_RESULT permId: " + new_dataset.getDataSetCode())
 
-
-def prepareFilesForRegistration(transaction, files=[]):
-    """ Bring files to the same file system as the dropbox.
-    The session workspace may be on a different file system from the dropbox.
-    We need to ensure that all files are on the dropbox file system.
-    """
-    # create a local temp dir with a timestamp
+    
+    # create temporary folder in incoming-dir ( openbis/servers/datastore_server/data/incoming )
     threadProperties = getThreadProperties(transaction)
-    #temp_dir =  os.path.join( threadProperties[u'incoming-dir'], str(time.time()) )
-    temp_dir =  threadProperties[u'incoming-dir']
-    File(temp_dir).mkdirs()
+    print(threadProperties)
+    dst_dir =  os.path.join( threadProperties[u'incoming-dir'], str(time.time()) )
+    print("incoming folder: " + dst_dir)
+    #File(dst_dir).mkdirs()
 
     dss_service = ServiceProvider.getDssServiceRpcGeneric().getService()
 
-    # download all files from the session workspace to the temp dir
-    for file_name in files:
-        # create input stream
-        print("file_name: " + file_name)
-        inputStream = dss_service.getFileFromSessionWorkspace(userSessionToken, file_name)
+    for file_name in file_names:
+        print("copying file: " + file_name)
+        file_path = os.path.join(dst_dir, file_name)
+        print("to: "+file_path)
 
         # ensure that all necessary folders exist
-        file_path = os.path.join(temp_dir, file_name)
-        print("file_path: "+file_path)
-        
         try:
             os.makedirs(os.path.dirname(file_path))
+            print("subdir created: " + os.path.dirname(file_path))
         except:
             pass
-        print("dirname: " + os.path.dirname(file_path))
 
-        # create output stream
-        tempFile = File(file_path)
-        outputStream = FileOutputStream(tempFile)
+
+        # create input and output stream
+        inputStream = dss_service.getFileFromSessionWorkspace(userSessionToken, file_name)
+        outputStream = FileOutputStream(File(file_path))
         IOUtils.copyLarge(inputStream, outputStream)
         IOUtils.closeQuietly(inputStream)
         IOUtils.closeQuietly(outputStream)
 
-    return temp_dir
+    dst_dir2 = os.path.join(dst_dir, 'results')
+    transaction.moveFile(File(dst_dir2).getAbsolutePath(), new_dataset);
+#    temp_dir = prepareFilesForRegistration(transaction, file_names)
+
+    # ...and delete all files from the session workspace
+    # TODO: delete it later
+    #dss_service = ServiceProvider.getDssServiceRpcGeneric().getService()
+    #for file_name in file_names:
+    #    file_path = os.path.join(temp_dir, file_name)
+    #    dss_service.deleteSessionWorkspaceFile(userSessionToken, file_name)
+
+    return new_dataset.getDataSetCode()
+
+
+def getThreadProperties(transaction):
+  threadPropertyDict = {}
+  threadProperties = transaction.getGlobalState().getThreadParameters().getThreadProperties()
+  for key in threadProperties:
+    try:
+      threadPropertyDict[key] = threadProperties.getProperty(key)
+    except:
+      pass
+  return threadPropertyDict
+
+def registerContainer(transaction, sample, parameters, contained_dataset_codes):
+
+    container_name = parameters.get("container").get("name")
+    container_description = parameters.get("container").get("description")
+
+    # make sure container dataset doesn't exist yet
+    container = get_dataset_for_name(transaction, container_name)
+    if container is None:
+        print("creating new JUPYTER_CONTAINER dataset...")
+        # Create new container (a dataset of type "JUPYTER_CONTAINER")
+        container = transaction.createNewDataSet("JUPYTER_CONTAINER")
+        container.setSample(sample)
+        container.setPropertyValue("NAME", container_name)
+        container.setPropertyValue("DESCRIPTION", container_description)
+    
+    
+    container.setContainedDataSetCodes(contained_dataset_codes)
+    print("JUPYTER_CONTAINER permId: " + container.getDataSetCode())
+
+    return container
+
 
 def registerNotebook(transaction, sample, container, parameters):
     """ creates a new dataset of type JUPYTER_NOTEBOOK.