SSDM-13794: Improved error messages in default ELN dropbox.

ui-eln-lims/src/core-plugins/eln-lims/1/dss/drop-boxes/eln-lims-dropbox/eln-lims-dropbox.py

 import re
-import uuid
-
 from ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.fetchoptions import ExperimentFetchOptions
 from ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.id import ExperimentIdentifier
 from ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.fetchoptions import SampleFetchOptions
 from ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.id import SampleIdentifier
 from ch.systemsx.cisd.common.mail import EMailAddress
-from ch.systemsx.cisd.openbis.generic.client.web.client.exception import UserFailureException
 from ch.systemsx.cisd.openbis.dss.generic.shared import ServiceProvider
+from ch.systemsx.cisd.openbis.generic.client.web.client.exception import UserFailureException
 from java.io import File
 from java.nio.file import Files, Paths, StandardCopyOption
 from java.util import List
-from org.json import JSONObject
 from org.apache.commons.io import FileUtils
+from org.json import JSONObject
 
 INVALID_FORMAT_ERROR_MESSAGE = "Invalid format for the folder name, should follow the pattern <ENTITY_KIND>+<SPACE_CODE>+<PROJECT_CODE>+[<EXPERIMENT_CODE>|<SAMPLE_CODE>]+<OPTIONAL_DATASET_TYPE>+<OPTIONAL_NAME>";
 ILLEGAL_CHARACTERS_IN_FILE_NAMES_ERROR_MESSAGE = "Directory or its content contain illegal characters: \"', ~, $, %\"";
@@ -25,174 +23,184 @@ EXPERIMENT_MISSING_ERROR_MESSAGE = "Experiment not found";
 NAME_PROPERTY_SET_IN_TWO_PLACES_ERROR_MESSAGE = "$NAME property specified twice, it should just be in either folder name or metadata.json"
 EMAIL_SUBJECT = "ELN LIMS Dropbox Error";
 ILLEGAL_FILES = ["desktop.ini", "IconCache.db", "thumbs.db"];
-ILLEGAL_FILES_ERROR_MESSAGE = "Directory or contains illegal files: " + str(ILLEGAL_FILES);
-HIDDEN_FILES_ERROR_MESSAGE = "Directory or contains hidden files: files starting with '.'";
+ILLEGAL_FILES_ERROR_MESSAGE = "Directory contains illegal files: " + str(ILLEGAL_FILES);
+HIDDEN_FILES_ERROR_MESSAGE = "Directory contains hidden files: files starting with '.'";
+
+errorMessages = []
 
 def process(transaction):
     incoming = transaction.getIncoming();
     folderName = incoming.getName();
-
-    if not folderName.startswith('.'):
-        datasetInfo = folderName.split("+");
-        entityKind = None;
-        sample = None;
-        experiment = None;
-        datasetType = None;
-        name = None;
-
-        # Parse entity Kind
-        if len(datasetInfo) >= 1:
-            entityKind = datasetInfo[0];
-        else:
-            raise UserFailureException(INVALID_FORMAT_ERROR_MESSAGE + ":" + FAILED_TO_PARSE_ERROR_MESSAGE);
-
-        v3 = ServiceProvider.getV3ApplicationService();
-        sessionToken = transaction.getOpenBisServiceSessionToken();
-        projectSamplesEnabled = v3.getServerInformation(sessionToken)['project-samples-enabled'] == 'true'
-
-        # Parse entity Kind Format
-        if entityKind == "O":
-            if len(datasetInfo) >= 4 and projectSamplesEnabled:
-                sampleSpace = datasetInfo[1];
-                projectCode = datasetInfo[2];
-                sampleCode = datasetInfo[3];
-
-                emailAddress = getSampleRegistratorsEmail(transaction, sampleSpace, projectCode, sampleCode)
-                sample = transaction.getSample("/" + sampleSpace + "/" + projectCode + "/" + sampleCode);
-                if sample is None:
-                    reportIssue(transaction,
-                                INVALID_FORMAT_ERROR_MESSAGE + ":" + SAMPLE_MISSING_ERROR_MESSAGE,
-                                None);
-                if len(datasetInfo) >= 5:
-                    datasetType = datasetInfo[4];
-                if len(datasetInfo) >= 6:
-                    name = datasetInfo[5];
-                if len(datasetInfo) > 6:
-                    reportIssue(transaction,
-                                INVALID_FORMAT_ERROR_MESSAGE + ":" + FAILED_TO_PARSE_SAMPLE_ERROR_MESSAGE,
-                                emailAddress)
-            elif len(datasetInfo) >= 3 and not projectSamplesEnabled:
-                sampleSpace = datasetInfo[1];
-                sampleCode = datasetInfo[2];
-
-                emailAddress = getSampleRegistratorsEmail(transaction, sampleSpace, None, sampleCode)
-                sample = transaction.getSample("/" + sampleSpace + "/" + sampleCode);
-                if sample is None:
-                    reportIssue(transaction,
-                                INVALID_FORMAT_ERROR_MESSAGE + ":" + SAMPLE_MISSING_ERROR_MESSAGE,
-                                None);
+    emailAddress = None
+
+    try:
+        if not folderName.startswith('.'):
+            datasetInfo = folderName.split("+");
+            entityKind = None;
+            sample = None;
+            experiment = None;
+            datasetType = None;
+            name = None;
+
+            # Parse entity Kind
+            if len(datasetInfo) >= 1:
+                entityKind = datasetInfo[0];
+            else:
+                raise UserFailureException(INVALID_FORMAT_ERROR_MESSAGE + ":" + FAILED_TO_PARSE_ERROR_MESSAGE);
+
+            v3 = ServiceProvider.getV3ApplicationService();
+            sessionToken = transaction.getOpenBisServiceSessionToken();
+            projectSamplesEnabled = v3.getServerInformation(sessionToken)['project-samples-enabled'] == 'true'
+
+            # Parse entity Kind Format
+            if entityKind == "O":
+                if len(datasetInfo) >= 4 and projectSamplesEnabled:
+                    sampleSpace = datasetInfo[1];
+                    projectCode = datasetInfo[2];
+                    sampleCode = datasetInfo[3];
+
+                    emailAddress = getSampleRegistratorsEmail(transaction, sampleSpace, projectCode, sampleCode)
+                    sample = transaction.getSample("/" + sampleSpace + "/" + projectCode + "/" + sampleCode);
+                    if sample is None:
+                        reportIssue(INVALID_FORMAT_ERROR_MESSAGE + ":" + SAMPLE_MISSING_ERROR_MESSAGE)
+                        raise UserFailureException(INVALID_FORMAT_ERROR_MESSAGE + ":" + SAMPLE_MISSING_ERROR_MESSAGE)
+                    if len(datasetInfo) >= 5:
+                        datasetType = datasetInfo[4];
+                    if len(datasetInfo) >= 6:
+                        name = datasetInfo[5];
+                    if len(datasetInfo) > 6:
+                        reportIssue(INVALID_FORMAT_ERROR_MESSAGE + ":" + FAILED_TO_PARSE_SAMPLE_ERROR_MESSAGE)
+                elif len(datasetInfo) >= 3 and not projectSamplesEnabled:
+                    sampleSpace = datasetInfo[1];
+                    sampleCode = datasetInfo[2];
+
+                    emailAddress = getSampleRegistratorsEmail(transaction, sampleSpace, None, sampleCode)
+                    sample = transaction.getSample("/" + sampleSpace + "/" + sampleCode);
+                    if sample is None:
+                        reportIssue(INVALID_FORMAT_ERROR_MESSAGE + ":" + SAMPLE_MISSING_ERROR_MESSAGE)
+                        raise UserFailureException(INVALID_FORMAT_ERROR_MESSAGE + ":" + SAMPLE_MISSING_ERROR_MESSAGE)
+                    if len(datasetInfo) >= 4:
+                        datasetType = datasetInfo[3];
+                    if len(datasetInfo) >= 5:
+                        name = datasetInfo[4];
+                    if len(datasetInfo) > 5:
+                        reportIssue(INVALID_FORMAT_ERROR_MESSAGE + ":" + FAILED_TO_PARSE_SAMPLE_ERROR_MESSAGE)
+                else:
+                    raise UserFailureException(INVALID_FORMAT_ERROR_MESSAGE + ":" + FAILED_TO_PARSE_SAMPLE_ERROR_MESSAGE);
+
+                hiddenFiles = getHiddenFiles(incoming)
+                if hiddenFiles:
+                    reportIssue(HIDDEN_FILES_ERROR_MESSAGE + ":" + FAILED_TO_PARSE_SAMPLE_ERROR_MESSAGE + ":\n" + pathListToStr(hiddenFiles))
+
+                illegalFiles = getIllegalFiles(incoming)
+                if illegalFiles:
+                    reportIssue(ILLEGAL_FILES_ERROR_MESSAGE + ":" + FAILED_TO_PARSE_SAMPLE_ERROR_MESSAGE + ":\n" + pathListToStr(illegalFiles))
+
+                filesWithIllegalCharacters = getFilesWithIllegalCharacters(incoming)
+                if filesWithIllegalCharacters:
+                    reportIssue(ILLEGAL_CHARACTERS_IN_FILE_NAMES_ERROR_MESSAGE + ":"
+                                + FAILED_TO_PARSE_SAMPLE_ERROR_MESSAGE + ":\n" + pathListToStr(filesWithIllegalCharacters))
+
+                readOnlyFiles = getReadOnlyFiles(incoming)
+                if readOnlyFiles:
+                    reportIssue(FOLDER_CONTAINS_NON_DELETABLE_FILES_ERROR_MESSAGE + ":" + FAILED_TO_PARSE_SAMPLE_ERROR_MESSAGE + ":\n" + pathListToStr(readOnlyFiles));
+            if entityKind == "E":
                 if len(datasetInfo) >= 4:
-                    datasetType = datasetInfo[3];
-                if len(datasetInfo) >= 5:
-                    name = datasetInfo[4];
-                if len(datasetInfo) > 5:
-                    reportIssue(transaction,
-                                INVALID_FORMAT_ERROR_MESSAGE + ":" + FAILED_TO_PARSE_SAMPLE_ERROR_MESSAGE,
-                                emailAddress)
+                    experimentSpace = datasetInfo[1];
+                    projectCode = datasetInfo[2];
+                    experimentCode = datasetInfo[3];
+
+                    emailAddress = getExperimentRegistratorsEmail(transaction, experimentSpace, projectCode,
+                                                                  experimentCode);
+                    experiment = transaction.getExperiment("/" + experimentSpace + "/" + projectCode + "/" + experimentCode);
+                    if experiment is None:
+                        reportIssue(INVALID_FORMAT_ERROR_MESSAGE + ":" + EXPERIMENT_MISSING_ERROR_MESSAGE)
+                        raise UserFailureException(INVALID_FORMAT_ERROR_MESSAGE + ":" + EXPERIMENT_MISSING_ERROR_MESSAGE)
+                    if len(datasetInfo) >= 5:
+                        datasetType = datasetInfo[4];
+                    if len(datasetInfo) >= 6:
+                        name = datasetInfo[5];
+                    if len(datasetInfo) > 6:
+                        reportIssue(INVALID_FORMAT_ERROR_MESSAGE + ":" + FAILED_TO_PARSE_EXPERIMENT_ERROR_MESSAGE);
+                else:
+                    raise UserFailureException(INVALID_FORMAT_ERROR_MESSAGE + ":" + FAILED_TO_PARSE_EXPERIMENT_ERROR_MESSAGE);
+
+                hiddenFiles = getHiddenFiles(incoming)
+                if hiddenFiles:
+                    reportIssue(HIDDEN_FILES_ERROR_MESSAGE + ":" + FAILED_TO_PARSE_EXPERIMENT_ERROR_MESSAGE + ":\n" + pathListToStr(hiddenFiles))
+
+                illegalFiles = getIllegalFiles(incoming)
+                if illegalFiles:
+                    reportIssue(ILLEGAL_FILES_ERROR_MESSAGE + ":" + FAILED_TO_PARSE_EXPERIMENT_ERROR_MESSAGE + ":\n" + pathListToStr(illegalFiles))
+
+                filedWithIllegalCharacters = getFilesWithIllegalCharacters(incoming)
+                if filedWithIllegalCharacters:
+                    reportIssue(ILLEGAL_CHARACTERS_IN_FILE_NAMES_ERROR_MESSAGE + ":"
+                                + FAILED_TO_PARSE_EXPERIMENT_ERROR_MESSAGE + ":\n" + pathListToStr(filedWithIllegalCharacters))
+
+                readOnlyFiles = getReadOnlyFiles(incoming)
+                if readOnlyFiles:
+                    reportIssue(FOLDER_CONTAINS_NON_DELETABLE_FILES_ERROR_MESSAGE + ":"
+                                + FAILED_TO_PARSE_EXPERIMENT_ERROR_MESSAGE + ":\n" + pathListToStr(readOnlyFiles))
+
+            # Create dataset
+            dataSet = None;
+            if datasetType is not None:  # Set type if found
+                dataSet = transaction.createNewDataSet(datasetType);
             else:
-                raise UserFailureException(INVALID_FORMAT_ERROR_MESSAGE + ":" + FAILED_TO_PARSE_SAMPLE_ERROR_MESSAGE);
-
-            if hasFolderHiddenFiles(incoming):
-                reportIssue(transaction, HIDDEN_FILES_ERROR_MESSAGE + ":"
-                            + FAILED_TO_PARSE_SAMPLE_ERROR_MESSAGE, emailAddress);
-            if hasFolderIllegalFiles(incoming):
-                reportIssue(transaction, ILLEGAL_FILES_ERROR_MESSAGE + ":"
-                            + FAILED_TO_PARSE_SAMPLE_ERROR_MESSAGE, emailAddress);
-            if hasFolderIllegalCharacters(incoming):
-                reportIssue(transaction, ILLEGAL_CHARACTERS_IN_FILE_NAMES_ERROR_MESSAGE + ":"
-                            + FAILED_TO_PARSE_SAMPLE_ERROR_MESSAGE, emailAddress);
-            if hasFolderReadOnlyFiles(incoming):
-                reportIssue(transaction, FOLDER_CONTAINS_NON_DELETABLE_FILES_ERROR_MESSAGE + ":"
-                            + FAILED_TO_PARSE_SAMPLE_ERROR_MESSAGE, emailAddress);
-        if entityKind == "E":
-            if len(datasetInfo) >= 4:
-                experimentSpace = datasetInfo[1];
-                projectCode = datasetInfo[2];
-                experimentCode = datasetInfo[3];
-
-                emailAddress = getExperimentRegistratorsEmail(transaction, experimentSpace, projectCode,
-                                                              experimentCode);
-                experiment = transaction.getExperiment("/" + experimentSpace + "/" + projectCode + "/" + experimentCode);
-                if experiment is None:
-                    reportIssue(transaction,
-                                INVALID_FORMAT_ERROR_MESSAGE + ":" + EXPERIMENT_MISSING_ERROR_MESSAGE,
-                                None);
-                if len(datasetInfo) >= 5:
-                    datasetType = datasetInfo[4];
-                if len(datasetInfo) >= 6:
-                    name = datasetInfo[5];
-                if len(datasetInfo) > 6:
-                    reportIssue(transaction,
-                                INVALID_FORMAT_ERROR_MESSAGE + ":" + FAILED_TO_PARSE_EXPERIMENT_ERROR_MESSAGE,
-                                emailAddress);
+                dataSet = transaction.createNewDataSet();
+
+            if name is not None:
+                dataSet.setPropertyValue("$NAME", name);  # Set name if found
+
+            # Set sample or experiment
+            if sample is not None:
+                dataSet.setSample(sample);
             else:
-                raise UserFailureException(INVALID_FORMAT_ERROR_MESSAGE + ":" + FAILED_TO_PARSE_EXPERIMENT_ERROR_MESSAGE);
-
-            if hasFolderHiddenFiles(incoming):
-                reportIssue(transaction, HIDDEN_FILES_ERROR_MESSAGE + ":"
-                            + FAILED_TO_PARSE_EXPERIMENT_ERROR_MESSAGE, emailAddress);
-            if hasFolderIllegalFiles(incoming):
-                reportIssue(transaction, ILLEGAL_FILES_ERROR_MESSAGE + ":"
-                            + FAILED_TO_PARSE_EXPERIMENT_ERROR_MESSAGE, emailAddress);
-            if hasFolderIllegalCharacters(incoming):
-                reportIssue(transaction, ILLEGAL_CHARACTERS_IN_FILE_NAMES_ERROR_MESSAGE + ":"
-                            + FAILED_TO_PARSE_EXPERIMENT_ERROR_MESSAGE, emailAddress);
-            if hasFolderReadOnlyFiles(incoming):
-                reportIssue(transaction, FOLDER_CONTAINS_NON_DELETABLE_FILES_ERROR_MESSAGE + ":"
-                            + FAILED_TO_PARSE_EXPERIMENT_ERROR_MESSAGE, emailAddress);
-
-        # Create dataset
-        dataSet = None;
-        if datasetType is not None:  # Set type if found
-            dataSet = transaction.createNewDataSet(datasetType);
-        else:
-            dataSet = transaction.createNewDataSet();
-
-        if name is not None:
-            dataSet.setPropertyValue("$NAME", name);  # Set name if found
-
-        # Set sample or experiment
-        if sample is not None:
-            dataSet.setSample(sample);
-        else:
-            dataSet.setExperiment(experiment);
-
-        # Move folder to dataset
-        filesInFolder = incoming.listFiles();
-
-        itemsInFolder = 0;
-        datasetItem = None;
-        for item in filesInFolder:
-            fileName = item.getName()
-            if fileName == "metadata.json":
-                root = JSONObject(FileUtils.readFileToString(item, "UTF-8"))
-                properties = root.get("properties")
-                for propertyKey in properties.keys():
-                    if propertyKey == "$NAME" and name is not None:
-                        raise UserFailureException(NAME_PROPERTY_SET_IN_TWO_PLACES_ERROR_MESSAGE)
-                    propertyValue = properties.get(propertyKey)
-                    if propertyValue is not None:
-                        propertyValueString = str(propertyValue)
-                        dataSet.setPropertyValue(propertyKey, propertyValueString)
+                dataSet.setExperiment(experiment);
+
+            # Move folder to dataset
+            filesInFolder = incoming.listFiles();
+
+            itemsInFolder = 0;
+            datasetItem = None;
+            for item in filesInFolder:
+                fileName = item.getName()
+                if fileName == "metadata.json":
+                    root = JSONObject(FileUtils.readFileToString(item, "UTF-8"))
+                    properties = root.get("properties")
+                    for propertyKey in properties.keys():
+                        if propertyKey == "$NAME" and name is not None:
+                            raise UserFailureException(NAME_PROPERTY_SET_IN_TWO_PLACES_ERROR_MESSAGE)
+                        propertyValue = properties.get(propertyKey)
+                        if propertyValue is not None:
+                            propertyValueString = str(propertyValue)
+                            dataSet.setPropertyValue(propertyKey, propertyValueString)
+                else:
+                    itemsInFolder = itemsInFolder + 1;
+                    datasetItem = item;
+
+            if itemsInFolder > 1:
+                tmpPath = incoming.getAbsolutePath() + "/default";
+                tmpDir = File(tmpPath);
+                tmpDir.mkdir();
+
+                try:
+                    for inputFile in filesInFolder:
+                        Files.move(inputFile.toPath(), Paths.get(tmpPath, inputFile.getName()),
+                                   StandardCopyOption.ATOMIC_MOVE);
+                    transaction.moveFile(tmpDir.getAbsolutePath(), dataSet);
+                finally:
+                    if tmpDir is not None:
+                        tmpDir.delete();
             else:
-                itemsInFolder = itemsInFolder + 1;
-                datasetItem = item;
-
-        if itemsInFolder > 1:
-            tmpPath = incoming.getAbsolutePath() + "/default";
-            tmpDir = File(tmpPath);
-            tmpDir.mkdir();
-
-            try:
-                for inputFile in filesInFolder:
-                    Files.move(inputFile.toPath(), Paths.get(tmpPath, inputFile.getName()),
-                               StandardCopyOption.ATOMIC_MOVE);
-                transaction.moveFile(tmpDir.getAbsolutePath(), dataSet);
-            finally:
-                if tmpDir is not None:
-                    tmpDir.delete();
-        else:
-            transaction.moveFile(datasetItem.getAbsolutePath(), dataSet);
+                transaction.moveFile(datasetItem.getAbsolutePath(), dataSet);
+    finally:
+        reportAllIssues(transaction, emailAddress)
+
+
+def pathListToStr(list):
+    return "\n".join(list)
 
 
 def getContactsEmailAddresses(transaction):
@@ -200,62 +208,69 @@ def getContactsEmailAddresses(transaction):
     return re.split("[,;]", emailString) if emailString is not None else []
 
 
-def reportIssue(transaction, errorMessage, emailAddress):
-    contacts = getContactsEmailAddresses(transaction);
-    allAddresses = [emailAddress] + contacts if emailAddress is not None else contacts;
-    sendMail(transaction, map(lambda address: EMailAddress(address), allAddresses), EMAIL_SUBJECT, errorMessage);
-    raise UserFailureException(errorMessage);
+def reportIssue(errorMessage):
+    errorMessages.append(errorMessage)
+
+
+def reportAllIssues(transaction, emailAddress):
+    if len(errorMessages) > 0:
+        contacts = getContactsEmailAddresses(transaction)
+        allAddresses = [emailAddress] + contacts if emailAddress is not None else contacts
+        joinedErrorMessages = "\n".join(errorMessages)
+        sendMail(transaction, map(lambda address: EMailAddress(address), allAddresses), EMAIL_SUBJECT, joinedErrorMessages);
+        raise UserFailureException(joinedErrorMessages)
 
 
-def hasFolderIllegalCharacters(incoming):
-    if bool(re.search(r"['~$%]", incoming.getName())):
-        return True;
+def getFilesWithIllegalCharacters(folder):
+    result = []
+    if bool(re.search(r"['~$%]", folder.getPath())):
+        result.append(folder.getName())
 
-    files = incoming.listFiles()
+    files = folder.listFiles()
     if files is not None:
         for f in files:
-            if hasFolderIllegalCharacters(f):
-                return True;
+            result.extend(getFilesWithIllegalCharacters(f))
 
     return False;
 
 
-def hasFolderHiddenFiles(incoming):
-    if incoming.getName().startswith("."):
-        return True;
+def getHiddenFiles(folder):
+    result = []
+    if folder.getName().startswith("."):
+        result.append(folder.getPath())
 
-    files = incoming.listFiles()
+    files = folder.listFiles()
     if files is not None:
         for f in files:
-            if hasFolderHiddenFiles(f):
-                return True;
+            result.extend(getHiddenFiles(f))
+
+    return result
 
-    return False;
 
-def hasFolderIllegalFiles(incoming):
-    if incoming.getName() in ILLEGAL_FILES:
-        return True;
+def getIllegalFiles(folder):
+    result = []
+    if folder.getName() in ILLEGAL_FILES:
+        result.append(folder.getPath())
 
-    files = incoming.listFiles()
+    files = folder.listFiles()
     if files is not None:
         for f in files:
-            if hasFolderIllegalFiles(f):
-                return True;
+            result.extend(getIllegalFiles(f))
 
-    return False;
+    return result
 
 
-def hasFolderReadOnlyFiles(incoming):
-    if not incoming.renameTo(incoming):
-        return True;
+def getReadOnlyFiles(folder):
+    result = []
+    if not folder.renameTo(folder):
+        result.append(folder.getPath())
 
-    files = incoming.listFiles()
+    files = folder.listFiles()
     if files is not None:
         for f in files:
-            if hasFolderReadOnlyFiles(f):
-                return True;
+            result.extend(getReadOnlyFiles(f))
 
-    return False;
+    return result
 
 
 def sendMail(transaction, emailAddresses, subject, body):