[LMS-2301] example reporting and processing scripts + simple handler and a validator

SVN: 21921

datastore_server/resource/examples/data-set-handler.py

+transaction = service.transaction()
+dataSet = transaction.createNewDataSet()
+transaction.moveFile(incoming.getAbsolutePath(), dataSet)
+dataSet.setExperiment(transaction.getExperiment("/CISD/NEMO/EXP1"))
\ No newline at end of file

datastore_server/resource/examples/data-set-processing-mail.py

+import org.apache.commons.io.IOUtils as IOUtils
+
+def process(dataSet):
+    dataSetCode = dataSet.getDataSetCode()
+    print "script processing " + dataSetCode
+    processNode(dataSet.getContent().getRootNode(), dataSet.getDataSetCode())
+    
+def processNode(node, dataSetCode):
+    print "process node: " + dataSetCode + "/" + node.getRelativePath()
+    if node.isDirectory():
+        for child in node.getChildNodes():
+            processNode(child, dataSetCode)
+    else:
+        fileAsString = IOUtils.readLines(node.getInputStream()).toString()
+        fileName = node.getName()
+        
+        if fileName.endswith(".txt"):
+            mailService.createEmailSender().\
+                withSubject("processed text file " + fileName).\
+                withBody("see the attached file").\
+                withAttachedText(fileAsString, fileName).\
+                send()
+        else:
+            filePath = node.getFile().getPath()
+            mailService.createEmailSender().\
+                withSubject("processed file " + fileName).\
+                withBody("see the attached file").\
+                withAttachedFile(filePath, fileName).\
+                send()
\ No newline at end of file

datastore_server/resource/examples/data-set-processing.py

+import org.apache.commons.io.IOUtils as IOUtils
+
+def process(dataSet):
+    dataSetCode = dataSet.getDataSetCode()
+    print "script processing " + dataSetCode
+    processNode(dataSet.getContent().getRootNode(), dataSet.getDataSetCode())
+    
+def processNode(node, dataSetCode):
+    print "process node: " + dataSetCode + "/" + node.getRelativePath()
+    if node.isDirectory():
+        for child in node.getChildNodes():
+            processNode(child, dataSetCode)
+    else:
+        print "content (" + str(node.getFileLength()) + "): " + \
+              IOUtils.readLines(node.getInputStream()).toString()
\ No newline at end of file

datastore_server/resource/examples/data-set-reporting-content.py

+import java.util.Date as Date
+
+CODE = "Code"
+FILE_NAME = "File Name"
+RELATIVE_PATH = "Relative Path"
+LAST_MODIFIED = "Last Modified"
+SIZE = "Size"
+
+def describe(dataSets, tableBuilder):
+    tableBuilder.addHeader(CODE)
+    tableBuilder.addHeader(FILE_NAME)
+    tableBuilder.addHeader(RELATIVE_PATH)
+    tableBuilder.addHeader(LAST_MODIFIED)
+    tableBuilder.addHeader(SIZE)
+    for dataSet in dataSets:
+        print "script reporting " + dataSet.getDataSetCode()
+        describeNode(dataSet.getContent().getRootNode(), dataSet.getDataSetCode(), tableBuilder)
+        
+
+def describeNode(node, dataSetCode, tableBuilder):
+    print "describe node: " + dataSetCode + "/" + node.getRelativePath()
+    if node.isDirectory():
+        for child in node.getChildNodes():
+            describeNode(child, dataSetCode, tableBuilder)
+    else:
+        row = tableBuilder.addRow()
+        row.setCell(CODE, dataSetCode)
+        row.setCell(FILE_NAME, node.getName())
+        row.setCell(RELATIVE_PATH, node.getRelativePath())
+        row.setCell(LAST_MODIFIED, Date(node.getLastModified()))
+        row.setCell(SIZE, node.getFileLength())
\ No newline at end of file

datastore_server/resource/examples/data-set-reporting-search.py

+CODE = "Data Set Code"
+EXPERIMENT_IDENTIFIER = "Experiment Identifier"
+EXPERIMENT_TYPE = "Experiment Type"
+EXPERIMENT_DESCRIPTION = "Description"
+
+def describe(dataSets, tableBuilder):
+    
+    tableBuilder.addHeader(CODE)
+    tableBuilder.addHeader(EXPERIMENT_IDENTIFIER)
+    tableBuilder.addHeader(EXPERIMENT_TYPE)
+    tableBuilder.addHeader(EXPERIMENT_DESCRIPTION)
+    
+    for dataSet in dataSets:
+        projectIdentifier = "/" + dataSet.getSpaceCode() + "/" + dataSet.getProjectCode()
+        print "script reporting " + dataSet.getDataSetCode() + " from " + projectIdentifier
+        experiments = searchService.listExperiments(projectIdentifier)
+        
+        for experiment in experiments:
+            row = tableBuilder.addRow()
+            row.setCell(CODE, dataSet.getDataSetCode())
+            row.setCell(EXPERIMENT_IDENTIFIER, experiment.getExperimentIdentifier())
+            row.setCell(EXPERIMENT_TYPE, experiment.getExperimentType())
+            row.setCell(EXPERIMENT_DESCRIPTION, experiment.getPropertyValue("DESCRIPTION"))
\ No newline at end of file

datastore_server/resource/examples/data-set-reporting.py

+CODE = "Code"
+TYPE = "Type"
+SIZE = "Size"
+LOCATION = "Location"
+SPEED_HINT = "Speed Hint"
+MAIN_PATTERN = "Main Data Set Pattern"
+MAIN_PATH = "Main Data Set Path"
+INSTANCE = "Instance"
+SPACE = "Space"
+PROJECT = "Project"
+EXPERIMENT_CODE = "Experiment Code"
+EXPERIMENT_IDENTIFIER = "Experiment Identifier"
+EXPERIMENT_TYPE = "Experiment Type"
+SAMPLE_CODE = "Sample Code"
+SAMPLE_IDENTIFIER = "Sample Identifier"
+SAMPLE_TYPE = "Sample Type"
+
+def describe(dataSets, tableBuilder):
+    
+    tableBuilder.addHeader(CODE)
+    tableBuilder.addHeader(TYPE)
+    tableBuilder.addHeader(SIZE)
+    tableBuilder.addHeader(LOCATION)
+    tableBuilder.addHeader(SPEED_HINT)
+    tableBuilder.addHeader(MAIN_PATTERN)
+    tableBuilder.addHeader(MAIN_PATH)
+    tableBuilder.addHeader(INSTANCE)
+    tableBuilder.addHeader(SPACE)
+    tableBuilder.addHeader(PROJECT)
+    tableBuilder.addHeader(EXPERIMENT_CODE)
+    tableBuilder.addHeader(EXPERIMENT_IDENTIFIER)
+    tableBuilder.addHeader(EXPERIMENT_TYPE)
+    tableBuilder.addHeader(SAMPLE_CODE)
+    tableBuilder.addHeader(SAMPLE_IDENTIFIER)
+    tableBuilder.addHeader(SAMPLE_TYPE)
+    
+    for dataSet in dataSets:
+        print "script reporting " + dataSet.getDataSetCode()
+        
+        row = tableBuilder.addRow()
+        row.setCell(CODE, dataSet.getDataSetCode())
+        row.setCell(TYPE, dataSet.getDataSetTypeCode())
+        row.setCell(SIZE, dataSet.getDataSetSize())
+        row.setCell(LOCATION, dataSet.getDataSetLocation())
+        row.setCell(SPEED_HINT, dataSet.getSpeedHint())
+        row.setCell(MAIN_PATTERN, dataSet.getMainDataSetPattern())
+        row.setCell(MAIN_PATH, dataSet.getMainDataSetPath())
+        row.setCell(INSTANCE, dataSet.getInstanceCode())
+        row.setCell(SPACE, dataSet.getSpaceCode())
+        row.setCell(PROJECT, dataSet.getProjectCode())
+        row.setCell(EXPERIMENT_CODE, dataSet.getExperimentCode())
+        row.setCell(EXPERIMENT_IDENTIFIER, dataSet.getExperimentIdentifier())
+        row.setCell(EXPERIMENT_TYPE, dataSet.getExperimentTypeCode())
+        row.setCell(SAMPLE_CODE, dataSet.getSampleCode())
+        row.setCell(SAMPLE_IDENTIFIER, dataSet.getSampleIdentifier())
+        row.setCell(SAMPLE_TYPE, dataSet.getSampleTypeCode())
\ No newline at end of file

datastore_server/resource/examples/data-set-validator.py

+import os
+
+def validate_data_set_file(file):
+
+    errors = []
+
+    if file.isFile():
+        errors.append(createFileValidationError("'" + file.getPath() + "' is a normal file. Expected a directory."))
+    else:
+        for f in file.listFiles():
+            if f.getName().startswith('file'):
+                pf = open(f.getPath(), "r")
+                if pf.read().find('invalid') > -1:
+                    errors.append(createFileValidationError("File '" + f.getPath() + "' contains invalid text."))
+            else:
+                errors.append(createFileValidationError("File '" + f.getPath() + "' doesn't match file name pattern 'file*'."))
+
+    return errors
\ No newline at end of file