From f6e522b790f4f03b2006e2965d1f53ad03ed416b Mon Sep 17 00:00:00 2001
From: buczekp <buczekp>
Date: Thu, 30 Jun 2011 07:52:29 +0000
Subject: [PATCH] [LMS-2301] example reporting and processing scripts + simple
handler and a validator
SVN: 21921
---
.../resource/examples/data-set-handler.py | 4 ++
.../examples/data-set-processing-mail.py | 29 ++++++++++
.../resource/examples/data-set-processing.py | 15 +++++
.../examples/data-set-reporting-content.py | 31 ++++++++++
.../examples/data-set-reporting-search.py | 23 ++++++++
.../resource/examples/data-set-reporting.py | 56 +++++++++++++++++++
.../resource/examples/data-set-validator.py | 18 ++++++
7 files changed, 176 insertions(+)
create mode 100644 datastore_server/resource/examples/data-set-handler.py
create mode 100644 datastore_server/resource/examples/data-set-processing-mail.py
create mode 100644 datastore_server/resource/examples/data-set-processing.py
create mode 100644 datastore_server/resource/examples/data-set-reporting-content.py
create mode 100644 datastore_server/resource/examples/data-set-reporting-search.py
create mode 100644 datastore_server/resource/examples/data-set-reporting.py
create mode 100644 datastore_server/resource/examples/data-set-validator.py
diff --git a/datastore_server/resource/examples/data-set-handler.py b/datastore_server/resource/examples/data-set-handler.py
new file mode 100644
index 00000000000..10e6e80b0da
--- /dev/null
+++ b/datastore_server/resource/examples/data-set-handler.py
@@ -0,0 +1,4 @@
+transaction = service.transaction()
+dataSet = transaction.createNewDataSet()
+transaction.moveFile(incoming.getAbsolutePath(), dataSet)
+dataSet.setExperiment(transaction.getExperiment("/CISD/NEMO/EXP1"))
\ No newline at end of file
diff --git a/datastore_server/resource/examples/data-set-processing-mail.py b/datastore_server/resource/examples/data-set-processing-mail.py
new file mode 100644
index 00000000000..e02266e75b9
--- /dev/null
+++ b/datastore_server/resource/examples/data-set-processing-mail.py
@@ -0,0 +1,29 @@
+import org.apache.commons.io.IOUtils as IOUtils
+
+def process(dataSet):
+ dataSetCode = dataSet.getDataSetCode()
+ print "script processing " + dataSetCode
+ processNode(dataSet.getContent().getRootNode(), dataSet.getDataSetCode())
+
+def processNode(node, dataSetCode):
+ print "process node: " + dataSetCode + "/" + node.getRelativePath()
+ if node.isDirectory():
+ for child in node.getChildNodes():
+ processNode(child, dataSetCode)
+ else:
+ fileAsString = IOUtils.readLines(node.getInputStream()).toString()
+ fileName = node.getName()
+
+ if fileName.endswith(".txt"):
+ mailService.createEmailSender().\
+ withSubject("processed text file " + fileName).\
+ withBody("see the attached file").\
+ withAttachedText(fileAsString, fileName).\
+ send()
+ else:
+ filePath = node.getFile().getPath()
+ mailService.createEmailSender().\
+ withSubject("processed file " + fileName).\
+ withBody("see the attached file").\
+ withAttachedFile(filePath, fileName).\
+ send()
\ No newline at end of file
diff --git a/datastore_server/resource/examples/data-set-processing.py b/datastore_server/resource/examples/data-set-processing.py
new file mode 100644
index 00000000000..6d362162f1a
--- /dev/null
+++ b/datastore_server/resource/examples/data-set-processing.py
@@ -0,0 +1,15 @@
+import org.apache.commons.io.IOUtils as IOUtils
+
+def process(dataSet):
+ dataSetCode = dataSet.getDataSetCode()
+ print "script processing " + dataSetCode
+ processNode(dataSet.getContent().getRootNode(), dataSet.getDataSetCode())
+
+def processNode(node, dataSetCode):
+ print "process node: " + dataSetCode + "/" + node.getRelativePath()
+ if node.isDirectory():
+ for child in node.getChildNodes():
+ processNode(child, dataSetCode)
+ else:
+ print "content (" + str(node.getFileLength()) + "): " + \
+ IOUtils.readLines(node.getInputStream()).toString()
\ No newline at end of file
diff --git a/datastore_server/resource/examples/data-set-reporting-content.py b/datastore_server/resource/examples/data-set-reporting-content.py
new file mode 100644
index 00000000000..2623cd93242
--- /dev/null
+++ b/datastore_server/resource/examples/data-set-reporting-content.py
@@ -0,0 +1,31 @@
+import java.util.Date as Date
+
+CODE = "Code"
+FILE_NAME = "File Name"
+RELATIVE_PATH = "Relative Path"
+LAST_MODIFIED = "Last Modified"
+SIZE = "Size"
+
+def describe(dataSets, tableBuilder):
+ tableBuilder.addHeader(CODE)
+ tableBuilder.addHeader(FILE_NAME)
+ tableBuilder.addHeader(RELATIVE_PATH)
+ tableBuilder.addHeader(LAST_MODIFIED)
+ tableBuilder.addHeader(SIZE)
+ for dataSet in dataSets:
+ print "script reporting " + dataSet.getDataSetCode()
+ describeNode(dataSet.getContent().getRootNode(), dataSet.getDataSetCode(), tableBuilder)
+
+
+def describeNode(node, dataSetCode, tableBuilder):
+ print "describe node: " + dataSetCode + "/" + node.getRelativePath()
+ if node.isDirectory():
+ for child in node.getChildNodes():
+ describeNode(child, dataSetCode, tableBuilder)
+ else:
+ row = tableBuilder.addRow()
+ row.setCell(CODE, dataSetCode)
+ row.setCell(FILE_NAME, node.getName())
+ row.setCell(RELATIVE_PATH, node.getRelativePath())
+ row.setCell(LAST_MODIFIED, Date(node.getLastModified()))
+ row.setCell(SIZE, node.getFileLength())
\ No newline at end of file
diff --git a/datastore_server/resource/examples/data-set-reporting-search.py b/datastore_server/resource/examples/data-set-reporting-search.py
new file mode 100644
index 00000000000..d174e6329be
--- /dev/null
+++ b/datastore_server/resource/examples/data-set-reporting-search.py
@@ -0,0 +1,23 @@
+CODE = "Data Set Code"
+EXPERIMENT_IDENTIFIER = "Experiment Identifier"
+EXPERIMENT_TYPE = "Experiment Type"
+EXPERIMENT_DESCRIPTION = "Description"
+
+def describe(dataSets, tableBuilder):
+
+ tableBuilder.addHeader(CODE)
+ tableBuilder.addHeader(EXPERIMENT_IDENTIFIER)
+ tableBuilder.addHeader(EXPERIMENT_TYPE)
+ tableBuilder.addHeader(EXPERIMENT_DESCRIPTION)
+
+ for dataSet in dataSets:
+ projectIdentifier = "/" + dataSet.getSpaceCode() + "/" + dataSet.getProjectCode()
+ print "script reporting " + dataSet.getDataSetCode() + " from " + projectIdentifier
+ experiments = searchService.listExperiments(projectIdentifier)
+
+ for experiment in experiments:
+ row = tableBuilder.addRow()
+ row.setCell(CODE, dataSet.getDataSetCode())
+ row.setCell(EXPERIMENT_IDENTIFIER, experiment.getExperimentIdentifier())
+ row.setCell(EXPERIMENT_TYPE, experiment.getExperimentType())
+ row.setCell(EXPERIMENT_DESCRIPTION, experiment.getPropertyValue("DESCRIPTION"))
\ No newline at end of file
diff --git a/datastore_server/resource/examples/data-set-reporting.py b/datastore_server/resource/examples/data-set-reporting.py
new file mode 100644
index 00000000000..31e484dc9a9
--- /dev/null
+++ b/datastore_server/resource/examples/data-set-reporting.py
@@ -0,0 +1,56 @@
+CODE = "Code"
+TYPE = "Type"
+SIZE = "Size"
+LOCATION = "Location"
+SPEED_HINT = "Speed Hint"
+MAIN_PATTERN = "Main Data Set Pattern"
+MAIN_PATH = "Main Data Set Path"
+INSTANCE = "Instance"
+SPACE = "Space"
+PROJECT = "Project"
+EXPERIMENT_CODE = "Experiment Code"
+EXPERIMENT_IDENTIFIER = "Experiment Identifier"
+EXPERIMENT_TYPE = "Experiment Type"
+SAMPLE_CODE = "Sample Code"
+SAMPLE_IDENTIFIER = "Sample Identifier"
+SAMPLE_TYPE = "Sample Type"
+
+def describe(dataSets, tableBuilder):
+
+ tableBuilder.addHeader(CODE)
+ tableBuilder.addHeader(TYPE)
+ tableBuilder.addHeader(SIZE)
+ tableBuilder.addHeader(LOCATION)
+ tableBuilder.addHeader(SPEED_HINT)
+ tableBuilder.addHeader(MAIN_PATTERN)
+ tableBuilder.addHeader(MAIN_PATH)
+ tableBuilder.addHeader(INSTANCE)
+ tableBuilder.addHeader(SPACE)
+ tableBuilder.addHeader(PROJECT)
+ tableBuilder.addHeader(EXPERIMENT_CODE)
+ tableBuilder.addHeader(EXPERIMENT_IDENTIFIER)
+ tableBuilder.addHeader(EXPERIMENT_TYPE)
+ tableBuilder.addHeader(SAMPLE_CODE)
+ tableBuilder.addHeader(SAMPLE_IDENTIFIER)
+ tableBuilder.addHeader(SAMPLE_TYPE)
+
+ for dataSet in dataSets:
+ print "script reporting " + dataSet.getDataSetCode()
+
+ row = tableBuilder.addRow()
+ row.setCell(CODE, dataSet.getDataSetCode())
+ row.setCell(TYPE, dataSet.getDataSetTypeCode())
+ row.setCell(SIZE, dataSet.getDataSetSize())
+ row.setCell(LOCATION, dataSet.getDataSetLocation())
+ row.setCell(SPEED_HINT, dataSet.getSpeedHint())
+ row.setCell(MAIN_PATTERN, dataSet.getMainDataSetPattern())
+ row.setCell(MAIN_PATH, dataSet.getMainDataSetPath())
+ row.setCell(INSTANCE, dataSet.getInstanceCode())
+ row.setCell(SPACE, dataSet.getSpaceCode())
+ row.setCell(PROJECT, dataSet.getProjectCode())
+ row.setCell(EXPERIMENT_CODE, dataSet.getExperimentCode())
+ row.setCell(EXPERIMENT_IDENTIFIER, dataSet.getExperimentIdentifier())
+ row.setCell(EXPERIMENT_TYPE, dataSet.getExperimentTypeCode())
+ row.setCell(SAMPLE_CODE, dataSet.getSampleCode())
+ row.setCell(SAMPLE_IDENTIFIER, dataSet.getSampleIdentifier())
+ row.setCell(SAMPLE_TYPE, dataSet.getSampleTypeCode())
\ No newline at end of file
diff --git a/datastore_server/resource/examples/data-set-validator.py b/datastore_server/resource/examples/data-set-validator.py
new file mode 100644
index 00000000000..a6459dad162
--- /dev/null
+++ b/datastore_server/resource/examples/data-set-validator.py
@@ -0,0 +1,18 @@
+import os
+
+def validate_data_set_file(file):
+
+ errors = []
+
+ if file.isFile():
+ errors.append(createFileValidationError("'" + file.getPath() + "' is a normal file. Expected a directory."))
+ else:
+ for f in file.listFiles():
+ if f.getName().startswith('file'):
+ pf = open(f.getPath(), "r")
+ if pf.read().find('invalid') > -1:
+ errors.append(createFileValidationError("File '" + f.getPath() + "' contains invalid text."))
+ else:
+ errors.append(createFileValidationError("File '" + f.getPath() + "' doesn't match file name pattern 'file*'."))
+
+ return errors
\ No newline at end of file
--
GitLab