yeastx: update doc, add a script to generate test data

SVN: 15863

rtd_yeastx/resource/examples/config/generate-test-data/generate-datasets.sh

+#!/bin/bash
+# Quick and dirty script ot generate test data for yeastX metabol database.
+# In your openBIS database there has to be a sample with the code 'TEST_SAMPLE' inside the 'TEST' space.
+# The sample has to be connected to any experiment.
+# After running the script copy the content of the 'test-datasets' directory to the:
+     targets/yeastx/incoming/
+
+# how many datasets should be created per one upload dir
+FILES_NUM=3
+# how many upload directories should be generated
+DIRS_NUM=3
+
+dir_num=1; 
+while [ $dir_num -le $DIRS_NUM ]; do 
+  DIR=test-datasets/t$dir_num
+
+  rm -rf $DIR
+  mkdir -p $DIR
+  cat header.tsv > $DIR/index.tsv; 
+  i=1; 
+  while [ $i -le $FILES_NUM ]; do 
+	l=`cat line-eicml.tsv`; 
+	# Uncomment this line and comment the one above to generate fiaML datasets 
+	#l=`cat line-fiaml.tsv`; 
+	echo $i$l | tr " " "\t" >> $DIR/index.tsv; 
+	cp x.eicML $DIR/$i.eicML; 
+	# Uncomment this line and comment the one above to generate fiaML datasets 
+	#cp x.fiaML $DIR/$i.fiaML; 
+	let i+=1; 
+  done
+
+  let dir_num+=1
+done
\ No newline at end of file

rtd_yeastx/resource/examples/config/generate-test-data/header.tsv

+# your-email@gmail.com
+file_name	sample	group	datasetcomments

rtd_yeastx/resource/examples/config/generate-test-data/line-eicml.tsv

+.eicML	TEST_SAMPLE	TEST	upload

rtd_yeastx/resource/examples/config/generate-test-data/line-fiaml.tsv

+.fiaML	TEST_SAMPLE	TEST	upload

rtd_yeastx/resource/examples/config/readme.txt

 This folder contains files needed to test yeatsx database locally.
 
 To test yeastx you should:
-- copy service.properties to 'etc' directory in datastore_server project
-- copy 'targets/yeastx to the 'targets' directory in datastore_server project
+- copy 'targets/yeastx to the 'targets' directory in rtd_yeastx project
 - restore 'metabol_dev' database from metabol_dev.sql file
 - unpack 'yeastx-store.tgz' to the 'targets/yeastx' directory in datastore_server project
-- use openbis_yeastx_db_test.sql as an openbis database
\ No newline at end of file
+- use openbis_yeastx_db_test.sql as an openbis database 
+-- restore the database
+-- change service.properties in openbis project to use the restored database
+- launch openBIS web client and 'yeastx Datastore Server'.
+
+ If you need more test data, see generate-test-data/generate-datasets.sh script.
\ No newline at end of file

rtd_yeastx/resource/examples/config/service.properties

-# Unique code of this Data Store Server. Not more than 40 characters.
-data-store-server-code = DSS1
-
-root = targets/yeastx
-
-# The root directory of the data store
-storeroot-dir = ${root}/store
-
-# The directory where the command queue file is located; defaults to storeroot-dir 
-commandqueue-dir = ${root}
-
-# Port
-port = 8444
-
-# Session timeout in minutes
-session-timeout = 30
-
-# Path to the keystore
-keystore.path = dist/etc/openBIS.keystore
-
-use-ssl = false
-
-# Password of the keystore
-keystore.password = changeit
-
-# Key password of the keystore
-keystore.key-password = changeit
-
-# The check interval (in seconds)
-check-interval = 2
-
-# The time-out for clean up work in the shutdown sequence (in seconds).
-# Note that that the maximal time for the shutdown sequence to complete can be as large 
-# as twice this time.
-shutdown-timeout = 2
-
-#�If free disk space goes below value defined here, a notification email will be sent.
-# Value must be specified in kilobytes (1048576 = 1024 * 1024 = 1GB). If no high water mark is
-# specified or if value is negative, the system will not be watching.
-highwater-mark = 1048576
-
-# The URL of the openBIS server
-server-url = http://localhost:8888/openbis
-
-# The username to use when contacting the openBIS server
-username = etlserver_yeastx
-
-# The password to use when contacting the openBIS server
-password = doesnotmatter
-
-# The base URL for Web client access.
-download-url = http://localhost:8444
-
-# SMTP properties (must start with 'mail' to be considered).
-mail.smtp.host = file://${root}
-mail.from = datastore_server@localhost
-mail.smtp.user = 
-mail.smtp.password = 
-
-data-sources = metabol-db
-metabol-db.version-holder-class = ch.systemsx.cisd.yeastx.db.MetabolDatabaseVersionHolder
-metabol-db.databaseEngineCode = postgresql
-metabol-db.basicDatabaseName = metabol
-metabol-db.databaseKind = dev
-metabol-db.readOnlyGroup = metabol_readonly
-metabol-db.readWriteGroup = metabol_readwrite
-metabol-db.scriptFolder = sql
-
-# ---------------------------------------------------------------------------
-
-#maintenance-plugins=ds-remover
-
-ds-remover.class = ch.systemsx.cisd.yeastx.etl.MetabolDatabaseUpdater
-ds-remover.interval = 10
-ds-remover.data-source = metabol-db
-
-# ---------------------------------------------------------------------------
-
-# list of additional web servlets which will be exposed
-plugin-services = chromatogram-image-download-servlet
-# class of the web servlet
-chromatogram-image-download-servlet.class = ch.systemsx.cisd.openbis.dss.yeastx.server.EICMLChromatogramGeneratorServlet
-# URL which will be mapped to this servlet
-chromatogram-image-download-servlet.path = /datastore_server/chromatogram
-chromatogram-image-download-servlet.data-source = metabol-db
-
-# ------------------------
-
-# Comma separated names of reporting plugins. Each plugin should have configuration properties prefixed with its name.
-reporting-plugins = eicml-chromatograms-reporter, eicml-runs-reporter, eicml-chromatogram-images-reporter
-
-# Label of the plugin which will be shown for the users.
-eicml-chromatograms-reporter.label = Show eicML chromatograms
-# Comma separated list of dataset type codes which can be handled by this plugin.
-eicml-chromatograms-reporter.dataset-types = EICML
-# Plugin class specification (together with the list of packages this class belongs to).
-eicml-chromatograms-reporter.class = ch.systemsx.cisd.yeastx.eicml.EICMLChromatogramsReporter
-# The property file. Its content will be passed as a parameter to the plugin.
-eicml-chromatograms-reporter.properties-file =
-eicml-chromatograms-reporter.data-source = metabol-db 
-
-# Label of the plugin which will be shown for the users.
-eicml-runs-reporter.label = Show eicML runs
-# Comma separated list of dataset type codes which can be handled by this plugin.
-eicml-runs-reporter.dataset-types = EICML
-# Plugin class specification (together with the list of packages this class belongs to).
-eicml-runs-reporter.class = ch.systemsx.cisd.yeastx.eicml.EICMLRunsReporter
-# The property file. Its content will be passed as a parameter to the plugin.
-eicml-runs-reporter.properties-file = 
-eicml-runs-reporter.data-source = metabol-db 
-
-# Label of the plugin which will be shown for the users.
-eicml-chromatogram-images-reporter.label = Show eicML chromatogram images
-# Comma separated list of dataset type codes which can be handled by this plugin.
-eicml-chromatogram-images-reporter.dataset-types = EICML
-# Plugin class specification (together with the list of packages this class belongs to).
-eicml-chromatogram-images-reporter.class = ch.systemsx.cisd.yeastx.eicml.EICMLChromatogramImagesReporter
-# The property file. Its content will be passed as a parameter to the plugin.
-eicml-chromatogram-images-reporter.properties-file = 
-eicml-chromatogram-images-reporter.data-source = metabol-db 
-
-# ------------------------
-
-# The period of no write access that needs to pass before an incoming data item is considered 
-# complete and ready to be processed (in seconds) [default: 300]. 
-# Valid only when auto-detection method is used to determine if an incoming data are ready to be processed.
-quiet-period = 3
-
-# ------------------------
-
-inputs=main-thread, eicml-uploader, fiaml-uploader, quantml-uploader
-
-# Globally used separator character which separates entities in a data set file name 
-data-set-file-name-entity-separator = .
-
-# ---------------------------------------------------------------------------
-# 'main-thread' thread configuration
-# ---------------------------------------------------------------------------
-
-# The directory to watch for incoming data.
-main-thread.incoming-dir = ${root}/incoming
-
-# Determines when the incoming data should be considered complete and ready to be processed.
-# Allowed values: 
-#  - auto-detection - when no write access will be detected for a specified 'quite-period'
-#  - marker-file		- when an appropriate marker file for the data exists. 
-# The default value is 'marker-file'.
-main-thread.incoming-data-completeness-condition = auto-detection
-main-thread.incoming-dir.format = 
-main-thread.reprocess-faulty-datasets = true
-
-sample-name-property-code = samplename
-experiment-name-property-code = name
-file-name-property-code = file_name
-
-gain-write-access-script = ../datastore_server_yeastx/takeCifsOwnershipRecursive.sh
-
-main-thread.dataset-handler = ch.systemsx.cisd.yeastx.etl.BatchDataSetHandler
-main-thread.dataset-handler.preprocessing-script = ${gain-write-access-script}
-main-thread.dataset-handler.preprocessing-script-max-retries = 4
-main-thread.dataset-handler.preprocessing-script-failure-interval = 3
-main-thread.dataset-handler.unique-sample-name-property-code = ${sample-name-property-code}
-main-thread.dataset-handler.unique-experiment-name-property-code = ${experiment-name-property-code}
-
-# The extractor class to use for code extraction
-main-thread.data-set-info-extractor = ch.systemsx.cisd.yeastx.etl.BatchDataSetInfoExtractor
-main-thread.data-set-info-extractor.file-name-property-code = ${file-name-property-code}
-main-thread.data-set-info-extractor.unique-sample-name-property-code = ${sample-name-property-code}
-main-thread.data-set-info-extractor.unique-experiment-name-property-code = ${experiment-name-property-code}
-
-# The extractor class to use for type extraction
-main-thread.type-extractor = ch.systemsx.cisd.yeastx.etl.TypeExtractorYeastX
-# comma separated list of pairs: file-extension file-type
-# It is assumed that for each file extension a dataset type with the same name is defined in openBIS.
-# The corresponding file types have to be defined in openBIS as well.
-# Files with unspecified extensions will have the file type and dataset type UNKNOWN in openBIS. 
-main-thread.type-extractor.file-types = pdf pdf, mat matlab, zip archive, eicml xml, fiaml xml, mzxml xml
-
-# The storage processor (IStorageProcessor implementation)
-main-thread.storage-processor = ch.systemsx.cisd.yeastx.etl.StorageProcessorWithDropboxes
-main-thread.storage-processor.eicml-dropbox-dir = ${root}/dropbox-eicml
-main-thread.storage-processor.fiaml-dropbox-dir = ${root}/dropbox-fiaml
-main-thread.storage-processor.entity-separator = .
-main-thread.storage-processor.processor = ch.systemsx.cisd.yeastx.etl.StorageProcessorWithMLUploader
-main-thread.storage-processor.processor.unique-sample-name-property-code = ${sample-name-property-code}
-main-thread.storage-processor.processor.unique-experiment-name-property-code = ${experiment-name-property-code}
-main-thread.storage-processor.processor.data-source = metabol-db
-main-thread.storage-processor.processor.processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor
-
-# ---------------------------------------------------------------------------
-# plugin properties for the thread which uploads eicML files
-# ---------------------------------------------------------------------------
-
-# The directory to watch for incoming data.
-eicml-uploader.incoming-dir = ${root}/incoming-eicml
-eicml-uploader.incoming-data-completeness-condition = auto-detection
-eicml-uploader.incoming-dir.format = 
-eicml-uploader.reprocess-faulty-datasets = false
-
-# The extractor class to use for code extraction
-eicml-uploader.data-set-info-extractor = ch.systemsx.cisd.yeastx.etl.DataSetInfoExtractorYeastX
-eicml-uploader.data-set-info-extractor.file-name-property-code = ${file-name-property-code}
-eicml-uploader.data-set-info-extractor.index-of-experiment-identifier = 0
-eicml-uploader.data-set-info-extractor.index-of-parent-data-set-codes = 1
-eicml-uploader.data-set-info-extractor.entity-separator = ${data-set-file-name-entity-separator}
-
-# The extractor class to use for type extraction
-eicml-uploader.type-extractor = ch.systemsx.cisd.etlserver.SimpleTypeExtractor
-eicml-uploader.type-extractor.file-format-type = XML
-eicml-uploader.type-extractor.locator-type = RELATIVE_LOCATION
-eicml-uploader.type-extractor.data-set-type = EICML
-eicml-uploader.type-extractor.is-measured = false
-
-# The storage processor which uploads the content of the files to the additional database
-eicml-uploader.storage-processor = ch.systemsx.cisd.yeastx.etl.StorageProcessorWithMLUploader
-eicml-uploader.storage-processor.unique-sample-name-property-code = ${sample-name-property-code}
-eicml-uploader.storage-processor.unique-experiment-name-property-code = ${experiment-name-property-code}
-eicml-uploader.storage-processor.data-source = metabol-db
-eicml-uploader.storage-processor.processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor
-
-# ---------------- plugin properties for the thread which uploads fiaML files
-
-# The directory to watch for incoming data.
-fiaml-uploader.incoming-dir = ${root}/incoming-fiaml
-fiaml-uploader.incoming-data-completeness-condition = auto-detection
-fiaml-uploader.incoming-dir.format = 
-fiaml-uploader.reprocess-faulty-datasets = false
-
-# The extractor class to use for code extraction
-fiaml-uploader.data-set-info-extractor = ch.systemsx.cisd.yeastx.etl.DataSetInfoExtractorYeastX
-fiaml-uploader.data-set-info-extractor.file-name-property-code = ${file-name-property-code}
-fiaml-uploader.data-set-info-extractor.index-of-experiment-identifier = 0
-fiaml-uploader.data-set-info-extractor.index-of-parent-data-set-codes = 1
-fiaml-uploader.data-set-info-extractor.entity-separator = ${data-set-file-name-entity-separator}
-
-# The extractor class to use for type extraction
-fiaml-uploader.type-extractor = ch.systemsx.cisd.etlserver.SimpleTypeExtractor
-fiaml-uploader.type-extractor.file-format-type = XML
-fiaml-uploader.type-extractor.locator-type = RELATIVE_LOCATION
-fiaml-uploader.type-extractor.data-set-type = FIAML
-fiaml-uploader.type-extractor.is-measured = false
-
-# The storage processor which uploads the content of the files to the additional database
-fiaml-uploader.storage-processor = ch.systemsx.cisd.yeastx.etl.StorageProcessorWithMLUploader
-fiaml-uploader.storage-processor.unique-sample-name-property-code = ${sample-name-property-code}
-fiaml-uploader.storage-processor.unique-experiment-name-property-code = ${experiment-name-property-code}
-fiaml-uploader.storage-processor.data-source = metabol-db
-fiaml-uploader.storage-processor.processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor
-
-# ---------------- plugin properties for the thread which uploads quantML files
-
-# The directory to watch for incoming data.
-quantml-uploader.incoming-dir = ${root}/incoming-quantml
-quantml-uploader.incoming-data-completeness-condition = auto-detection
-quantml-uploader.incoming-dir.format = 
-quantml-uploader.reprocess-faulty-datasets = false
-
-# The extractor class to use for code extraction
-quantml-uploader.data-set-info-extractor = ch.systemsx.cisd.yeastx.etl.QuantMLDataSetInfoExtractor
-quantml-uploader.data-set-info-extractor.index-of-experiment-identifier = 0
-quantml-uploader.data-set-info-extractor.sub-entity-separator = &
-# unused but required to be different than sub-entity-separator
-quantml-uploader.data-set-info-extractor.entity-separator = .
-
-# The extractor class to use for type extraction
-quantml-uploader.type-extractor = ch.systemsx.cisd.etlserver.SimpleTypeExtractor
-quantml-uploader.type-extractor.file-format-type = XML
-quantml-uploader.type-extractor.locator-type = RELATIVE_LOCATION
-quantml-uploader.type-extractor.data-set-type = UNKNOWN
-quantml-uploader.type-extractor.is-measured = false
-
-# The storage processor which uploads the content of the files to the additional database
-quantml-uploader.storage-processor = ch.systemsx.cisd.yeastx.etl.QuantMLStorageProcessor
-quantml-uploader.storage-processor.preprocessing-script = ${gain-write-access-script}
-quantml-uploader.storage-processor.unique-sample-name-property-code = ${sample-name-property-code}
-quantml-uploader.storage-processor.unique-experiment-name-property-code = ${experiment-name-property-code}
-quantml-uploader.storage-processor.data-source = metabol-db
-quantml-uploader.storage-processor.processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor
-
-archiver.class = ch.systemsx.cisd.yeastx.etl.MLArchiverTask
-archiver.unique-sample-name-property-code = ${sample-name-property-code}
-archiver.unique-experiment-name-property-code = ${experiment-name-property-code}
-archiver.data-source = metabol-db
\ No newline at end of file