LMS-211 multiple ETL server threads in one instance, integration tests

SVN: 3277

integration-tests/run.sh

@@ -217,8 +217,7 @@ function install_lims_client {
 # unpack everything, override default configuration with test configuation	
 function install_etls {
     unpack etlserver
-    prepare etlserver etlserver-raw
-    prepare etlserver etlserver-analys
+    prepare etlserver etlserver-all
     remove_unpacked etlserver
 }
 
@@ -378,7 +377,7 @@ function create_test_data_dir {
     local DIR=$2
     mkdir $DIR/$NAME
     local i=0  
-    while [  $i -lt 5 ]; do
+    while [  $i -lt 15 ]; do
 	create_test_data_file $DIR/$NAME/$NAME-data$i.txt
 	let i=i+1 
     done
@@ -438,8 +437,7 @@ function switch_dmv {
 
 function switch_processing_pipeline {
     new_state=$1
-    switch_etl $new_state etlserver-analys
-    switch_etl $new_state etlserver-raw 
+    switch_etl $new_state etlserver-all
     switch_dmv $new_state datamover-analys
     switch_sth $new_state dummy-img-analyser start.sh stop.sh
     switch_dmv $new_state datamover-raw
@@ -472,9 +470,11 @@ function assert_correct_results {
     assert_dir_empty $DATA/in-analys
     assert_dir_empty $DATA/out-analys
     assert_dir_empty $DATA/analys-copy
-    imgAnalys="$DATA/store-analys/Project_NEMO/Experiment_EXP1/ObservableType_IMAGE_ANALYSIS_DATA/Barcode_3VCP1/1"
+    
+    local store_dir=$DATA/main-store
+    local imgAnalys="$store_dir/Project_NEMO/Experiment_EXP1/ObservableType_IMAGE_ANALYSIS_DATA/Barcode_3VCP1/1"
     assert_dir_exists "$imgAnalys"
-    rawData="$DATA/store-raw/3V/Project_NEMO/Experiment_EXP1/ObservableType_IMAGE/Barcode_3VCP1/1"
+    local rawData="$store_dir/3V/Project_NEMO/Experiment_EXP1/ObservableType_IMAGE/Barcode_3VCP1/1"
     assert_dir_exists "$rawData"
 }
 

integration-tests/templates/etlserver-raw/etc/service.properties

 # The root directory of the data store
-storeroot-dir = ../data/store-raw
-
-# The directory to watch for incoming data.
-incoming-dir = ../data/out-raw
+storeroot-dir = ../data/main-store
 
 # The check interval (in seconds)
 check-interval = 2
@@ -11,7 +8,7 @@ check-interval = 2
 server-url = https://localhost:8443/openbis/services/etl-openbis
 
 # The username to use when contacting the LIMS server
-username = etlserver2
+username = etlserver
 
 # The password to use when contacting the LIMS server
 password = <change this>
@@ -29,30 +26,62 @@ mail.smtp.host = mail.ethz.ch
 # The number of milliseconds to wait before retrying to execute the renaming process.
 # renaming.failure.millis-to-sleep = 5000
 
+# Comma separated names of processing threads. Each thread should have configuration properties prefixed with its name
+# E.g. 'code-extractor' property for the thread 'my-etl' should be specified as 'my-etl.code-extractor'
+inputs=raw-data,image-analysis
+
 # ---------------------------------------------------------------------------
-# Plugin properties
+# Raw data thread
 # ---------------------------------------------------------------------------
+# The directory to watch for incoming data.
+raw-data.incoming-dir = ../data/out-raw
 
-# The extractor class to use for code extraction
-code-extractor = ch.systemsx.cisd.etlserver.threev.CodeExtractor
+# The extractor plugin class to use for code extraction
+raw-data.code-extractor = ch.systemsx.cisd.etlserver.threev.CodeExtractor
 # The separator that separates an experiment code from the data set one
-code-extractor.entity-separator = _
-code-extractor.number-of-prefixes = 2
+raw-data.code-extractor.entity-separator = _
+raw-data.code-extractor.number-of-prefixes = 2
 # The separator that separates project code from experiment code
-code-extractor.project-separator = .
+raw-data.code-extractor.project-separator = .
+# ...........................................................................
+
+# The extractor plugin class to use for type extraction
+raw-data.type-extractor = ch.systemsx.cisd.etlserver.SimpleTypeExtractor
+raw-data.type-extractor.file-format-type = TIFF
+raw-data.type-extractor.locator-type = RELATIVE LOCATION
+raw-data.type-extractor.observable-type = IMAGE
+raw-data.type-extractor.procedure-type = DATA ACQUISITION
+
+# The storage processor plugin (IStorageProcessor implementation)
+# raw-data.storage-processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor
+raw-data.storage-processor = ch.systemsx.cisd.etlserver.BDSStorageProcessor
+raw-data.storage-processor.version = 1.0
+raw-data.storage-processor.measurementEntityTypeDescription = screening plate
+raw-data.storage-processor.processingType = RAW_DATA
+raw-data.storage-processor.format = HCS_IMAGE V1.0
+raw-data.storage-processor.number_of_channels = 1
+raw-data.storage-processor.contains_original_data = true
+raw-data.storage-processor.well_geometry = 3x3
+raw-data.storage-processor.plate_geometry = 16x24
+raw-data.storage-processor.file-extractor = ch.systemsx.cisd.etlserver.threev.HCSImageFileExtractor
+
+# ---------------------------------------------------------------------------
+# image-analysis thread
+# ---------------------------------------------------------------------------
+image-analysis.incoming-dir = ../data/out-analys
+
+image-analysis.code-extractor = ch.systemsx.cisd.etlserver.threev.CodeExtractor
+image-analysis.code-extractor.entity-separator = _
+image-analysis.code-extractor.number-of-prefixes = 2
+image-analysis.code-extractor.project-separator = .
 # ...........................................................................
 
 # The extractor class to use for type extraction
-type-extractor = ch.systemsx.cisd.etlserver.SimpleTypeExtractor
-type-extractor.file-format-type = TIFF
-type-extractor.locator-type = RELATIVE LOCATION
-type-extractor.observable-type = IMAGE
-type-extractor.procedure-type = DATA ACQUISITION
+image-analysis.type-extractor = ch.systemsx.cisd.etlserver.SimpleTypeExtractor
+image-analysis.type-extractor.file-format-type = 3VPROPRIETARY
+image-analysis.type-extractor.locator-type = RELATIVE LOCATION
+image-analysis.type-extractor.observable-type = IMAGE_ANALYSIS_DATA
+image-analysis.type-extractor.procedure-type = IMAGE ANALYSIS
 
 # The storage processor (IStorageProcessor implementation)
-# storage-processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor
-storage-processor = ch.systemsx.cisd.etlserver.BDSStorageProcessor
-storage-processor.version = 1.0
-storage-processor.measurementEntityTypeDescription = screening plate
-storage-processor.processingType = RAW_DATA
-storage-processor.format = UNKNOWN V1.0
+image-analysis.storage-processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor
\ No newline at end of file

integration-tests/templates/openBIS-server/roles.conf

@@ -2,5 +2,5 @@
 poweruser
 
 [Roles]
-etlserver = etlserver1,etlserver2
+etlserver = etlserver
 admin = test
\ No newline at end of file