LMS-1180 YeastX: new integration tests for quantification model

SVN: 12775

integration-tests/templates/data-yeastx/incoming-quantml/TEST&TEST_PROJECT&EXP_TEST.quantML

+<?xml version="1.0" encoding="utf-8"?>
+<quantML>
+   <msQuantification>
+      <source>msSoft</source>
+      <valid>true</valid>
+      <concentration>
+         <datasetParent>20090925182754736-36</datasetParent>
+         <amount>0.689155</amount>
+         <unit/>
+         <comment/>
+         <valid>true</valid>
+         <retentionTime>939.5</retentionTime>
+         <Q1>664.1</Q1>
+         <Q3>408</Q3>
+         <internalStandard/>
+         <identity number="1">
+            <cmpdId>23806</cmpdId>
+            <cmpdId>23807</cmpdId>
+         </identity>
+      </concentration>
+      <concentration>
+         <datasetParent>20090925182754736-36</datasetParent>
+         <amount>3.775339</amount>
+         <unit/>
+         <comment/>
+         <valid>true</valid>
+         <retentionTime>922.2</retentionTime>
+         <Q1>664.1</Q1>
+         <Q3>408</Q3>
+         <internalStandard/>
+         <identity number="1">
+            <cmpdId>23806</cmpdId>
+         </identity>
+      </concentration>
+   </msQuantification>
+</quantML>
\ No newline at end of file

integration-tests/templates/datastore_server_yeastx/etc/service.properties

@@ -96,7 +96,7 @@ dataset-deletion-synchronizer.interval = 60
 
 # ------------------------
 
-inputs=main-thread, eicml-uploader, fiaml-uploader
+inputs=main-thread, eicml-uploader, fiaml-uploader, quantml-uploader
 
 # Globally used separator character which separates entities in a data set file name 
 data-set-file-name-entity-separator = .
@@ -225,4 +225,34 @@ fiaml-uploader.storage-processor.unique-experiment-name-property-code = ${experi
 fiaml-uploader.storage-processor.database.databaseEngineCode = ${yeastx-databaseEngineCode}
 fiaml-uploader.storage-processor.database.basicDatabaseName = ${yeastx-basicDatabaseName}
 fiaml-uploader.storage-processor.database.databaseKind = ${yeastx-databaseKind}
-fiaml-uploader.storage-processor.processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor
+fiaml-uploader.storage-processor.processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor
\ No newline at end of file
+
+# ---------------- plugin properties for the thread which uploads quantML files
+
+# The directory to watch for incoming data.
+quantml-uploader.incoming-dir = ${root}/incoming-quantml
+quantml-uploader.incoming-data-completeness-condition = auto-detection
+quantml-uploader.incoming-dir.format = 
+
+# The extractor class to use for code extraction
+quantml-uploader.data-set-info-extractor = ch.systemsx.cisd.yeastx.quant.QuantMLDataSetInfoExtractor
+quantml-uploader.data-set-info-extractor.index-of-experiment-identifier = 0
+quantml-uploader.data-set-info-extractor.sub-entity-separator = &
+# unused but required to be different than sub-entity-separator
+quantml-uploader.data-set-info-extractor.entity-separator = .
+
+# The extractor class to use for type extraction
+quantml-uploader.type-extractor = ch.systemsx.cisd.etlserver.SimpleTypeExtractor
+quantml-uploader.type-extractor.file-format-type = XML
+quantml-uploader.type-extractor.locator-type = RELATIVE_LOCATION
+quantml-uploader.type-extractor.data-set-type = UNKNOWN
+quantml-uploader.type-extractor.is-measured = false
+
+# The storage processor which uploads the content of the files to the additional database
+quantml-uploader.storage-processor = ch.systemsx.cisd.yeastx.etl.StorageProcessorWithMLUploader
+quantml-uploader.storage-processor.unique-sample-name-property-code = ${sample-name-property-code}
+quantml-uploader.storage-processor.unique-experiment-name-property-code = ${experiment-name-property-code}
+quantml-uploader.storage-processor.database.databaseEngineCode = ${yeastx-databaseEngineCode}
+quantml-uploader.storage-processor.database.basicDatabaseName = ${yeastx-basicDatabaseName}
+quantml-uploader.storage-processor.database.databaseKind = ${yeastx-databaseKind}
+quantml-uploader.storage-processor.processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor
\ No newline at end of file

integration-tests/test-yeastx.sh

@@ -15,21 +15,6 @@ METABOL_DB=metabol_dev
 
 # --------------------
 
-# Changes last-modification-time so that the order is the same as the alphabetical order.
-function change_modification_time_alphabetically {
-	local dir=$1
-
-  local file
-  # modification data format: YYYYMMDDhhmm
-  local modification_time=200811221122
-  for file in `ls -1 $dir | sort`; do
-  	local full_path=$dir/$file
-  	# change the modification date format
-  	touch -t $modification_time $full_path
-  	modification_time=$(($modification_time+1))
-  done
-}
-
 # Prepare template incoming data and some destination data structures
 function prepare_data {
 		# Prepare empty incoming data
@@ -40,8 +25,6 @@ function prepare_data {
     clean_svn $MY_DATA
 
     chmod -R 700 $MY_DATA/incoming*
-	  # DSS processes files ordered by modification time, so in this way we make the tests more predictable.
-		change_modification_time_alphabetically $MY_DATA/incoming
 }
 
 function build_and_install_components {
@@ -151,16 +134,22 @@ function count_db_table_records {
 function assert_correct_datasets_metabol_database {
     local eicms_runs=`count_db_table_records $METABOL_DB eic_ms_runs`
     local fiams_runs=`count_db_table_records $METABOL_DB fia_ms_runs`
-
+    local quantms_runs=`count_db_table_records $METABOL_DB ms_quantifications`
+    local quantms_concentrations=`count_db_table_records $METABOL_DB ms_quant_concentrations`
+    local quantms_component_ids=`count_db_table_records $METABOL_DB ms_quant_compounds`
+    
 		# one run comes from the incoming and one from incoming-*ml
     assert_equals "Wrong number of eic MS runs in the metablomics db" 2 $eicms_runs  
     assert_equals "Wrong number of fia MS runs in the metablomics db" 2 $fiams_runs
+    assert_equals "Wrong number of quantifications in the metablomics db" 1 $quantms_runs
+    assert_equals "Wrong number of quant. concentrations in the metablomics db" 2 $quantms_concentrations
+    assert_equals "Wrong number of quant. component ids in the metablomics db" 3 $quantms_component_ids
 }
 
 function assert_correct_incoming_contents {
 	local incoming_dir=$1
 
-	# check content of incoming directory
+	# check content of incoming directory for faulty or ignored datasets
   assert_files_number $incoming_dir 12
 	assert_correct_incoming_content faulty-duplicated-mapping 4 true
 	assert_correct_incoming_content faulty-experiment-code 4 true
@@ -180,8 +169,9 @@ function assert_correct_incoming_contents {
 	# check content of dropboxes
 	assert_files_number "$MY_DATA/dropbox-eicml/TEST&TEST_PROJECT&EXP_TEST.*.mzXML" 6
 	assert_files_number "$MY_DATA/dropbox-fiaml/TEST&TEST_PROJECT&EXP_TEST.*.mzXML" 2
-
-	local registered_datasets=16
+	assert_dir_empty $MY_DATA/incoming-quantml
+	
+	local registered_datasets=17
 	# check content of the store	
 	local store=$MY_DATA/store
 	local store_files_count=`find $store -type f | wc -l`
@@ -200,7 +190,7 @@ function assert_correct_incoming_contents {
 	#   id;experiment_code;data_store_code;code;is_placeholder;data_id_parent;is_complete;data_producer_code;production_timestamp
 	local pattern="[0-9]*;EXP_TEST;DSS1;20[0-9]*-[0-9]*;[ft];[0-9]*;[TFU]*;;.*"
 	local i=2; 
-	while [ $i -lt 18 ]; do 
+	while [ $i -lt $(($registered_datasets+2)) ]; do 
 		assert_correct_dataset_content_in_database $i $pattern
 		i=$(( $i +1))
 	done