diff --git a/integration-tests/run.sh b/integration-tests/run.sh index a58b4503b8f5fad738547799d8076ca52a5fbf6b..716b60288e9618c3e6f6f656286dd8c8d3cf4de8 100755 --- a/integration-tests/run.sh +++ b/integration-tests/run.sh @@ -242,10 +242,12 @@ function install_dsss { local install_dss=$1 if [ $install_dss == "true" ]; then unpack datastore_server - prepare datastore_server datastore_server-all + prepare datastore_server datastore_server1 + prepare datastore_server datastore_server2 remove_unpacked datastore_server else - copy_templates datastore_server-all + copy_templates datastore_server1 + copy_templates datastore_server2 fi } @@ -538,7 +540,8 @@ function switch_dmv { function switch_processing_pipeline { new_state=$1 - switch_dss $new_state datastore_server-all + switch_dss $new_state datastore_server1 + switch_dss $new_state datastore_server2 switch_dmv $new_state datamover-analysis switch_sth $new_state dummy-img-analyser start.sh stop.sh $TRUE switch_dmv $new_state datamover-raw diff --git a/integration-tests/templates/data/incoming-a/.gitignore b/integration-tests/templates/data/incoming-a/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/integration-tests/templates/datastore_server1/datastore_server.sh b/integration-tests/templates/datastore_server1/datastore_server.sh new file mode 100755 index 0000000000000000000000000000000000000000..c69f1dc109ed10d7f2e549aa14787a57c45ff75d --- /dev/null +++ b/integration-tests/templates/datastore_server1/datastore_server.sh @@ -0,0 +1,3 @@ +#! /bin/sh +./shutdown.sh +java -ea -Djavax.net.ssl.trustStore=etc/openBIS.keystore -jar lib/datastore_server.jar "$@" & echo $! > running.pid \ No newline at end of file diff --git a/integration-tests/templates/datastore_server1/etc/openBIS.keystore b/integration-tests/templates/datastore_server1/etc/openBIS.keystore new file mode 100644 index 0000000000000000000000000000000000000000..3982a68319e9e1fed5ace51f72682d79e448eaa2 Binary files /dev/null and b/integration-tests/templates/datastore_server1/etc/openBIS.keystore differ diff --git a/integration-tests/templates/datastore_server1/etc/service.properties b/integration-tests/templates/datastore_server1/etc/service.properties new file mode 100644 index 0000000000000000000000000000000000000000..434c11170050d49d591655453079b953d3a840ba --- /dev/null +++ b/integration-tests/templates/datastore_server1/etc/service.properties @@ -0,0 +1,134 @@ +# Unique code of this Data Store Server. Not more than 40 characters. +data-store-server-code = DSS1 + +# The root directory of the data store +storeroot-dir = ../data/main-store + +# Port +port = 8444 + +# Session timeout in minutes +session-timeout = 30 + +# Path to the keystore +keystore.path = etc/openBIS.keystore + +# Password of the keystore +keystore.password = changeit + +# Key password of the keystore +keystore.key-password = changeit + +# The check interval (in seconds) +check-interval = 2 + +# The time-out for clean up work in the shutdown sequence (in seconds). +# Note that that the maximal time for the shutdown sequence to complete can be as large +# as twice this time. +shutdown-timeout = 2 + +# If free disk space goes below value defined here, a notification email will be sent. +# Value must be specified in kilobytes (1048576 = 1024 * 1024 = 1GB). If no high water mark is +# specified or if value is negative, the system will not be watching. +highwater-mark = 1048576 + +# The URL of the LIMS server +server-url = https://localhost:8443/openbis/openbis + +# The username to use when contacting the LIMS server +username = etlserver + +# The password to use when contacting the LIMS server +password = <change this> + +# The base URL for Web client access. +download-url = https://localhost:8444 + +# SMTP properties (must start with 'mail' to be considered). +mail.smtp.host = mail.ethz.ch +# mail.from = datastore_server@localhost +# mail.smtp.user = +# mail.smtp.password = + +# Maximum number of retries if renaming failed. +# renaming.failure.max-retries = 12 + +# The number of milliseconds to wait before retrying to execute the renaming process. +# renaming.failure.millis-to-sleep = 5000 + +# Globally used separator character which separates entities in a data set file name +data-set-file-name-entity-separator = _ + +processors = DATA_ACQUISITION +processor.DATA_ACQUISITION.prefix-for-relative-paths = ../data/ +processor.DATA_ACQUISITION.data-set-code-prefix-glue = ${data-set-file-name-entity-separator} +processor.DATA_ACQUISITION.finished-file-template = .MARKER_openbis_is_finished_{0} +processor.DATA_ACQUISITION.parameters-file = processing-parameters-from-openbis +# Can be one of PROPRIETARY (the data as acquired from the measurement device) +# or BDS_DIRECTORY (the data in BDS format in a directory container). +processor.DATA_ACQUISITION.input-storage-format = PROPRIETARY + +# Comma separated names of processing threads. Each thread should have configuration properties prefixed with its name +# E.g. 'code-extractor' property for the thread 'my-etl' should be specified as 'my-etl.code-extractor' +inputs=raw-data,image-analysis + +# --------------------------------------------------------------------------- +# Raw data thread +# --------------------------------------------------------------------------- +# The directory to watch for incoming data. +raw-data.incoming-dir = ../data/out-raw + +# The extractor plugin class to use for data set information extraction +raw-data.data-set-info-extractor = ch.systemsx.cisd.etlserver.threev.DataSetInfoExtractorForDataAcquisition +# Separator used to extract the barcode in the data set file name +raw-data.data-set-info-extractor.entity-separator = ${data-set-file-name-entity-separator} +raw-data.data-set-info-extractor.group-code = CISD +raw-data.data-set-info-extractor.index-of-data-producer-code = 0 +raw-data.data-set-info-extractor.index-of-data-production-date = 1 +raw-data.data-set-info-extractor.data-production-date-format = yyyyMMddHHmm +raw-data.data-set-info-extractor.indices-of-data-set-code-entities = 0 -1 +raw-data.data-set-info-extractor.data-set-code-entities-glue = - +# ........................................................................... + +# The extractor plugin class to use for type extraction +raw-data.type-extractor = ch.systemsx.cisd.etlserver.SimpleTypeExtractor +raw-data.type-extractor.file-format-type = TIFF +raw-data.type-extractor.locator-type = RELATIVE_LOCATION +raw-data.type-extractor.data-set-type = HCS_IMAGE +raw-data.type-extractor.processor-type = DATA_ACQUISITION + +# The storage processor plugin (IStorageProcessor implementation) +# raw-data.storage-processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor +raw-data.storage-processor = ch.systemsx.cisd.etlserver.BDSStorageProcessor +raw-data.storage-processor.version = 1.1 +raw-data.storage-processor.sampleTypeCode = CELL_PLATE +raw-data.storage-processor.sampleTypeDescription = Screening Plate +raw-data.storage-processor.format = HCS_IMAGE V1.0 +raw-data.storage-processor.number_of_channels = 2 +raw-data.storage-processor.contains_original_data = TRUE +raw-data.storage-processor.well_geometry = 3x3 +raw-data.storage-processor.file-extractor = ch.systemsx.cisd.etlserver.imsb.HCSImageFileExtractor + +# --------------------------------------------------------------------------- +# image-analysis thread +# --------------------------------------------------------------------------- +image-analysis.incoming-dir = ../data/out-analysis + +image-analysis.data-set-info-extractor = ch.systemsx.cisd.etlserver.threev.DataSetInfoExtractorForImageAnalysis +# Separator used to extract the barcode in the data set file name +image-analysis.data-set-info-extractor.entity-separator = ${data-set-file-name-entity-separator} +image-analysis.data-set-info-extractor.group-code = CISD +image-analysis.data-set-info-extractor.indices-of-parent-data-set-code-entities = 0 -1 +image-analysis.data-set-info-extractor.data-set-code-entities-glue = - +# ........................................................................... + +# The extractor class to use for type extraction +image-analysis.type-extractor = ch.systemsx.cisd.etlserver.SimpleTypeExtractor +image-analysis.type-extractor.file-format-type = 3VPROPRIETARY +image-analysis.type-extractor.locator-type = RELATIVE_LOCATION +image-analysis.type-extractor.data-set-type = HCS_IMAGE_ANALYSIS_DATA +image-analysis.type-extractor.is-measured = false + +# The storage processor (IStorageProcessor implementation) +image-analysis.storage-processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor + diff --git a/integration-tests/templates/datastore_server1/shutdown.sh b/integration-tests/templates/datastore_server1/shutdown.sh new file mode 100755 index 0000000000000000000000000000000000000000..2e5771bca66e45d176d99d9f5f83f7c519ed08e2 --- /dev/null +++ b/integration-tests/templates/datastore_server1/shutdown.sh @@ -0,0 +1,48 @@ +#! /bin/sh +# This is an exact copy of datamover.stop + +awkBin() +{ + # We need a awk that accepts variable assignments with '-v' + case `uname -s` in + "SunOS") + echo "nawk" + return + ;; + esac + # default + echo "awk" +} + +isPIDRunning() +{ + if [ "$1" = "" ]; then + return 0 + fi + # This will have a return value of 0 on BSDish systems + isBSD="`ps aux > /dev/null 2>&1; echo $?`" + AWK=`awkBin` + if [ "$isBSD" = "0" ]; then + if [ "`ps aux | $AWK -v PID=$1 '{if ($2==PID) {print "FOUND"}}'`" = "FOUND" ]; then + return 0 + else + return 1 + fi + else + if [ "`ps -ef | $AWK -v PID=$1 '{if ($2==PID) {print "FOUND"}}'`" = "FOUND" ]; then + return 0 + else + return 1 + fi + fi +} + +if [ -f running.pid ]; then + PID=`cat running.pid` + isPIDRunning $PID + if [ $? -eq 0 ]; then + kill $PID + echo "Previously running program with PID $PID was found and killed" + fi + rm running.pid +fi diff --git a/integration-tests/templates/datastore_server2/datastore_server.sh b/integration-tests/templates/datastore_server2/datastore_server.sh new file mode 100755 index 0000000000000000000000000000000000000000..c69f1dc109ed10d7f2e549aa14787a57c45ff75d --- /dev/null +++ b/integration-tests/templates/datastore_server2/datastore_server.sh @@ -0,0 +1,3 @@ +#! /bin/sh +./shutdown.sh +java -ea -Djavax.net.ssl.trustStore=etc/openBIS.keystore -jar lib/datastore_server.jar "$@" & echo $! > running.pid \ No newline at end of file diff --git a/integration-tests/templates/datastore_server2/etc/openBIS.keystore b/integration-tests/templates/datastore_server2/etc/openBIS.keystore new file mode 100644 index 0000000000000000000000000000000000000000..3982a68319e9e1fed5ace51f72682d79e448eaa2 Binary files /dev/null and b/integration-tests/templates/datastore_server2/etc/openBIS.keystore differ diff --git a/integration-tests/templates/datastore_server2/etc/service.properties b/integration-tests/templates/datastore_server2/etc/service.properties new file mode 100644 index 0000000000000000000000000000000000000000..652218d23eb46fd6a59542cd2fd91f8d1f63d370 --- /dev/null +++ b/integration-tests/templates/datastore_server2/etc/service.properties @@ -0,0 +1,85 @@ +# Unique code of this Data Store Server. Not more than 40 characters. +data-store-server-code = DSS2 + +# The root directory of the data store +storeroot-dir = ../data/main-store + +# Port +port = 8445 + +# Session timeout in minutes +session-timeout = 30 + +# Path to the keystore +keystore.path = etc/openBIS.keystore + +# Password of the keystore +keystore.password = changeit + +# Key password of the keystore +keystore.key-password = changeit + +# The check interval (in seconds) +check-interval = 2 + +# The time-out for clean up work in the shutdown sequence (in seconds). +# Note that that the maximal time for the shutdown sequence to complete can be as large +# as twice this time. +shutdown-timeout = 2 + +# If free disk space goes below value defined here, a notification email will be sent. +# Value must be specified in kilobytes (1048576 = 1024 * 1024 = 1GB). If no high water mark is +# specified or if value is negative, the system will not be watching. +highwater-mark = 1048576 + +# The URL of the LIMS server +server-url = https://localhost:8443/openbis/openbis + +# The username to use when contacting the LIMS server +username = etlserver + +# The password to use when contacting the LIMS server +password = <change this> + +# The base URL for Web client access. +download-url = https://localhost:${port} + +# SMTP properties (must start with 'mail' to be considered). +mail.smtp.host = mail.ethz.ch +# mail.from = datastore_server@localhost +# mail.smtp.user = +# mail.smtp.password = + +# Maximum number of retries if renaming failed. +# renaming.failure.max-retries = 12 + +# The number of milliseconds to wait before retrying to execute the renaming process. +# renaming.failure.millis-to-sleep = 5000 + +# The period of no write access that needs to pass before an incoming data item is considered +# complete and ready to be processed (in seconds) [default: 300]. +# Valid only when auto-detection method is used to determine if an incoming data are ready to be processed. +quiet-period = 10 + +# Globally used separator character which separates entities in a data set file name +data-set-file-name-entity-separator = _ + +# Comma separated names of processing threads. Each thread should have configuration properties prefixed with its name +# E.g. 'code-extractor' property for the thread 'my-etl' should be specified as 'my-etl.code-extractor' +inputs=a + +# The directory to watch for incoming data. +a.incoming-dir = ../data/incoming-a +a.incoming-data-completeness-condition = auto-detection + +# The extractor plugin class to use for data set information extraction +a.data-set-info-extractor = ch.systemsx.cisd.etlserver.DefaultDataSetInfoExtractor +# Separator used to extract the barcode in the data set file name +a.data-set-info-extractor.entity-separator = ${data-set-file-name-entity-separator} +a.data-set-info-extractor.group-code = CISD +a.type-extractor = ch.systemsx.cisd.etlserver.SimpleTypeExtractor +a.type-extractor.file-format-type = TIFF +a.type-extractor.locator-type = RELATIVE_LOCATION +a.type-extractor.data-set-type = HCS_IMAGE +a.storage-processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor + diff --git a/integration-tests/templates/datastore_server2/shutdown.sh b/integration-tests/templates/datastore_server2/shutdown.sh new file mode 100755 index 0000000000000000000000000000000000000000..2e5771bca66e45d176d99d9f5f83f7c519ed08e2 --- /dev/null +++ b/integration-tests/templates/datastore_server2/shutdown.sh @@ -0,0 +1,48 @@ +#! /bin/sh +# This is an exact copy of datamover.stop + +awkBin() +{ + # We need a awk that accepts variable assignments with '-v' + case `uname -s` in + "SunOS") + echo "nawk" + return + ;; + esac + # default + echo "awk" +} + +isPIDRunning() +{ + if [ "$1" = "" ]; then + return 0 + fi + # This will have a return value of 0 on BSDish systems + isBSD="`ps aux > /dev/null 2>&1; echo $?`" + AWK=`awkBin` + if [ "$isBSD" = "0" ]; then + if [ "`ps aux | $AWK -v PID=$1 '{if ($2==PID) {print "FOUND"}}'`" = "FOUND" ]; then + return 0 + else + return 1 + fi + else + if [ "`ps -ef | $AWK -v PID=$1 '{if ($2==PID) {print "FOUND"}}'`" = "FOUND" ]; then + return 0 + else + return 1 + fi + fi +} + +if [ -f running.pid ]; then + PID=`cat running.pid` + isPIDRunning $PID + if [ $? -eq 0 ]; then + kill $PID + echo "Previously running program with PID $PID was found and killed" + fi + rm running.pid +fi