From 0995ccf6182620cf5ec26317d9bc660048f7c895 Mon Sep 17 00:00:00 2001
From: felmer <felmer>
Date: Wed, 25 Feb 2009 11:14:49 +0000
Subject: [PATCH] LMS-768 rename 'etlserver' project to 'datastore_server'
SVN: 9972
---
datastore_server/.checkstyle | 9 +
datastore_server/.classpath | 23 +
datastore_server/.project | 23 +
datastore_server/build/antrun.sh | 6 +
datastore_server/build/build.xml | 82 ++
.../dist/data/incoming/.gitignore | 0
datastore_server/dist/data/store/.gitignore | 0
datastore_server/dist/etc/etlserver.conf | 18 +
datastore_server/dist/etc/log.xml | 55 ++
datastore_server/dist/etc/openBIS.keystore | Bin 0 -> 1426 bytes
datastore_server/dist/etc/service.properties | 110 +++
datastore_server/dist/etlserver.bat | 3 +
datastore_server/dist/etlserver.sh | 250 +++++
datastore_server/dist/lib/.gitignore | 0
datastore_server/dist/log/.gitignore | 0
datastore_server/etc/log.xml | 19 +
datastore_server/etc/service.properties | 111 +++
.../resource/dependency-structure.ddf | 18 +
.../eclipse/ETL Server Fast Suite.launch | 22 +
.../resource/eclipse/ETL Server.launch | 13 +
.../AbstractDataSetInfoExtractor.java | 57 ++
.../etlserver/AbstractStorageProcessor.java | 65 ++
.../cisd/etlserver/BDSStorageProcessor.java | 598 ++++++++++++
.../cisd/etlserver/BaseDirectoryHolder.java | 68 ++
.../cisd/etlserver/ChannelSetHelper.java | 106 ++
.../cisd/etlserver/DataSetInformation.java | 274 ++++++
.../DataSetNameEntitiesProvider.java | 101 ++
.../cisd/etlserver/DataStoreStrategyKey.java | 50 +
.../cisd/etlserver/DataStrategyStore.java | 192 ++++
.../DefaultDataSetInfoExtractor.java | 246 +++++
.../etlserver/DefaultStorageProcessor.java | 87 ++
.../cisd/etlserver/ETLServerPlugin.java | 67 ++
.../etlserver/EncapsulatedLimsService.java | 211 ++++
.../cisd/etlserver/FileBasedFile.java | 187 ++++
.../cisd/etlserver/FileBasedFileFactory.java | 98 ++
.../systemsx/cisd/etlserver/FileRenamer.java | 83 ++
.../cisd/etlserver/HCSImageCheckList.java | 173 ++++
.../HCSImageFileExtractionResult.java | 82 ++
.../cisd/etlserver/IDataSetInfoExtractor.java | 75 ++
.../cisd/etlserver/IDataStoreStrategy.java | 55 ++
.../cisd/etlserver/IDataStrategyStore.java | 43 +
.../cisd/etlserver/IETLServerPlugin.java | 41 +
.../etlserver/IEncapsulatedLimsService.java | 75 ++
.../ch/systemsx/cisd/etlserver/IFile.java | 58 ++
.../systemsx/cisd/etlserver/IFileFactory.java | 30 +
.../cisd/etlserver/IHCSImageFileAccepter.java | 37 +
.../etlserver/IHCSImageFileExtractor.java | 44 +
.../IProcedureAndDataTypeExtractor.java | 55 ++
.../systemsx/cisd/etlserver/IProcessor.java | 42 +
.../cisd/etlserver/IProcessorFactory.java | 38 +
.../cisd/etlserver/IStorageProcessor.java | 90 ++
.../etlserver/IStoreRootDirectoryHolder.java | 41 +
.../etlserver/IdentifiedDataStrategy.java | 180 ++++
.../java/ch/systemsx/cisd/etlserver/Main.java | 497 ++++++++++
.../cisd/etlserver/NamedDataStrategy.java | 94 ++
.../systemsx/cisd/etlserver/Parameters.java | 403 ++++++++
.../cisd/etlserver/PlateDimension.java | 73 ++
.../cisd/etlserver/PlateDimensionParser.java | 120 +++
.../PropertiesBasedETLServerPlugin.java | 123 +++
.../cisd/etlserver/SimpleTypeExtractor.java | 93 ++
.../cisd/etlserver/StandardProcessor.java | 187 ++++
.../etlserver/StandardProcessorFactory.java | 135 +++
.../cisd/etlserver/ThreadParameters.java | 146 +++
.../etlserver/TransferredDataSetHandler.java | 743 ++++++++++++++
.../etlserver/imsb/HCSImageFileExtractor.java | 335 +++++++
.../AbstractDataSetInfoExtractorFor3V.java | 108 +++
...ataSetInfoExtractorForDataAcquisition.java | 83 ++
.../DataSetInfoExtractorForImageAnalysis.java | 83 ++
.../threev/HCSImageFileExtractor.java | 211 ++++
.../sourceTest/bash/createFakeDataSet.sh | 179 ++++
.../etlserver/BDSStorageProcessorTest.java | 539 +++++++++++
.../cisd/etlserver/ChannelSetHelperTest.java | 68 ++
.../etlserver/CodeExtractortTestCase.java | 44 +
.../DataSetNameEntitiesProviderTest.java | 94 ++
.../cisd/etlserver/DataStrategyStoreTest.java | 261 +++++
.../DefaultDataSetInfoExtractorTest.java | 155 +++
.../DefaultStorageProcessorTest.java | 157 +++
.../EncapsulatedLimsServiceTest.java | 143 +++
.../cisd/etlserver/FileBasedFileTest.java | 87 ++
.../cisd/etlserver/HCSImageCheckListTest.java | 108 +++
.../etlserver/IdentifiedDataStrategyTest.java | 141 +++
.../ch/systemsx/cisd/etlserver/MainTest.java | 136 +++
.../cisd/etlserver/NamedDataStrategyTest.java | 127 +++
.../etlserver/SimpleTypeExtractorTest.java | 68 ++
.../StandardProcessingFactoryTest.java | 127 +++
.../cisd/etlserver/StandardProcessorTest.java | 209 ++++
.../cisd/etlserver/ThreadParametersTest.java | 48 +
.../TransferredDataSetHandlerTest.java | 905 ++++++++++++++++++
.../imsb/HCSImageFileExtractorTest.java | 247 +++++
...etInfoExtractorForDataAcquisitionTest.java | 119 +++
...aSetInfoExtractorForImageAnalysisTest.java | 70 ++
.../threev/HCSImageFileExtractorTest.java | 186 ++++
datastore_server/sourceTest/java/tests.xml | 14 +
.../sourceTest/java/tests_fast.xml | 15 +
94 files changed, 12092 insertions(+)
create mode 100644 datastore_server/.checkstyle
create mode 100644 datastore_server/.classpath
create mode 100644 datastore_server/.project
create mode 100755 datastore_server/build/antrun.sh
create mode 100644 datastore_server/build/build.xml
create mode 100644 datastore_server/dist/data/incoming/.gitignore
create mode 100644 datastore_server/dist/data/store/.gitignore
create mode 100644 datastore_server/dist/etc/etlserver.conf
create mode 100644 datastore_server/dist/etc/log.xml
create mode 100644 datastore_server/dist/etc/openBIS.keystore
create mode 100644 datastore_server/dist/etc/service.properties
create mode 100755 datastore_server/dist/etlserver.bat
create mode 100755 datastore_server/dist/etlserver.sh
create mode 100644 datastore_server/dist/lib/.gitignore
create mode 100644 datastore_server/dist/log/.gitignore
create mode 100644 datastore_server/etc/log.xml
create mode 100644 datastore_server/etc/service.properties
create mode 100644 datastore_server/resource/dependency-structure.ddf
create mode 100644 datastore_server/resource/eclipse/ETL Server Fast Suite.launch
create mode 100644 datastore_server/resource/eclipse/ETL Server.launch
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/AbstractDataSetInfoExtractor.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/AbstractStorageProcessor.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/BDSStorageProcessor.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/BaseDirectoryHolder.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/ChannelSetHelper.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/DataSetInformation.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/DataSetNameEntitiesProvider.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/DataStoreStrategyKey.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/DataStrategyStore.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/DefaultDataSetInfoExtractor.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/DefaultStorageProcessor.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/ETLServerPlugin.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/EncapsulatedLimsService.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/FileBasedFile.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/FileBasedFileFactory.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/FileRenamer.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/HCSImageCheckList.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/HCSImageFileExtractionResult.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/IDataSetInfoExtractor.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/IDataStoreStrategy.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/IDataStrategyStore.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/IETLServerPlugin.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/IEncapsulatedLimsService.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/IFile.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/IFileFactory.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/IHCSImageFileAccepter.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/IHCSImageFileExtractor.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/IProcedureAndDataTypeExtractor.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/IProcessor.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/IProcessorFactory.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/IStorageProcessor.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/IStoreRootDirectoryHolder.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/IdentifiedDataStrategy.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/Main.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/NamedDataStrategy.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/Parameters.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/PlateDimension.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/PlateDimensionParser.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/PropertiesBasedETLServerPlugin.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/SimpleTypeExtractor.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/StandardProcessor.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/StandardProcessorFactory.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/ThreadParameters.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandler.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/imsb/HCSImageFileExtractor.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/threev/AbstractDataSetInfoExtractorFor3V.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/threev/DataSetInfoExtractorForDataAcquisition.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/threev/DataSetInfoExtractorForImageAnalysis.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/threev/HCSImageFileExtractor.java
create mode 100755 datastore_server/sourceTest/bash/createFakeDataSet.sh
create mode 100644 datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/BDSStorageProcessorTest.java
create mode 100644 datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/ChannelSetHelperTest.java
create mode 100644 datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/CodeExtractortTestCase.java
create mode 100644 datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/DataSetNameEntitiesProviderTest.java
create mode 100644 datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/DataStrategyStoreTest.java
create mode 100644 datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/DefaultDataSetInfoExtractorTest.java
create mode 100644 datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/DefaultStorageProcessorTest.java
create mode 100644 datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/EncapsulatedLimsServiceTest.java
create mode 100644 datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/FileBasedFileTest.java
create mode 100644 datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/HCSImageCheckListTest.java
create mode 100644 datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/IdentifiedDataStrategyTest.java
create mode 100644 datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/MainTest.java
create mode 100644 datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/NamedDataStrategyTest.java
create mode 100644 datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/SimpleTypeExtractorTest.java
create mode 100644 datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/StandardProcessingFactoryTest.java
create mode 100644 datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/StandardProcessorTest.java
create mode 100644 datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/ThreadParametersTest.java
create mode 100644 datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandlerTest.java
create mode 100644 datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/imsb/HCSImageFileExtractorTest.java
create mode 100644 datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/threev/DataSetInfoExtractorForDataAcquisitionTest.java
create mode 100644 datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/threev/DataSetInfoExtractorForImageAnalysisTest.java
create mode 100644 datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/threev/HCSImageFileExtractorTest.java
create mode 100644 datastore_server/sourceTest/java/tests.xml
create mode 100644 datastore_server/sourceTest/java/tests_fast.xml
diff --git a/datastore_server/.checkstyle b/datastore_server/.checkstyle
new file mode 100644
index 00000000000..f048a51b530
--- /dev/null
+++ b/datastore_server/.checkstyle
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<fileset-config file-format-version="1.2.0" simple-config="false">
+ <local-check-config name="CISD Checks" location="/build_resources/checkstyle/cisd_checkstyle.xml" type="project" description="">
+ <additional-data name="protect-config-file" value="false"/>
+ </local-check-config>
+ <fileset name="all" enabled="true" check-config-name="CISD Checks" local="true">
+ <file-match-pattern match-pattern=".+ch/systemsx/cisd.+" include-pattern="true"/>
+ </fileset>
+</fileset-config>
diff --git a/datastore_server/.classpath b/datastore_server/.classpath
new file mode 100644
index 00000000000..105dc732c45
--- /dev/null
+++ b/datastore_server/.classpath
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+ <classpathentry kind="src" path="source/java"/>
+ <classpathentry kind="src" path="sourceTest/java"/>
+ <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+ <classpathentry combineaccessrules="false" kind="src" path="/common"/>
+ <classpathentry kind="lib" path="/libraries/log4j/log4j.jar" sourcepath="/libraries/log4j/src.zip"/>
+ <classpathentry kind="lib" path="/libraries/commons-lang/commons-lang.jar" sourcepath="/libraries/commons-lang/src.zip"/>
+ <classpathentry kind="lib" path="/libraries/commons-io/commons-io.jar" sourcepath="/libraries/commons-io/src.zip"/>
+ <classpathentry kind="lib" path="/libraries/testng/testng-jdk15.jar" sourcepath="/libraries/testng/src.zip"/>
+ <classpathentry kind="lib" path="/libraries/mail/mail.jar"/>
+ <classpathentry kind="lib" path="/libraries/jmock/jmock.jar"/>
+ <classpathentry combineaccessrules="false" kind="src" path="/bds"/>
+ <classpathentry kind="lib" path="/libraries/restrictionchecker/restrictions.jar"/>
+ <classpathentry combineaccessrules="false" kind="src" path="/args4j"/>
+ <classpathentry kind="lib" path="/libraries/cglib/cglib-nodep.jar"/>
+ <classpathentry kind="lib" path="/libraries/jmock/hamcrest/hamcrest-core.jar"/>
+ <classpathentry kind="lib" path="/libraries/jmock/hamcrest/hamcrest-library.jar"/>
+ <classpathentry kind="lib" path="/libraries/jmock/objenesis/objenesis-1.0.jar"/>
+ <classpathentry combineaccessrules="false" kind="src" path="/openbis"/>
+ <classpathentry combineaccessrules="false" kind="src" path="/server-common"/>
+ <classpathentry kind="output" path="targets/classes"/>
+</classpath>
diff --git a/datastore_server/.project b/datastore_server/.project
new file mode 100644
index 00000000000..834c47cfcc7
--- /dev/null
+++ b/datastore_server/.project
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+ <name>datastore_server</name>
+ <comment></comment>
+ <projects>
+ </projects>
+ <buildSpec>
+ <buildCommand>
+ <name>org.eclipse.jdt.core.javabuilder</name>
+ <arguments>
+ </arguments>
+ </buildCommand>
+ <buildCommand>
+ <name>com.atlassw.tools.eclipse.checkstyle.CheckstyleBuilder</name>
+ <arguments>
+ </arguments>
+ </buildCommand>
+ </buildSpec>
+ <natures>
+ <nature>org.eclipse.jdt.core.javanature</nature>
+ <nature>com.atlassw.tools.eclipse.checkstyle.CheckstyleNature</nature>
+ </natures>
+</projectDescription>
diff --git a/datastore_server/build/antrun.sh b/datastore_server/build/antrun.sh
new file mode 100755
index 00000000000..5ff1a62b259
--- /dev/null
+++ b/datastore_server/build/antrun.sh
@@ -0,0 +1,6 @@
+#! /bin/bash
+
+ME="$0"
+MYDIR=${ME%/*}
+cd $MYDIR
+ant -lib ../../build_resources/lib/ecj.jar "$@"
diff --git a/datastore_server/build/build.xml b/datastore_server/build/build.xml
new file mode 100644
index 00000000000..6d5817155d4
--- /dev/null
+++ b/datastore_server/build/build.xml
@@ -0,0 +1,82 @@
+<project name="etlserver" default="dist" basedir="..">
+ <import file="../../build_resources/ant/build-common.xml" />
+ <project-classpath name="ecp" classes="${classes}" />
+
+ <property name="original.dist" value="dist" />
+ <property name="mainfolder" value="etlserver" />
+ <property name="dist.etlserver" value="${dist}/${mainfolder}" />
+ <property name="dist.etlserver.lib" value="${dist.etlserver}/lib" />
+ <property name="jar.file" value="${dist.etlserver.lib}/etlserver.jar" />
+ <property name="dist.file.prefix" value="${dist}/etlserver" />
+ <property name="nativesrc" value="${lib}/unix/native" />
+ <property name="nativeroot" value="${targets}/ant" />
+ <property name="native" value="${nativeroot}/native" />
+
+ <target name="clean">
+ <delete dir="${dist}" />
+ </target>
+
+ <target name="compile" depends="build-common.compile, clean" />
+
+ <target name="run-tests">
+ <antcall target="build-common.run-tests">
+ <param name="test.suite" value="tests_fast.xml" />
+ </antcall>
+ </target>
+
+ <target name="jar" depends="compile">
+ <mkdir dir="${dist.etlserver.lib}" />
+ <build-info revision="revision.number" version="version.number" clean="clean.flag" />
+ <echo file="${build.info.file}">${version.number}:${revision.number}:${clean.flag}</echo>
+ <copy todir="${native}">
+ <fileset dir="${nativesrc}">
+ <include name="**/unix.so" />
+ </fileset>
+ </copy>
+ <recursive-jar destfile="${jar.file}">
+ <fileset dir="${classes}">
+ <include name="**/*.class" />
+ <include name="${build.info.filename}" />
+ </fileset>
+ <fileset dir="${nativeroot}">
+ <include name="**/unix.so"/>
+ </fileset>
+ <manifest>
+ <attribute name="Main-Class" value="ch.systemsx.cisd.etlserver.Main" />
+ <attribute name="Class-Path"
+ value="etlserver-plugins.jar log4j.jar activation.jar mail.jar spring.jar fast-md5.jar
+ commons-codec.jar commons-lang.jar commons-io.jar commons-logging.jar commons-httpclient.jar" />
+ <attribute name="Version" value="${version.number}" />
+ <attribute name="Build-Number"
+ value="${version.number} (r${revision.number},${clean.flag})" />
+ </manifest>
+ </recursive-jar>
+ </target>
+
+ <target name="dist" depends="jar">
+ <copy file="${lib}/activation/activation.jar" todir="${dist.etlserver.lib}" />
+ <copy file="${lib}/mail/mail.jar" todir="${dist.etlserver.lib}" />
+ <copy file="${lib}/log4j/log4j.jar" todir="${dist.etlserver.lib}" />
+ <copy file="${lib}/commons-codec/commons-codec.jar" todir="${dist.etlserver.lib}" />
+ <copy file="${lib}/commons-io/commons-io.jar" todir="${dist.etlserver.lib}" />
+ <copy file="${lib}/commons-lang/commons-lang.jar" todir="${dist.etlserver.lib}" />
+ <copy file="${lib}/commons-logging/commons-logging.jar" todir="${dist.etlserver.lib}" />
+ <copy file="${lib}/commons-httpclient/commons-httpclient.jar"
+ todir="${dist.etlserver.lib}" />
+ <copy file="${lib}/spring/spring.jar" todir="${dist.etlserver.lib}" />
+ <copy file="${lib}/fast-md5/fast-md5.jar" todir="${dist.etlserver.lib}" />
+ <property name="dist.file"
+ value="${dist.file.prefix}-${version.number}-r${revision.number}.zip" />
+ <zip basedir="${dist}" destfile="${dist.file}">
+ <zipfileset dir="${original.dist}" excludes="**/etlserver.sh" prefix="${mainfolder}" />
+ <zipfileset file="${original.dist}/etlserver.sh"
+ filemode="755"
+ prefix="${mainfolder}" />
+ </zip>
+ <delete dir="${dist.etlserver}" />
+ </target>
+
+ <target name="ci" depends="run-tests, check-dependencies, dist">
+ </target>
+
+</project>
\ No newline at end of file
diff --git a/datastore_server/dist/data/incoming/.gitignore b/datastore_server/dist/data/incoming/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/datastore_server/dist/data/store/.gitignore b/datastore_server/dist/data/store/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/datastore_server/dist/etc/etlserver.conf b/datastore_server/dist/etc/etlserver.conf
new file mode 100644
index 00000000000..f51846de0bd
--- /dev/null
+++ b/datastore_server/dist/etc/etlserver.conf
@@ -0,0 +1,18 @@
+#
+# ETL Server configuration file
+#
+
+#
+# Home directory of the JRE that should be used
+#
+#JAVA_HOME=${JAVA_HOME:=/usr/java/latest}
+
+#
+# Options to the JRE
+#
+JAVA_OPTS=${JAVA_OPTS:=-server}
+
+#
+# Maximal number of log files to keep
+#
+MAXLOGS=5
\ No newline at end of file
diff --git a/datastore_server/dist/etc/log.xml b/datastore_server/dist/etc/log.xml
new file mode 100644
index 00000000000..41d695d4d78
--- /dev/null
+++ b/datastore_server/dist/etc/log.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+<log4j:configuration xmlns:log4j='http://jakarta.apache.org/log4j/'>
+
+ <appender name="DEFAULT" class="org.apache.log4j.DailyRollingFileAppender">
+
+ <param name="File" value="log/etlserver_log.txt"/>
+ <param name="DatePattern" value="'.'yyyy-MM-dd"/>
+
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p [%t] %c - %m%n"/>
+ </layout>
+
+ </appender>
+
+ <appender name="STDOUT" class="org.apache.log4j.ConsoleAppender">
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p [%t] %c - %m%n"/>
+ </layout>
+ </appender>
+
+ <appender name="NULL" class="org.apache.log4j.varia.NullAppender" />
+
+ <appender name="EMAIL" class="org.apache.log4j.net.SMTPAppender">
+
+ <param name="BufferSize" value="512" />
+ <param name="SMTPHost" value="localhost" />
+ <param name="From" value="etlserver@localhost" />
+ <param name="To" value="root@localhost" />
+ <param name="Subject" value="ATTENTION: etl server" />
+ <param name="EvaluatorClass" value="ch.systemsx.cisd.common.logging.AlwaysTrueTriggeringEventEvaluator" />
+
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p [%t] %c - %m%n"/>
+ </layout>
+
+ <!--filter class="org.apache.log4j.varia.LevelRangeFilter">
+ <param name="LevelMin" value="ERROR"/>
+ <param name="LevelMax" value="FATAL"/>
+ </filter-->
+
+ </appender>
+
+ <category name="NOTIFY">
+ <priority value="info" />
+ <appender-ref ref="EMAIL" />
+ </category>
+
+ <root>
+ <priority value="info" />
+ <appender-ref ref="DEFAULT" />
+ </root>
+
+</log4j:configuration>
diff --git a/datastore_server/dist/etc/openBIS.keystore b/datastore_server/dist/etc/openBIS.keystore
new file mode 100644
index 0000000000000000000000000000000000000000..b727bd0fb777fddb3463c81cb56963a7541f7b45
GIT binary patch
literal 1426
zcmezO_TO6u1_mY|W&~r_+{*0KN+9!5EW=$#pv*3VCZ=r$d~96WY>X_7T1<kBjI0bS
zO-zd#wG~&$%iMm`c71|$Y5$YitEL_J$ZnqF`)W$d-o2eMs`U?M6zI7t-(A`+-cpqw
zHFxi9-Rrrt%0A4!7X5I`p&hRtn0Nl1W1MMabV=o@O##>LX-9cnmv_92vyZY^nksPD
zdgGz_%YMiA%<+G`?zc{_^OTmKsam}GdFt7neK!{P|Bco8Yq|9PvTGT0HqQC!@x9l1
z+kfrLjk$U4PfrzX4>?lEY_X)NU`c_upzVKwLzZV_Z=OB7;%H;~#q=-kB7C15mGAK^
zvQDWpwV1Q=%|)J*vS+h8eD_X&KBLa;y!Ym4fd?y?u6<@yiQRcgW4YSWJ1qI-Q@4qE
za4m1Vxao>O&#^E1)!s>&+m1~AFk#ErRgE9c$R|1RHZn>ST)F2PCX`h;GftH|P3C*s
zWzB^ZGW^U2Io6G9<Cm+dn@b%$@cxkDi?0v*4*&Xhu{qw<UFz0*cKw3~R!=THb>jl>
z6zzz+mrM40Funf&SL@T49sAWnZvUNcc-}kaM`;+(`I*O}e=OTwz2vdB%C^t0KV2dN
z8|5bLIJC=m`3p1S>EdUM&gly9o_)k?62R=&u_P~(KYzPEv+V7u$GBF7Yt*targsEg
zTQzgVqnKmgxtO%H)!0fto_uWX^v1c;QdM?#toT{0*tgvCKBx=r`VraapZ2`+cgggB
z-Pvqw9={ifyv<wt>(DLz=%jkh_cpEkNBn#fQtx!K-V#22ApgPb-K(wNrdO3Et=(zk
zH%r#4Uj9|=lZlRA5;d`xvpUaban7~tb)L?vqO<o@r$c({0U_Q-$DE%pD%yDqwYKp-
ze(+{JQ~%8mo`p*G>(?31t6P_KXXUmI|NTdEW^*XD%W$kc6J6C`ZJL<Zk@PWEN6V*F
zkgcZ1#>_TRc(ux#yKj=Ue_0-Rb!XA^BcIEU<sO}THR1C69iJfyIzrFXz!I2{Cjk?3
zpFtDjcP19gPahu`@Un4gwRyCC=VfGMVP!CA>@^esCUfRc7B*q_(7f!t{PH}Q2nU7;
zJ3>UpKnkRbOIX-9zo<mPDX~()IX|zsG^ZppFWpeVKn^6!EiCR|kea6uoL^d$oT}iG
zT9%rVUyz%cS7In?APiE$EX?DTT2fM}5S&_6mRe*WC(dhNU}#`uWN2(^YGM)v<Qf_n
z8W}*jgU-E;^N~XoSVk~6_5wq-lc}+hVe40umt9-5IP9-{sGrmj5_UQxbn3f#Rhcal
zb$96Tc-UWFAhSV0y0D>s$!bwcyY3l3?6uy+&YLB9AjrXwMJ9=TVxnh^gTTRrgQDAx
z_HixoIbPSC|G?I^w5omSz6+<W?mi^z$(h6X=Z@>C2WR&Q742Af!tZ{F_B0c(^#Lh&
zza|JWF*7nSB0CQl;mkmH*$SFDWnY}}>VU(o3#<5V+huGG`Dn2??1_qdd2)==i--F?
zs#KSqOx?Wrm7jRA(!0Yuu5HVhrWZZce5Lb}5Bq8!uvR=V-JM%7`INM|?JE9h{&PR9
zPS5OYv=7|qeAeX{<0}8%Th2N96wN*x(D1*r!0!3JEkDAd&tKmoUZ_-IpH*z9SS<f#
VnWy8+Ul;a#sHi_<p_OCR1^^CaPjmnP
literal 0
HcmV?d00001
diff --git a/datastore_server/dist/etc/service.properties b/datastore_server/dist/etc/service.properties
new file mode 100644
index 00000000000..3392d3b2f92
--- /dev/null
+++ b/datastore_server/dist/etc/service.properties
@@ -0,0 +1,110 @@
+# The root directory of the data store
+storeroot-dir = data/store
+
+# The check interval (in seconds)
+check-interval = 60
+
+# The time-out for clean up work in the shutdown sequence (in seconds).
+# Note that that the maximal time for the shutdown sequence to complete can be as large
+# as twice this time.
+# Remark: On a network file system, it is not recommended to turn this value to something
+# lower than 180.
+shutdown-timeout = 180
+
+# If free disk space goes below value defined here, a notification email will be sent.
+# Value must be specified in kilobytes (1048576 = 1024 * 1024 = 1GB). If no high water mark is
+# specified or if value is negative, the system will not be watching.
+highwater-mark = 1048576
+
+# If a data set is successfully registered it sends out an email to the registrator.
+# If this property is not specified, no email is sent to the registrator. This property
+# does not affect the mails which are sent, when the data set could not be registered.
+notify-successful-registration = false
+
+# The URL of the openBIS server
+server-url = https://localhost:8443/openbis
+
+# The username to use when contacting the openBIS server
+username = etlserver
+
+# The password to use when contacting the openBIS server
+password = <change this>
+
+# SMTP properties (must start with 'mail' to be considered).
+# mail.smtp.host = localhost
+# mail.from = etlserver@localhost
+
+# ---------------- Timing parameters for file system operations on remote shares.
+
+# Time (in seconds) to wait for any file system operation to finish. Operations exceeding this
+# timeout will be terminated.
+timeout = 60
+# Number of times that a timed out operation will be tried again (0 means: every file system
+# operation will only ever be performed once).
+max-retries = 11
+# Time (in seconds) to wait after an operation has been timed out before re-trying.
+failure-interval = 10
+
+# Globally used separator character which separates entities in a data set file name
+data-set-file-name-entity-separator = _
+
+# Prefixes for processing paths for all procedure types.
+# default-prefix-for-absolute-paths is the key for paths starting with '/'.
+# default-prefix-for-relative-paths is the key for paths not starting with '/'.
+#
+default-prefix-for-absolute-paths =
+
+# Processors of processing instructions.
+#
+# processors: comma separated list of procedure type codes
+# processor.<procedure type code>.prefix-for-absolute-paths: Key for a processing path starting with '/'.
+# processor.<procedure type code>.prefix-for-relative-paths: Key for a processing path not starting with '/'.
+# processor.<procedure type code>.parameters-file: Name of the file containing the processing parameters.
+# processor.<procedure type code>.finished-file-template: Name of the marker file which finishes processing.
+
+processors = DATA_ACQUISITION
+processor.DATA_ACQUISITION.prefix-for-absolute-paths = ${default-prefix-for-absolute-paths}
+processor.DATA_ACQUISITION.prefix-for-relative-paths = targets/processing
+processor.DATA_ACQUISITION.parameters-file = parameters
+processor.DATA_ACQUISITION.data-set-code-prefix-glue = ${data-set-file-name-entity-separator}
+processor.DATA_ACQUISITION.finished-file-template = .MARKER_is_finished_{0}
+# Can be one of PROPRIETARY (the data as acquired from the measurement device)
+# or BDS_DIRECTORY (the data in BDS format in a directory container).
+processor.DATA_ACQUISITION.input-storage-format = PROPRIETARY
+
+# Comma separated names of processing threads. Each thread should have configuration properties prefixed with its name.
+# E.g. 'code-extractor' property for the thread 'my-etl' should be specified as 'my-etl.code-extractor'
+inputs=main-thread
+
+# ---------------------------------------------------------------------------
+# 'main-thread' thread configuration
+# ---------------------------------------------------------------------------
+
+# The directory to watch for incoming data.
+main-thread.incoming-dir = data/incoming
+# The group the samples extracted by this thread belong to. If commented out or empty, then samples
+# are considered associated to a database instance (not group private).
+# main-thread.group-code = <change this>
+
+# ---------------- Plugin properties
+
+# The extractor class to use for code extraction
+main-thread.data-set-info-extractor = ch.systemsx.cisd.etlserver.DefaultDataSetInfoExtractor
+# Separator used to extract the barcode in the data set file name
+main-thread.data-set-info-extractor.entity-separator = ${data-set-file-name-entity-separator}
+
+# The extractor class to use for type extraction
+main-thread.type-extractor = ch.systemsx.cisd.etlserver.SimpleTypeExtractor
+main-thread.type-extractor.file-format-type = TIFF
+main-thread.type-extractor.locator-type = RELATIVE_LOCATION
+main-thread.type-extractor.data-set-type = HCS_IMAGE
+main-thread.type-extractor.procedure-type = DATA_ACQUISITION
+
+# The storage processor (IStorageProcessor implementation)
+main-thread.storage-processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor
+# main-thread.storage-processor = ch.systemsx.cisd.etlserver.BDSStorageProcessor
+# main-thread.storage-processor.version = 1.1
+# main-thread.storage-processor.sampleTypeCode = CELL_PLATE
+# main-thread.storage-processor.sampleTypeDescription = Screening Plate
+# main-thread.storage-processor.format = UNKNOWN V1.0
+
diff --git a/datastore_server/dist/etlserver.bat b/datastore_server/dist/etlserver.bat
new file mode 100755
index 00000000000..5da5824e65f
--- /dev/null
+++ b/datastore_server/dist/etlserver.bat
@@ -0,0 +1,3 @@
+@echo off
+
+java -Djavax.net.ssl.trustStore=etc\openBIS.keystore -jar lib\etlserver.jar %1 %2 %3 %4 %5 %6 %7
diff --git a/datastore_server/dist/etlserver.sh b/datastore_server/dist/etlserver.sh
new file mode 100755
index 00000000000..2b1199085d2
--- /dev/null
+++ b/datastore_server/dist/etlserver.sh
@@ -0,0 +1,250 @@
+#!/bin/bash
+#
+# Control script for CISD openBIS ETL Server on Unix / Linux systems
+# -------------------------------------------------------------------------
+
+awkBin()
+{
+ # We need a awk that accepts variable assignments with '-v'
+ case `uname -s` in
+ "SunOS")
+ echo "nawk"
+ return
+ ;;
+ esac
+ # default
+ echo "awk"
+}
+
+isPIDRunning()
+{
+ if [ "$1" = "" ]; then
+ return 1
+ fi
+ if [ "$1" = "fake" ]; then # for unit tests
+ return 0
+ fi
+ # This will have a return value of 0 on BSDish systems
+ isBSD="`ps aux > /dev/null 2>&1; echo $?`"
+ AWK=`awkBin`
+ if [ "$isBSD" = "0" ]; then
+ if [ "`ps aux | $AWK -v PID=$1 '{if ($2==PID) {print "FOUND"}}'`" = "FOUND" ]; then
+ return 0
+ else
+ return 1
+ fi
+ else
+ if [ "`ps -ef | $AWK -v PID=$1 '{if ($2==PID) {print "FOUND"}}'`" = "FOUND" ]; then
+ return 0
+ else
+ return 1
+ fi
+ fi
+}
+
+rotateLogFiles()
+{
+ logfile=$1
+ max=$2
+ if [ -z "$logfile" ]; then
+ echo "Error: rotateLogFiles: logfile argument missing"
+ return 1
+ fi
+ if [ -z "$max" ]; then
+ echo "Error: rotateLogFiles: max argument missing"
+ return 1
+ fi
+ test -f $logfile.$max && rm $logfile.$max
+ n=$max
+ while [ $n -gt 1 ]; do
+ nnew=$(($n-1))
+ test -f $logfile.$nnew && mv $logfile.$nnew $logfile.$n
+ n=$nnew
+ done
+ test -f $logfile && mv $logfile $logfile.1
+}
+
+getStatus()
+{
+ if [ -f $PIDFILE ]; then
+ PID=`cat $PIDFILE`
+ isPIDRunning $PID
+ if [ $? -eq 0 ]; then
+ return 0
+ else
+ return 1
+ fi
+ else
+ return 2
+ fi
+}
+
+printStatus()
+{
+ if [ -f $PIDFILE ]; then
+ PID=`cat $PIDFILE`
+ isPIDRunning $PID
+ if [ $? -eq 0 ]; then
+ echo "ETL Server is running (pid $PID)"
+ return 0
+ else
+ echo "ETL Server is dead (stale pid $PID)"
+ return 1
+ fi
+ else
+ echo "ETL Server is not running."
+ return 2
+ fi
+}
+
+#
+# definitions
+#
+
+PIDFILE=${ETLSERVER_PID:-etlserver.pid}
+CONFFILE=etc/etlserver.conf
+LOGFILE=log/etlserver_log.txt
+STARTUPLOG=log/startup_log.txt
+SUCCESS_MSG="etlserver ready and waiting for data"
+JAR_FILE=lib/etlserver.jar
+MAX_LOOPS=10
+
+#
+# change to installation directory
+#
+bin=$0
+if [ -L $bin ]; then
+ bin=`dirname $bin`/`readlink $bin`
+fi
+WD=`dirname $bin`
+cd $WD
+SCRIPT=./`basename $0`
+
+#
+# source configuration script, if any
+#
+test -f $CONFFILE && source $CONFFILE
+if [ "$JAVA_HOME" != "" ]; then
+ JAVA_BIN="$JAVA_HOME/bin/java"
+else
+ JAVA_BIN="java"
+fi
+
+command=$1
+ALL_JAVA_OPTS="-Djavax.net.ssl.trustStore=etc/openBIS.keystore $JAVA_OPTS"
+# ensure that we ignore a possible prefix "--" for any command
+command="${command#--*}"
+case "$command" in
+ start)
+ getStatus
+ EXIT_STATUS=$?
+ if [ $EXIT_STATUS -eq 0 ]; then
+ echo "Cannot start ETL Server: already running."
+ exit 100
+ fi
+
+ echo -n "Starting ETL Server "
+ rotateLogFiles $LOGFILE $MAXLOGS
+ shift 1
+ ${JAVA_BIN} ${ALL_JAVA_OPTS} -jar $JAR_FILE "$@" > $STARTUPLOG 2>&1 & echo $! > $PIDFILE
+ if [ $? -eq 0 ]; then
+ # wait for initial self-test to finish
+ n=0
+ while [ $n -lt $MAX_LOOPS ]; do
+ sleep 1
+ if [ ! -f $PIDFILE ]; then
+ break
+ fi
+ if [ -s $STARTUPLOG ]; then
+ PID=`cat $PIDFILE 2> /dev/null`
+ isPIDRunning $PID
+ if [ $? -ne 0 ]; then
+ break
+ fi
+ fi
+ grep "$SUCCESS_MSG" $LOGFILE > /dev/null 2>&1
+ if [ $? -eq 0 ]; then
+ break
+ fi
+ n=$(($n+1))
+ done
+ PID=`cat $PIDFILE 2> /dev/null`
+ isPIDRunning $PID
+ if [ $? -eq 0 ]; then
+ grep "$SUCCESS_MSG" $LOGFILE > /dev/null 2>&1
+ if [ $? -ne 0 ]; then
+ echo "(pid $PID - WARNING: SelfTest not yet finished)"
+ else
+ echo "(pid $PID)"
+ fi
+ else
+ echo "FAILED"
+ if [ -s $STARTUPLOG ]; then
+ echo "startup log says:"
+ cat $STARTUPLOG
+ else
+ echo "log file says:"
+ tail $LOGFILE
+ fi
+ fi
+ else
+ echo "FAILED"
+ fi
+ ;;
+ stop)
+ echo -n "Stopping ETL Server "
+ if [ -f $PIDFILE ]; then
+ PID=`cat $PIDFILE 2> /dev/null`
+ isPIDRunning $PID
+ if [ $? -eq 0 ]; then
+ kill $PID
+ n=0
+ while [ $n -lt $MAX_LOOPS ]; do
+ isPIDRunning $PID
+ if [ $? -ne 0 ]; then
+ break
+ fi
+ sleep 1
+ n=$(($n+1))
+ done
+ isPIDRunning $PID
+ if [ $? -ne 0 ]; then
+ echo "(pid $PID)"
+ test -f $PIDFILE && rm $PIDFILE 2> /dev/null
+ else
+ echo "FAILED"
+ fi
+ else
+ if [ -f $PIDFILE ]; then
+ rm $PIDFILE 2> /dev/null
+ echo "(was dead - cleaned up pid file)"
+ fi
+ fi
+ else
+ echo "(not running - nothing to do)"
+ fi
+ ;;
+ status)
+ printStatus
+ EXIT_STATUS=$?
+ exit $EXIT_STATUS
+ ;;
+ restart)
+ $SCRIPT stop
+ $SCRIPT start
+ ;;
+ help)
+ ${JAVA_BIN} ${ALL_JAVA_OPTS} -jar $JAR_FILE --help
+ ;;
+ version)
+ ${JAVA_BIN} ${ALL_JAVA_OPTS} -jar $JAR_FILE --version
+ ;;
+ show-shredder)
+ ${JAVA_BIN} ${ALL_JAVA_OPTS} -jar $JAR_FILE --show-shredder
+ ;;
+ *)
+ echo $"Usage: $0 {start|stop|restart|status|help|version|show-shredder}"
+ exit 200
+ ;;
+esac
+exit 0
diff --git a/datastore_server/dist/lib/.gitignore b/datastore_server/dist/lib/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/datastore_server/dist/log/.gitignore b/datastore_server/dist/log/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/datastore_server/etc/log.xml b/datastore_server/etc/log.xml
new file mode 100644
index 00000000000..5cee0a68436
--- /dev/null
+++ b/datastore_server/etc/log.xml
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+<log4j:configuration xmlns:log4j='http://jakarta.apache.org/log4j/'>
+
+ <appender name="STDOUT" class="org.apache.log4j.ConsoleAppender">
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p [%t] %c - %m%n"/>
+ </layout>
+ </appender>
+
+ <appender name="NULL" class="org.apache.log4j.varia.NullAppender" />
+
+ <root>
+ <priority value ="info" />
+ <appender-ref ref="STDOUT" />
+ </root>
+
+</log4j:configuration>
diff --git a/datastore_server/etc/service.properties b/datastore_server/etc/service.properties
new file mode 100644
index 00000000000..71b9a000811
--- /dev/null
+++ b/datastore_server/etc/service.properties
@@ -0,0 +1,111 @@
+# The root directory of the data store
+storeroot-dir = targets/store
+
+# The check interval (in seconds)
+check-interval = 5
+
+# The time-out for clean up work in the shutdown sequence (in seconds).
+# Note that that the maximal time for the shutdown sequence to complete can be as large
+# as twice this time.
+shutdown-timeout = 2
+
+# If free disk space goes below value defined here, a notification email will be sent.
+# Value must be specified in kilobytes (1048576 = 1024 * 1024 = 1GB). If no high water mark is
+# specified or if value is negative, the system will not be watching.
+highwater-mark = 1048576
+
+# If a data set is successfully registered it sends out an email to the registrator.
+# If this property is not specified, no email is sent to the registrator. This property
+# does not affect the mails which are sent, when the data set could not be registered.
+notify-successful-registration = false
+
+# The URL of the openBIS server
+server-url = http://localhost:8080/openbis
+
+# The username to use when contacting the openBIS server
+username = etlserver
+
+# The password to use when contacting the openBIS server
+password = doesnotmatter
+
+# SMTP properties (must start with 'mail' to be considered).
+# mail.smtp.host = localhost
+# mail.from = etlserver@localhost
+# mail.smtp.user =
+# mail.smtp.password =
+
+# Maximum number of retries if renaming failed.
+# renaming.failure.max-retries = 12
+
+# The number of milliseconds to wait before retrying to execute the renaming process.
+# renaming.failure.millis-to-sleep = 5000
+
+# Globally used separator character which separates entities in a data set file name
+data-set-file-name-entity-separator = _
+
+# Prefixes for processing paths for all procedure types.
+# default-prefix-for-absolute-paths is the key for paths starting with '/'.
+# default-prefix-for-relative-paths is the key for paths not starting with '/'.
+#
+default-prefix-for-absolute-paths =
+
+# Processors of processing instructions.
+#
+# processors: comma separated list of procedure type codes
+# processor.<procedure type code>.prefix-for-absolute-paths: Key for a processing path starting with '/'.
+# processor.<procedure type code>.prefix-for-relative-paths: Key for a processing path not starting with '/'.
+# processor.<procedure type code>.parameters-file: Name of the file containing the processing parameters.
+# processor.<procedure type code>.finished-file-template: Name of the marker file which finishes processing.
+
+processors = DATA_ACQUISITION
+processor.DATA_ACQUISITION.prefix-for-absolute-paths = ${default-prefix-for-absolute-paths}
+processor.DATA_ACQUISITION.prefix-for-relative-paths = targets/processing
+processor.DATA_ACQUISITION.parameters-file = parameters
+processor.DATA_ACQUISITION.data-set-code-prefix-glue = ${data-set-file-name-entity-separator}
+processor.DATA_ACQUISITION.finished-file-template = .MARKER_is_finished_{0}
+processor.DATA_ACQUISITION.input-storage-format = BDS_DIRECTORY
+# time after which the copy of a single file for processing should complete.
+# If that will not happen, operation will be terminated and relaunched.
+processor.DATA_ACQUISITION.data-copy-timeout = 10
+
+# Comma separated names of processing threads. Each thread should have configuration properties prefixed with its name.
+# E.g. 'code-extractor' property for the thread 'my-etl' should be specified as 'my-etl.code-extractor'
+inputs=main-thread
+
+# ---------------------------------------------------------------------------
+# 'main-thread' thread configuration
+# ---------------------------------------------------------------------------
+
+# The directory to watch for incoming data.
+main-thread.incoming-dir = targets/incoming
+# The group the samples extracted by this thread belong to. If commented out or empty, then samples
+# are considered associated to a database instance (not group private).
+# main-thread.group-code = CISD
+
+# The store format that should be applied in the incoming directory.
+main-thread.incoming-dir.format =
+
+# ---------------- Plugin properties
+
+# The extractor plugin class to use for code extraction
+main-thread.data-set-info-extractor = ch.systemsx.cisd.etlserver.DefaultDataSetInfoExtractor
+# Separator used to extract the barcode in the data set file name
+main-thread.data-set-info-extractor.entity-separator = ${data-set-file-name-entity-separator}
+
+main-thread.type-extractor = ch.systemsx.cisd.etlserver.SimpleTypeExtractor
+main-thread.type-extractor.file-format-type = TIFF
+main-thread.type-extractor.locator-type = RELATIVE_LOCATION
+main-thread.type-extractor.data-set-type = HCS_IMAGE
+main-thread.type-extractor.procedure-type = DATA_ACQUISITION
+
+# The storage processor (IStorageProcessor implementation)
+#main-thread.storage-processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor
+main-thread.storage-processor = ch.systemsx.cisd.etlserver.BDSStorageProcessor
+main-thread.storage-processor.version = 1.1
+main-thread.storage-processor.sampleTypeCode = CELL_PLATE
+main-thread.storage-processor.sampleTypeDescription = Screening Plate
+main-thread.storage-processor.format = HCS_IMAGE V1.0
+main-thread.storage-processor.number_of_channels = 2
+main-thread.storage-processor.contains_original_data = TRUE
+main-thread.storage-processor.well_geometry = 3x3
+main-thread.storage-processor.file-extractor = ch.systemsx.cisd.etlserver.imsb.HCSImageFileExtractor
diff --git a/datastore_server/resource/dependency-structure.ddf b/datastore_server/resource/dependency-structure.ddf
new file mode 100644
index 00000000000..f05d29b3583
--- /dev/null
+++ b/datastore_server/resource/dependency-structure.ddf
@@ -0,0 +1,18 @@
+#
+#
+#
+#show allResults
+
+{root} = ch.systemsx.cisd
+{etlserver} = ${root}.etlserver
+
+######################################################################
+# Check dependencies to openbis
+
+{openbis} = ${root}.openbis
+[etlserver] = ${etlserver}.*
+[private_openbis] = ${openbis}.* excluding ${openbis}.generic.shared.dto.* ${openbis}.generic.shared.IETLLIMSService
+
+check sets [etlserver]
+
+check [etlserver] independentOf [private_openbis]
diff --git a/datastore_server/resource/eclipse/ETL Server Fast Suite.launch b/datastore_server/resource/eclipse/ETL Server Fast Suite.launch
new file mode 100644
index 00000000000..1a5b7c04520
--- /dev/null
+++ b/datastore_server/resource/eclipse/ETL Server Fast Suite.launch
@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<launchConfiguration type="org.testng.eclipse.launchconfig">
+<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
+<listEntry value="/libraries/testng/testng-jdk15.jar"/>
+</listAttribute>
+<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
+<listEntry value="1"/>
+</listAttribute>
+<booleanAttribute key="org.eclipse.debug.core.appendEnvironmentVariables" value="true"/>
+<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="org.testng.remote.RemoteTestNG"/>
+<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="datastore_server"/>
+<mapAttribute key="org.testng.eclipse.ALL_CLASS_METHODS"/>
+<listAttribute key="org.testng.eclipse.CLASS_TEST_LIST"/>
+<stringAttribute key="org.testng.eclipse.COMPLIANCE_LEVEL" value="JDK"/>
+<listAttribute key="org.testng.eclipse.GROUP_LIST"/>
+<listAttribute key="org.testng.eclipse.GROUP_LIST_CLASS"/>
+<stringAttribute key="org.testng.eclipse.LOG_LEVEL" value="2"/>
+<listAttribute key="org.testng.eclipse.SUITE_TEST_LIST">
+<listEntry value="sourceTest/java/tests_fast.xml"/>
+</listAttribute>
+<intAttribute key="org.testng.eclipse.TYPE" value="3"/>
+</launchConfiguration>
diff --git a/datastore_server/resource/eclipse/ETL Server.launch b/datastore_server/resource/eclipse/ETL Server.launch
new file mode 100644
index 00000000000..afdd321a796
--- /dev/null
+++ b/datastore_server/resource/eclipse/ETL Server.launch
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<launchConfiguration type="org.eclipse.jdt.launching.localJavaApplication">
+<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
+<listEntry value="/datastore_server/source/java/ch/systemsx/cisd/etlserver/Main.java"/>
+</listAttribute>
+<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
+<listEntry value="1"/>
+</listAttribute>
+<booleanAttribute key="org.eclipse.debug.core.appendEnvironmentVariables" value="true"/>
+<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="ch.systemsx.cisd.etlserver.Main"/>
+<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="datastore_server"/>
+<stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-ea"/>
+</launchConfiguration>
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/AbstractDataSetInfoExtractor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/AbstractDataSetInfoExtractor.java
new file mode 100644
index 00000000000..d6fcec47156
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/AbstractDataSetInfoExtractor.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.util.Properties;
+
+import ch.systemsx.cisd.common.utilities.ExtendedProperties;
+import ch.systemsx.cisd.common.utilities.PropertyUtils;
+
+/**
+ * An abstract <code>ICodeExtractor</code> implementation.
+ *
+ * @author Christian Ribeaud
+ */
+public abstract class AbstractDataSetInfoExtractor implements IDataSetInfoExtractor
+{
+
+ /** The name of the property to get the experiment separator from. */
+ protected static final String ENTITY_SEPARATOR_PROPERTY_NAME = "entity-separator";
+
+ /** The default entity separator. */
+ protected static final char DEFAULT_ENTITY_SEPARATOR = '.';
+
+ protected static final String STRIP_EXTENSION = "strip-file-extension";
+
+ protected final Properties properties;
+
+ /** Separator character that divides entities in a data set name. */
+ protected final char entitySeparator;
+
+ protected final boolean stripExtension;
+
+ protected AbstractDataSetInfoExtractor(final Properties globalProperties)
+ {
+ assert globalProperties != null : "Global properties can not be null.";
+ properties = ExtendedProperties.getSubset(globalProperties, EXTRACTOR_KEY + '.', true);
+ stripExtension = PropertyUtils.getBoolean(properties, STRIP_EXTENSION, false);
+ entitySeparator =
+ PropertyUtils.getChar(properties, ENTITY_SEPARATOR_PROPERTY_NAME,
+ DEFAULT_ENTITY_SEPARATOR);
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/AbstractStorageProcessor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/AbstractStorageProcessor.java
new file mode 100644
index 00000000000..49eff0618ed
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/AbstractStorageProcessor.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.io.File;
+import java.util.Properties;
+
+import ch.systemsx.cisd.common.utilities.PropertyUtils;
+
+/**
+ * An <code>abtract</code> implementation of <code>IStorageProcessor</code>.
+ *
+ * @author Christian Ribeaud
+ */
+public abstract class AbstractStorageProcessor implements IStorageProcessor
+{
+ protected final Properties properties;
+
+ private File storeRootDir;
+
+ protected AbstractStorageProcessor(final Properties properties)
+ {
+ this.properties = properties;
+ }
+
+ protected final String getMandatoryProperty(final String propertyKey)
+ {
+ return PropertyUtils.getMandatoryProperty(properties, propertyKey);
+ }
+
+ protected static final void checkParameters(final File incomingDataSetPath,
+ final File targetPath)
+ {
+ assert incomingDataSetPath != null : "Given incoming data set path can not be null.";
+ assert targetPath != null : "Given target path can not be null.";
+ }
+
+ //
+ // IStorageProcessor
+ //
+
+ public final File getStoreRootDirectory()
+ {
+ return storeRootDir;
+ }
+
+ public final void setStoreRootDirectory(final File storeRootDirectory)
+ {
+ this.storeRootDir = storeRootDirectory;
+ }
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/BDSStorageProcessor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/BDSStorageProcessor.java
new file mode 100644
index 00000000000..031f801a490
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/BDSStorageProcessor.java
@@ -0,0 +1,598 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.time.DurationFormatUtils;
+import org.apache.log4j.Logger;
+
+import ch.systemsx.cisd.bds.Constants;
+import ch.systemsx.cisd.bds.DataSet;
+import ch.systemsx.cisd.bds.DataStructureFactory;
+import ch.systemsx.cisd.bds.DataStructureLoader;
+import ch.systemsx.cisd.bds.ExperimentRegistrationTimestamp;
+import ch.systemsx.cisd.bds.ExperimentRegistrator;
+import ch.systemsx.cisd.bds.Format;
+import ch.systemsx.cisd.bds.FormatParameter;
+import ch.systemsx.cisd.bds.IFormatParameterFactory;
+import ch.systemsx.cisd.bds.IFormattedData;
+import ch.systemsx.cisd.bds.Reference;
+import ch.systemsx.cisd.bds.ReferenceType;
+import ch.systemsx.cisd.bds.Sample;
+import ch.systemsx.cisd.bds.UnknownFormatV1_0;
+import ch.systemsx.cisd.bds.Version;
+import ch.systemsx.cisd.bds.IDataStructure.Mode;
+import ch.systemsx.cisd.bds.hcs.Geometry;
+import ch.systemsx.cisd.bds.hcs.HCSImageAnnotations;
+import ch.systemsx.cisd.bds.hcs.IHCSImageFormattedData;
+import ch.systemsx.cisd.bds.hcs.Location;
+import ch.systemsx.cisd.bds.hcs.PlateGeometry;
+import ch.systemsx.cisd.bds.hcs.IHCSImageFormattedData.NodePath;
+import ch.systemsx.cisd.bds.storage.IDirectory;
+import ch.systemsx.cisd.bds.storage.IFile;
+import ch.systemsx.cisd.bds.storage.ILink;
+import ch.systemsx.cisd.bds.storage.INode;
+import ch.systemsx.cisd.bds.storage.filesystem.FileStorage;
+import ch.systemsx.cisd.bds.storage.filesystem.NodeFactory;
+import ch.systemsx.cisd.bds.v1_1.ExperimentIdentifierWithUUID;
+import ch.systemsx.cisd.bds.v1_1.IDataStructureV1_1;
+import ch.systemsx.cisd.bds.v1_1.SampleWithOwner;
+import ch.systemsx.cisd.common.collections.CollectionUtils;
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.common.exceptions.UserFailureException;
+import ch.systemsx.cisd.common.filesystem.FileOperations;
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
+import ch.systemsx.cisd.common.filesystem.IFileOperations;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.mail.IMailClient;
+import ch.systemsx.cisd.common.utilities.ClassUtils;
+import ch.systemsx.cisd.etlserver.HCSImageCheckList.FullLocation;
+import ch.systemsx.cisd.openbis.generic.shared.dto.DataSetType;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ExperimentPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.PersonPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.SamplePropertyPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.StorageFormat;
+import ch.systemsx.cisd.openbis.generic.shared.dto.types.SampleTypeCode;
+
+/**
+ * The <code>AbstractStorageAdapter</code> extension for <i>BDS</i> (Biological Data Standards).
+ * <p>
+ * When declared in <code>service.properties</code> file, it must specify a property called
+ * <code>storage-adapter.version</code>. Otherwise instantiation will fail.
+ * </p>
+ *
+ * @author Christian Ribeaud
+ */
+public final class BDSStorageProcessor extends AbstractStorageProcessor implements
+ IHCSImageFileAccepter
+{
+ private static final Logger operationLog =
+ LogFactory.getLogger(LogCategory.OPERATION, BDSStorageProcessor.class);
+
+ private static final Logger notificationLog =
+ LogFactory.getLogger(LogCategory.OPERATION, BDSStorageProcessor.class);
+
+ private static final String PROPERTY_PREFIX = "Property '%s': ";
+
+ private static final String NO_FORMAT_FORMAT =
+ PROPERTY_PREFIX + "no valid and known format could be extracted from text '%s'.";
+
+ static final String VERSION_KEY = "version";
+
+ static final String SAMPLE_TYPE_DESCRIPTION_KEY = "sampleTypeDescription";
+
+ static final String SAMPLE_TYPE_CODE_KEY = "sampleTypeCode";
+
+ static final String FORMAT_KEY = "format";
+
+ static final String FILE_EXTRACTOR_KEY = "file-extractor";
+
+ static final String NO_VERSION_FORMAT =
+ PROPERTY_PREFIX + "no version could be extracted from text '%s'.";
+
+ private final Format format;
+
+ private final SampleTypeCode sampleType;
+
+ private final String sampleTypeDescription;
+
+ private IDataStructureV1_1 dataStructure;
+
+ private File imageFileRootDirectory;
+
+ private File dataStructureDir;
+
+ private IHCSImageFileExtractor imageFileExtractor;
+
+ private List<FormatParameter> formatParameters;
+
+ private IHCSImageFormattedData imageFormattedData;
+
+ private HCSImageCheckList imageCheckList;
+
+ private final Version version;
+
+ public BDSStorageProcessor(final Properties properties)
+ {
+ super(properties);
+ version = parseVersion(getMandatoryProperty(VERSION_KEY));
+ if (version.equals(new Version(1, 1)) == false)
+ {
+ throw new ConfigurationFailureException("Invalid version: " + version);
+ }
+ format = parseFormat(getMandatoryProperty(FORMAT_KEY));
+ createFormatParameters();
+ sampleTypeDescription = getMandatoryProperty(SAMPLE_TYPE_DESCRIPTION_KEY);
+ if (needsImageFileExtractor())
+ {
+ final String property = getMandatoryProperty(FILE_EXTRACTOR_KEY);
+ imageFileExtractor =
+ ClassUtils.create(IHCSImageFileExtractor.class, property, properties);
+ }
+ try
+ {
+ sampleType =
+ SampleTypeCode.getSampleTypeCode(getMandatoryProperty(SAMPLE_TYPE_CODE_KEY));
+ } catch (final IllegalArgumentException ex)
+ {
+ throw new ConfigurationFailureException(ex.getMessage());
+ }
+ }
+
+ private void createFormatParameters()
+ {
+ final List<String> parameterNames = format.getParameterNames();
+ final IFormatParameterFactory formatParameterFactory = format.getFormatParameterFactory();
+ formatParameters = new ArrayList<FormatParameter>();
+ for (final String parameterName : parameterNames)
+ {
+ final String value = properties.getProperty(parameterName);
+ if (value == null)
+ {
+ throw ConfigurationFailureException.fromTemplate(
+ "No value has been defined for parameter '%s'", parameterName);
+ }
+ final FormatParameter formatParameter =
+ formatParameterFactory.createFormatParameter(parameterName, value);
+ if (formatParameter == null)
+ {
+ throw ConfigurationFailureException.fromTemplate(
+ "Given value '%s' is not understandable for parameter '%s'", value,
+ parameterName);
+ }
+ formatParameters.add(formatParameter);
+ }
+ }
+
+ final static Version parseVersion(final String versionString)
+ {
+ final Version version = Version.createVersionFromString(versionString);
+ if (version == null)
+ {
+ throw ConfigurationFailureException.fromTemplate(NO_VERSION_FORMAT, VERSION_KEY,
+ versionString);
+ }
+ return version;
+ }
+
+ final static Format parseFormat(final String formatString)
+ {
+ final Format format = Format.tryToCreateFormatFromString(formatString);
+ if (format == null)
+ {
+ throw ConfigurationFailureException.fromTemplate(NO_FORMAT_FORMAT, FORMAT_KEY,
+ formatString);
+ }
+ return format;
+ }
+
+ private final ExperimentIdentifierWithUUID createExperimentIdentifier(
+ final DataSetInformation dataSetInformation)
+ {
+ final ch.systemsx.cisd.openbis.generic.shared.dto.identifier.ExperimentIdentifier experimentIdentifier =
+ dataSetInformation.getExperimentIdentifier();
+ final String projectCode = experimentIdentifier.getProjectCode();
+ final String experimentCode = experimentIdentifier.getExperimentCode();
+ final String groupCode = experimentIdentifier.getGroupCode();
+ final String instanceCode = dataSetInformation.getInstanceCode();
+ return new ExperimentIdentifierWithUUID(instanceCode, dataSetInformation.getInstanceUUID(),
+ groupCode, projectCode, experimentCode);
+ }
+
+ private final static void checkDataSetInformation(final DataSetInformation dataSetInformation)
+ {
+ assert dataSetInformation != null : "Unspecified data set information";
+ assert dataSetInformation.getSampleIdentifier() != null : "Unspecified sample identifier";
+ final ch.systemsx.cisd.openbis.generic.shared.dto.identifier.ExperimentIdentifier experimentIdentifier =
+ dataSetInformation.getExperimentIdentifier();
+ assert experimentIdentifier != null : "Unspecified experiment identifier";
+ checkExperimentIdentifier(experimentIdentifier);
+ }
+
+ private final static void checkExperimentIdentifier(
+ final ch.systemsx.cisd.openbis.generic.shared.dto.identifier.ExperimentIdentifier experimentIdentifier)
+ {
+ assert experimentIdentifier.getGroupCode() != null : "Group code is null";
+ assert experimentIdentifier.getExperimentCode() != null : "Experiment code is null";
+ assert experimentIdentifier.getProjectCode() != null : "Project code is null";
+ }
+
+ private final IDataStructureV1_1 createDataStructure(final ExperimentPE experiment,
+ final DataSetInformation dataSetInformation,
+ final IProcedureAndDataTypeExtractor typeExtractor, final File incomingDataSetPath,
+ final File rootDir)
+ {
+ final FileStorage storage = new FileStorage(rootDir);
+ final IDataStructureV1_1 structure =
+ (IDataStructureV1_1) DataStructureFactory.createDataStructure(storage, version);
+ structure.create();
+ structure.setFormat(format);
+ structure.setExperimentIdentifier(createExperimentIdentifier(dataSetInformation));
+ structure.setExperimentRegistrationTimestamp(new ExperimentRegistrationTimestamp(experiment
+ .getRegistrationDate()));
+ final PersonPE registrator = experiment.getRegistrator();
+ final String firstName = registrator.getFirstName();
+ final String lastName = registrator.getLastName();
+ final String email = registrator.getEmail();
+ structure.setExperimentRegistrator(new ExperimentRegistrator(firstName, lastName, email));
+ structure.setSample(createSample(dataSetInformation));
+ structure.setDataSet(createDataSet(dataSetInformation, typeExtractor, incomingDataSetPath));
+ for (final FormatParameter formatParameter : formatParameters)
+ {
+ structure.addFormatParameter(formatParameter);
+ }
+ final SamplePropertyPE[] sampleProperties = dataSetInformation.getProperties();
+ final PlateDimension plateDimension =
+ PlateDimensionParser.tryToGetPlateDimension(sampleProperties);
+ if (plateDimension == null)
+ {
+ throw new EnvironmentFailureException(
+ "Missing plate geometry for the plate registered for sample identifier '"
+ + dataSetInformation.getSampleIdentifier() + "'.");
+ }
+ final Geometry plateGeometry =
+ new PlateGeometry(plateDimension.getRowsNum(), plateDimension.getColsNum());
+ structure.addFormatParameter(new FormatParameter(PlateGeometry.PLATE_GEOMETRY,
+ plateGeometry));
+ return structure;
+ }
+
+ private final Sample createSample(final DataSetInformation dataSetInformation)
+ {
+ final String groupCode = StringUtils.defaultString(dataSetInformation.getGroupCode());
+ final String instanceCode = dataSetInformation.getInstanceCode();
+ final String instanceUUID = dataSetInformation.getInstanceUUID();
+ assert instanceCode != null : "Unspecified database instance code";
+ assert instanceUUID != null : "Unspecified database instance UUID";
+ return new SampleWithOwner(dataSetInformation.getSampleIdentifier().getSampleCode(),
+ sampleType.getCode(), sampleTypeDescription, instanceUUID, instanceCode, groupCode);
+ }
+
+ private final static DataSet createDataSet(final DataSetInformation dataSetInformation,
+ final IProcedureAndDataTypeExtractor typeExtractor, final File incomingDataSetPath)
+ {
+ final String dataSetCode = dataSetInformation.getDataSetCode();
+ final String parentDataSetCode = dataSetInformation.getParentDataSetCode();
+ final List<String> parentCodes = getParentCodeList(parentDataSetCode);
+ final boolean isMeasured =
+ typeExtractor.getProcedureType(incomingDataSetPath).isDataAcquisition();
+ final DataSetType dataSetType = typeExtractor.getDataSetType(incomingDataSetPath);
+ final DataSet dataSet =
+ new DataSet(dataSetCode, dataSetType.getCode(),
+ ch.systemsx.cisd.bds.Utilities.Boolean.fromBoolean(isMeasured),
+ dataSetInformation.getProductionDate(), dataSetInformation
+ .getProducerCode(), parentCodes);
+ return dataSet;
+ }
+
+ private final static List<String> getParentCodeList(final String parentDataSetCode)
+ {
+ if (parentDataSetCode == null)
+ {
+ return Collections.<String> emptyList();
+ } else
+ {
+ return Collections.singletonList(parentDataSetCode);
+ }
+ }
+
+ private final static String getPathOf(final INode node)
+ {
+ final StringBuilder builder = new StringBuilder(node.getName());
+ IDirectory parent = node.tryGetParent();
+ while (parent != null)
+ {
+ builder.insert(0, '/');
+ builder.insert(0, parent.getName());
+ parent = parent.tryGetParent();
+ }
+ return builder.toString();
+ }
+
+ // TODO 2007-12-09, Christian Ribeaud: It will be a better choice to make two different
+ // implementations here: one for 'UnknownFormat1_0' and one for 'HCSImageFormat1_0'.
+ private final boolean needsImageFileExtractor()
+ {
+ return format.getCode().equals(UnknownFormatV1_0.UNKNOWN_1_0.getCode()) == false;
+ }
+
+ // Although this check should be performed in the BDS library when closing is performed, we set
+ // the complete flag here as we want to inform the registrator about the incompleteness.
+ private void checkCompleteness(final DataSetInformation dataSetInformation,
+ final String dataSetFileName, final IMailClient mailClientOrNull)
+ {
+ final List<FullLocation> fullLocations = imageCheckList.getCheckedOnFullLocations();
+ final boolean complete = fullLocations.size() == 0;
+ final IDataStructureV1_1 thisStructure = getDataStructure(dataStructureDir);
+ final DataSet dataSet = thisStructure.getDataSet();
+ dataSet.setComplete(complete);
+ thisStructure.setDataSet(dataSet);
+ dataSetInformation.setComplete(complete);
+ if (complete == false)
+ {
+ final String message =
+ String.format("Incomplete data set '%s': %d image file(s) "
+ + "are missing (locations: %s)", dataSetFileName, fullLocations.size(),
+ CollectionUtils.abbreviate(fullLocations, 10));
+ operationLog.warn(message);
+ if (mailClientOrNull != null)
+ {
+ final ExperimentRegistrator registrator = thisStructure.getExperimentRegistrator();
+ final String email = registrator.getEmail();
+ if (StringUtils.isBlank(email) == false)
+ {
+ try
+ {
+ mailClientOrNull.sendMessage("Incomplete data set '" + dataSetFileName
+ + "'", message, null, email);
+ } catch (final EnvironmentFailureException e)
+ {
+ notificationLog.error("Couldn't send the following e-mail to '" + email
+ + "': " + message, e);
+ }
+ } else
+ {
+ notificationLog.error("Unspecified e-mail address of experiment registrator "
+ + registrator);
+ }
+ }
+ }
+ }
+
+ /**
+ * For given <var>storedDataDirectory</var> returns the {@link IDataStructureV1_1}.
+ * <p>
+ * In ideal case returns internally saved <code>dataStructureDir</code> but when given
+ * <var>storedDataDirectory</var> changed (meaning no longer equal to storage root), then we
+ * have to reload the data structure.
+ * </p>
+ */
+ private final IDataStructureV1_1 getDataStructure(final File storedDataDirectory)
+ {
+ final IDataStructureV1_1 thisStructure;
+ if (storedDataDirectory.equals(dataStructureDir) == false)
+ {
+ final DataStructureLoader dataStructureLoader =
+ new DataStructureLoader(storedDataDirectory.getParentFile());
+ thisStructure =
+ (IDataStructureV1_1) dataStructureLoader.load(storedDataDirectory.getName());
+ } else
+ {
+ thisStructure = dataStructure;
+ if (thisStructure.isOpenOrCreated() == false)
+ {
+ thisStructure.open(Mode.READ_ONLY);
+ }
+ }
+ return thisStructure;
+ }
+
+ //
+ // AbstractStorageProcessor
+ //
+
+ public final File storeData(final ExperimentPE experiment,
+ final DataSetInformation dataSetInformation,
+ final IProcedureAndDataTypeExtractor typeExtractor, final IMailClient mailClient,
+ final File incomingDataSetDirectory, final File rootDirectory)
+ {
+ checkDataSetInformation(dataSetInformation);
+ assert rootDirectory != null : "Root directory can not be null.";
+ assert incomingDataSetDirectory != null : "Incoming data set directory can not be null.";
+ assert typeExtractor != null : "Unspecified IProcedureAndDataTypeExtractor implementation.";
+
+ dataStructureDir = rootDirectory;
+ dataStructureDir.mkdirs();
+ dataStructure =
+ createDataStructure(experiment, dataSetInformation, typeExtractor,
+ incomingDataSetDirectory, dataStructureDir);
+ final IFormattedData formattedData = dataStructure.getFormattedData();
+ if (formattedData instanceof IHCSImageFormattedData)
+ {
+ imageFormattedData = (IHCSImageFormattedData) formattedData;
+ final int channels = imageFormattedData.getChannelCount();
+ final Geometry plateGeometry = imageFormattedData.getPlateGeometry();
+ final Geometry wellGeometry = imageFormattedData.getWellGeometry();
+ imageCheckList = new HCSImageCheckList(channels, plateGeometry, wellGeometry);
+ }
+ if (needsImageFileExtractor())
+ {
+ imageFileRootDirectory = incomingDataSetDirectory;
+ final HCSImageFileExtractionResult result =
+ imageFileExtractor.process(NodeFactory
+ .createDirectoryNode(incomingDataSetDirectory), dataSetInformation,
+ this);
+ if (operationLog.isInfoEnabled())
+ {
+ operationLog.info(String.format("Extraction of %d files took %s.", result
+ .getTotalFiles(), DurationFormatUtils.formatDurationHMS(result
+ .getDuration())));
+ }
+ if (result.getInvalidFiles().size() > 0)
+ {
+ throw UserFailureException.fromTemplate(
+ "Following invalid files %s have been found.", CollectionUtils.abbreviate(
+ result.getInvalidFiles(), 10));
+ }
+ if (result.getTotalFiles() == 0)
+ {
+ throw UserFailureException.fromTemplate(
+ "No extractable files were found inside a dataset '%s'."
+ + " Have you changed your naming convention?",
+ incomingDataSetDirectory.getAbsolutePath());
+ }
+ dataStructure.setAnnotations(new HCSImageAnnotations(result.getChannels()));
+ checkCompleteness(dataSetInformation, incomingDataSetDirectory.getName(), mailClient);
+ } else
+ {
+ dataStructure.getOriginalData().addFile(incomingDataSetDirectory, null, true);
+ if (operationLog.isInfoEnabled())
+ {
+ operationLog.info(String.format("File '%s' added to original data.",
+ incomingDataSetDirectory));
+ }
+ }
+ dataStructure.close();
+ return dataStructureDir;
+ }
+
+ public final void unstoreData(final File incomingDataSetDirectory,
+ final File storedDataDirectory)
+ {
+ checkParameters(incomingDataSetDirectory, storedDataDirectory);
+
+ if (dataStructure == null)
+ {
+ // Nothing to do here.
+ return;
+ }
+ final IDirectory originalData = getDataStructure(dataStructureDir).getOriginalData();
+ final INode node = originalData.tryGetNode(incomingDataSetDirectory.getName());
+ // If the 'incoming' data have been moved to 'original' directory. This only happens if
+ // 'containsOriginalData' returns 'true'.
+ if (node != null)
+ {
+ final File incomingDirectory = incomingDataSetDirectory.getParentFile();
+ try
+ {
+ node.moveTo(incomingDirectory);
+ if (operationLog.isInfoEnabled())
+ {
+ operationLog.info(String.format(
+ "Node '%s' has moved to incoming directory '%s'.", node,
+ incomingDirectory.getAbsolutePath()));
+ }
+ } catch (final EnvironmentFailureException ex)
+ {
+ notificationLog.error(String.format(
+ "Could not move '%s' to incoming directory '%s'.", node, incomingDirectory
+ .getAbsolutePath()), ex);
+ return;
+ }
+ }
+ final IFileOperations fileOps = FileOperations.getMonitoredInstanceForCurrentThread();
+ if (fileOps.exists(incomingDataSetDirectory))
+ {
+ if (fileOps.removeRecursivelyQueueing(storedDataDirectory) == false)
+ {
+ operationLog
+ .error("Cannot delete '" + storedDataDirectory.getAbsolutePath() + "'.");
+ }
+ } else
+ {
+ notificationLog.error(String.format("Incoming data set directory '%s' does not "
+ + "exist, keeping store directory '%s'.", incomingDataSetDirectory,
+ storedDataDirectory));
+ }
+ }
+
+ public final File tryGetProprietaryData(final File storedDataDirectory)
+ {
+ assert storedDataDirectory != null : "Unspecified stored data directory.";
+ if (dataStructure == null)
+ {
+ operationLog.error("No data structure defined.");
+ return null;
+ }
+ if (imageFormattedData != null)
+ {
+ if (imageFormattedData.containsOriginalData() == false)
+ {
+ operationLog.warn("Original data are not available.");
+ return null;
+ }
+ }
+ final IDataStructureV1_1 thisStructure = getDataStructure(storedDataDirectory);
+ final IDirectory originalData = thisStructure.getOriginalData();
+ final Iterator<INode> iterator = originalData.iterator();
+ if (iterator.hasNext() == false)
+ {
+ return null;
+ }
+ final INode node = iterator.next();
+ final String path = getPathOf(node);
+ final File originalDataFile = new File(path);
+ if (originalDataFile.exists() == false)
+ {
+ operationLog.error("Original data set file '" + originalDataFile.getAbsolutePath()
+ + "' does not exist.");
+ return null;
+ }
+ return originalDataFile;
+ }
+
+ public final StorageFormat getStorageFormat()
+ {
+ return StorageFormat.BDS_DIRECTORY;
+ }
+
+ //
+ // IHCSImageFileAccepter
+ //
+
+ public final void accept(final int channel, final Location wellLocation,
+ final Location tileLocation, final IFile imageFile)
+ {
+ assert imageFileRootDirectory != null : "Incoming data set directory has not been set.";
+ final String imageRelativePath =
+ FileUtilities
+ .getRelativeFile(imageFileRootDirectory, new File(imageFile.getPath()));
+ assert imageRelativePath != null : "Image relative path should not be null.";
+ final NodePath nodePath =
+ imageFormattedData.addStandardNode(imageFileRootDirectory, imageRelativePath,
+ channel, wellLocation, tileLocation);
+ imageCheckList.checkOff(channel, wellLocation, tileLocation);
+ if (nodePath.getNode() instanceof ILink)
+ {
+ // We made a link in the 'standard' directory to the 'original' directory image file
+ // name. The image file did not change during the operation.
+ final Reference reference =
+ new Reference(nodePath.getPath(), imageFileRootDirectory.getName()
+ + Constants.PATH_SEPARATOR + imageRelativePath, ReferenceType.IDENTICAL);
+ getDataStructure(dataStructureDir).addReference(reference);
+ }
+ }
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/BaseDirectoryHolder.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/BaseDirectoryHolder.java
new file mode 100644
index 00000000000..fbe4029b62d
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/BaseDirectoryHolder.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.io.File;
+
+/**
+ * A tiny class which holds the <i>base directory</i> and ensures that the <i>target file</i>
+ * lazily gets computed only once.
+ *
+ * @author Christian Ribeaud
+ */
+final class BaseDirectoryHolder
+{
+
+ private final File baseDirectory;
+
+ private final IDataStoreStrategy dataStoreStrategy;
+
+ private final File incomingDataSetPath;
+
+ private File targetFile;
+
+ BaseDirectoryHolder(final IDataStoreStrategy dataStoreStrategy, final File baseDirectory,
+ final File incomingDataSetPath)
+ {
+ assert dataStoreStrategy != null : "Data store strategy can not be null.";
+ assert baseDirectory != null : "Base directory can not be null";
+ assert incomingDataSetPath != null : "Incoming data set can not be null.";
+ this.dataStoreStrategy = dataStoreStrategy;
+ this.baseDirectory = baseDirectory;
+ this.incomingDataSetPath = incomingDataSetPath;
+ }
+
+ private final File createTargetFile()
+ {
+ return dataStoreStrategy.getTargetPath(baseDirectory, incomingDataSetPath);
+ }
+
+ final File getBaseDirectory()
+ {
+ return baseDirectory;
+ }
+
+ final synchronized File getTargetFile()
+ {
+ if (targetFile == null)
+ {
+ targetFile = createTargetFile();
+ }
+ return targetFile;
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/ChannelSetHelper.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/ChannelSetHelper.java
new file mode 100644
index 00000000000..beaf87b8ee2
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/ChannelSetHelper.java
@@ -0,0 +1,106 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import ch.systemsx.cisd.bds.hcs.Channel;
+
+/**
+ * Helps to construct a sorted set of {@link Channel}.
+ *
+ * @author Christian Ribeaud
+ */
+public final class ChannelSetHelper
+{
+ private final SortedSet<Integer> wavelengths;
+
+ private Set<Channel> channels;
+
+ private boolean locked = false;
+
+ private Map<Integer, Channel> channelsByWavelength;
+
+ public ChannelSetHelper()
+ {
+ wavelengths = new TreeSet<Integer>();
+ }
+
+ /**
+ * Adds given <var>wavelength</var> to the internal set of wavelengths.
+ * <p>
+ * Wavelengths are ensured to be unique and are internally sorted.
+ * </p>
+ */
+ public final void addWavelength(final int wavelength)
+ {
+ assert locked == false : "You can no longer change the state of this class.";
+ wavelengths.add(wavelength);
+ }
+
+ /**
+ * Returns an unmodifiable sorted (based on {@link Channel#getCounter()}) set of channels.
+ * <p>
+ * This is typically called after having added all the wavelengths.
+ * </p>
+ */
+ public final Set<Channel> getChannelSet()
+ {
+ locked = true;
+ if (channels == null)
+ {
+ final Set<Channel> set = new TreeSet<Channel>();
+ final Iterator<Integer> iter = wavelengths.iterator();
+ for (int i = 0; iter.hasNext(); i++)
+ {
+ set.add(new Channel(i + 1, iter.next()));
+ }
+ channels = Collections.unmodifiableSet(set);
+ }
+ return channels;
+ }
+
+ /**
+ * For given wavelength returns corresponding <code>Channel</code>.
+ * <p>
+ * Never returns <code>null</code> and prefers to throw an exception if given <var>wavelength</var>
+ * can not be found.
+ * </p>
+ */
+ public final Channel getChannelForWavelength(final int wavelength)
+ {
+ locked = true;
+ if (channelsByWavelength == null)
+ {
+ final Map<Integer, Channel> map = new HashMap<Integer, Channel>();
+ for (final Channel channel : getChannelSet())
+ {
+ map.put(channel.getWavelength(), channel);
+ }
+ channelsByWavelength = Collections.unmodifiableMap(map);
+ }
+ final Channel channel = channelsByWavelength.get(wavelength);
+ assert channel != null : String.format("Given wavelength %d can not be found.", wavelength);
+ return channel;
+ }
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/DataSetInformation.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/DataSetInformation.java
new file mode 100644
index 00000000000..1713d9768c9
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/DataSetInformation.java
@@ -0,0 +1,274 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.io.Serializable;
+import java.util.Date;
+
+import org.apache.commons.lang.builder.ToStringBuilder;
+
+import ch.systemsx.cisd.common.types.BooleanOrUnknown;
+import ch.systemsx.cisd.common.utilities.ModifiedShortPrefixToStringStyle;
+import ch.systemsx.cisd.openbis.generic.shared.IWebService;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ExperimentPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ExternalData;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ExtractableData;
+import ch.systemsx.cisd.openbis.generic.shared.dto.SamplePropertyPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.DatabaseInstanceIdentifier;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.ExperimentIdentifier;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.GroupIdentifier;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifier;
+
+/**
+ * Container class for data extracted from the data set directory.
+ *
+ * @author Bernd Rinn
+ */
+public final class DataSetInformation implements Serializable
+{
+
+ private static final long serialVersionUID = IWebService.VERSION;
+
+ /** The sample code (aka <i>barcode</i>). <b>CAN NOT</b> be <code>null</code>. */
+ private String sampleCode;
+
+ private SamplePropertyPE[] properties = SamplePropertyPE.EMPTY_ARRAY;
+
+ /**
+ * The database instance <i>UUID</i>.
+ */
+ private String instanceUUID;
+
+ /**
+ * The database instance code.
+ */
+ private String instanceCode;
+
+ /**
+ * The group code (set by the {@link IDataSetInfoExtractor} or as specified in the
+ * <code>service.properties</code> file).
+ */
+ private String groupCode;
+
+ /** An object that uniquely identifies the experiment. Can be <code>null</code>. */
+ private ExperimentIdentifier experimentIdentifier;
+
+ /** Required information about the experiment. */
+ private transient ExperimentPE experiment;
+
+ private BooleanOrUnknown isCompleteFlag = BooleanOrUnknown.U;
+
+ /**
+ * A subset of {@link ExternalData} which gets set by the code extractor.
+ * <p>
+ * Initialized with <code>new ExtractableData()</code>.
+ * </p>
+ */
+ private ExtractableData extractableData = new ExtractableData();
+
+ /** This constructor is for serialization. */
+ public DataSetInformation()
+ {
+ }
+
+ public final BooleanOrUnknown getIsCompleteFlag()
+ {
+ return isCompleteFlag;
+ }
+
+ public final void setComplete(final boolean complete)
+ {
+ isCompleteFlag = BooleanOrUnknown.resolve(complete);
+ }
+
+ /**
+ * Returns the sample properties.
+ *
+ * @return never <code>null</code> but could return an empty array.
+ */
+ public final SamplePropertyPE[] getProperties()
+ {
+ return properties;
+ }
+
+ public final void setProperties(final SamplePropertyPE[] properties)
+ {
+ this.properties = properties;
+ }
+
+ public final String getInstanceCode()
+ {
+ return instanceCode;
+ }
+
+ public final void setInstanceCode(final String instanceCode)
+ {
+ this.instanceCode = instanceCode;
+ }
+
+ public final String getInstanceUUID()
+ {
+ return instanceUUID;
+ }
+
+ public final void setInstanceUUID(final String instanceUUID)
+ {
+ this.instanceUUID = instanceUUID;
+ }
+
+ /** Sets <code>experimentIdentifier</code>. */
+ public final void setExperimentIdentifier(final ExperimentIdentifier experimentIdentifier)
+ {
+ this.experimentIdentifier = experimentIdentifier;
+ }
+
+ /**
+ * Returns the identifier of experiment which makes it unique.
+ *
+ * @return <code>null</code> if no <code>ExperimentIdentifier</code> has been set.
+ */
+ public final ExperimentIdentifier getExperimentIdentifier()
+ {
+ return experimentIdentifier;
+ }
+
+ /**
+ * Returns the basic information about the experiment.
+ */
+ public ExperimentPE getExperiment()
+ {
+ return experiment;
+ }
+
+ /**
+ * Sets the basic information about the experiment.
+ */
+ public void setExperiment(final ExperimentPE experiment)
+ {
+ this.experiment = experiment;
+ }
+
+ /**
+ * Returns the sample identifier.
+ *
+ * @return <code>null</code> if <code>sampleCode</code> has not been set.
+ */
+ public final SampleIdentifier getSampleIdentifier()
+ {
+ if (sampleCode == null)
+ {
+ return null;
+ }
+ final DatabaseInstanceIdentifier databaseInstanceIdentifier =
+ new DatabaseInstanceIdentifier(instanceCode);
+ if (groupCode == null)
+ {
+ return new SampleIdentifier(databaseInstanceIdentifier, sampleCode);
+ }
+ return new SampleIdentifier(new GroupIdentifier(databaseInstanceIdentifier, groupCode),
+ sampleCode);
+ }
+
+ public final void setSampleCode(final String sampleCode)
+ {
+ this.sampleCode = sampleCode;
+ }
+
+ public final String getDataSetCode()
+ {
+ return extractableData.getCode();
+ }
+
+ public final void setDataSetCode(final String dataSetCode)
+ {
+ extractableData.setCode(dataSetCode);
+ }
+
+ public final String getProducerCode()
+ {
+ return extractableData.getDataProducerCode();
+ }
+
+ public final void setProducerCode(final String producerCode)
+ {
+ extractableData.setDataProducerCode(producerCode);
+ }
+
+ public final Date getProductionDate()
+ {
+ return extractableData.getProductionDate();
+ }
+
+ public final void setProductionDate(final Date productionDate)
+ {
+ extractableData.setProductionDate(productionDate);
+ }
+
+ public final ExtractableData getExtractableData()
+ {
+ return extractableData;
+ }
+
+ public final void setExtractableData(final ExtractableData extractableData)
+ {
+ this.extractableData = extractableData;
+ }
+
+ public final String getParentDataSetCode()
+ {
+ return extractableData.getParentDataSetCode();
+ }
+
+ public final void setParentDataSetCode(final String parentDataSetCode)
+ {
+ extractableData.setParentDataSetCode(parentDataSetCode);
+ }
+
+ public final void setGroupCode(final String groupCode)
+ {
+ this.groupCode = groupCode;
+ }
+
+ public final String getGroupCode()
+ {
+ return groupCode;
+ }
+
+ public final String describe()
+ {
+ if (experimentIdentifier == null)
+ {
+ return String.format("CODE('%s') SAMPLE_CODE('%s')", extractableData.getCode(),
+ sampleCode);
+ } else
+ {
+ return String.format("CODE('%s') SAMPLE_CODE('%s') EXPERIMENT('%s')", extractableData
+ .getCode(), sampleCode, experimentIdentifier.describe());
+ }
+ }
+
+ //
+ // Object
+ //
+
+ @Override
+ public final String toString()
+ {
+ return ToStringBuilder.reflectionToString(this,
+ ModifiedShortPrefixToStringStyle.MODIFIED_SHORT_PREFIX_STYLE);
+ }
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/DataSetNameEntitiesProvider.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/DataSetNameEntitiesProvider.java
new file mode 100644
index 00000000000..43357913ee3
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/DataSetNameEntitiesProvider.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.io.File;
+
+import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.lang.StringUtils;
+
+import ch.systemsx.cisd.common.exceptions.UserFailureException;
+
+/**
+ * Class which splits a data set name into entities and makes them accessible.
+ *
+ * @author Franz-Josef Elmer
+ */
+public class DataSetNameEntitiesProvider
+{
+ private final char entitySeparatorCharacter;
+
+ private final String[] entities;
+
+ private final String errorMessagePrefix;
+
+ /**
+ * Creates an instance based on the name of the specified file using the specified character
+ * which separates entities.
+ */
+ public DataSetNameEntitiesProvider(File dataSetFile, char entitySeparatorCharacter,
+ boolean stripExtension)
+ {
+ this(dataSetFile.getName(), entitySeparatorCharacter, stripExtension);
+ }
+
+ /**
+ * Creates an instance for the specified name using the specified character which separates
+ * entities.
+ */
+ public DataSetNameEntitiesProvider(String dataSetName, char entitySeparatorCharacter,
+ boolean stripExtension)
+ {
+ assert dataSetName != null : "Unspecified data set name.";
+
+ this.entitySeparatorCharacter = entitySeparatorCharacter;
+ final String name;
+ if (stripExtension)
+ {
+ name = FilenameUtils.getBaseName(dataSetName);
+ } else
+ {
+ name = dataSetName;
+ }
+ entities = StringUtils.split(name, entitySeparatorCharacter);
+ errorMessagePrefix = "Invalid data set name '" + dataSetName + "'. ";
+ }
+
+ /**
+ * Returns the entity of specified index. Negative arguments can also be used. They are
+ * interpreted as an index counting from the end of the sequence of entities. For example, -1
+ * denotes the last entity.
+ */
+ public String getEntity(int index)
+ {
+ if (index >= entities.length)
+ {
+ throwUserFailureException(index + 1);
+ }
+ int actualIndex = index;
+ if (index < 0)
+ {
+ actualIndex = entities.length + index;
+ if (actualIndex < 0)
+ {
+ throwUserFailureException(-index);
+ }
+ }
+ return entities[actualIndex];
+ }
+
+ private void throwUserFailureException(int expectedNumberOfEntities)
+ {
+ throw new UserFailureException(errorMessagePrefix + "We need " + expectedNumberOfEntities
+ + " entities, separated by '" + entitySeparatorCharacter + "', but got only "
+ + entities.length + ".");
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/DataStoreStrategyKey.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/DataStoreStrategyKey.java
new file mode 100644
index 00000000000..ba26e78155a
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/DataStoreStrategyKey.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+/**
+ * Key associated with each {@link IDataStoreStrategy}.
+ *
+ * @author Christian Ribeaud
+ */
+enum DataStoreStrategyKey
+{
+ /**
+ * This <code>IDataStoreStrategy</code> implementation if for data set that has been
+ * identified as <i>unidentified</i>, meaning that, for instance, no experiment could be mapped
+ * to the one found in given {@link DataSetInformation} (if we try to find out the sample to
+ * which this data set should be registered through the experiment).
+ */
+ UNIDENTIFIED,
+ /**
+ * This <code>IDataStoreStrategy</code> implementation if for data set that has been
+ * <i>identified</i>, meaning that kind of connection to this data set could be found in the
+ * database (through the derived <i>Master Plate</i> or through the experiment specified).
+ */
+ IDENTIFIED,
+ /**
+ * This <code>IDataStoreStrategy</code> implementation if for data set that has been
+ * identified as <i>invalid</i>, meaning that the data set itself or its
+ * <code>Master Plate</code> code is not registered in the database. So there is no
+ * possibility to link the data set to an already existing sample.
+ */
+ INVALID,
+ /**
+ * States that the transformation part could not be processed correctly.
+ */
+ ERROR;
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/DataStrategyStore.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/DataStrategyStore.java
new file mode 100644
index 00000000000..66abc5ce46c
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/DataStrategyStore.java
@@ -0,0 +1,192 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.io.File;
+import java.util.EnumMap;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
+
+import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.mail.IMailClient;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ExperimentPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.GroupPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.PersonPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ProjectPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.SamplePropertyPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.ExperimentIdentifier;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifier;
+
+/**
+ * Default <code>IDataStrategyStore</code> implementation.
+ * <p>
+ * Decides which {@link IDataStoreStrategy} will be applied for an incoming data set.
+ * </p>
+ *
+ * @author Christian Ribeaud
+ */
+final class DataStrategyStore implements IDataStrategyStore
+{
+ static final String SUBJECT_FORMAT = "ATTENTION: experiment '%s'";
+
+ private static final Logger notificationLog =
+ LogFactory.getLogger(LogCategory.NOTIFY, DataStrategyStore.class);
+
+ private static final Logger operationLog =
+ LogFactory.getLogger(LogCategory.OPERATION, DataStrategyStore.class);
+
+ private final IEncapsulatedLimsService limsService;
+
+ private final IMailClient mailClient;
+
+ private final Map<DataStoreStrategyKey, IDataStoreStrategy> dataStoreStrategies;
+
+ DataStrategyStore(final IEncapsulatedLimsService limsService, final IMailClient mailClient)
+ {
+ this.mailClient = mailClient;
+ dataStoreStrategies = createDataStoreStrategies();
+ this.limsService = limsService;
+ }
+
+ private final static void putDataStoreStrategy(
+ final Map<DataStoreStrategyKey, IDataStoreStrategy> map,
+ final IDataStoreStrategy dataStoreStrategy)
+ {
+ map.put(dataStoreStrategy.getKey(), dataStoreStrategy);
+ }
+
+ private final static Map<DataStoreStrategyKey, IDataStoreStrategy> createDataStoreStrategies()
+ {
+ final Map<DataStoreStrategyKey, IDataStoreStrategy> map =
+ new EnumMap<DataStoreStrategyKey, IDataStoreStrategy>(DataStoreStrategyKey.class);
+ putDataStoreStrategy(map, new IdentifiedDataStrategy());
+ putDataStoreStrategy(map, new NamedDataStrategy(DataStoreStrategyKey.UNIDENTIFIED));
+ putDataStoreStrategy(map, new NamedDataStrategy(DataStoreStrategyKey.INVALID));
+ return map;
+ }
+
+ final static String createInvalidSampleCodeMessage(final DataSetInformation dataSetInfo)
+ {
+ return "ETL server: Sample '" + dataSetInfo.getSampleIdentifier()
+ + "' is not valid for experiment '" + dataSetInfo.getExperimentIdentifier()
+ + "' (it has maybe been invalidated?).";
+ }
+
+ private static String createNotificationMessage(final DataSetInformation dataSetInfo,
+ final File incomingDataSetPathForLogging)
+ {
+ final SampleIdentifier sampleIdentifier = dataSetInfo.getSampleIdentifier();
+ return String.format("Directory '%s', sample identifier '%s': unknown to openBIS",
+ incomingDataSetPathForLogging, sampleIdentifier);
+ }
+
+ private final static ExperimentIdentifier createExperimentIdentifier(
+ final ExperimentPE experiment)
+ {
+ ExperimentIdentifier experimentIdentifier;
+ experimentIdentifier = new ExperimentIdentifier();
+ experimentIdentifier.setExperimentCode(experiment.getCode());
+ final ProjectPE project = experiment.getProject();
+ assert project != null : "Unspecified project";
+ experimentIdentifier.setProjectCode(project.getCode());
+ final GroupPE group = project.getGroup();
+ assert group != null : "Unspecified group";
+ experimentIdentifier.setGroupCode(group.getCode());
+ return experimentIdentifier;
+ }
+
+ //
+ // IDataStrategyStore
+ //
+
+ public final IDataStoreStrategy getDataStoreStrategy(final DataSetInformation dataSetInfo,
+ final File incomingDataSetPath)
+ {
+
+ assert incomingDataSetPath != null : "Incoming data set path can not be null.";
+ if (dataSetInfo == null)
+ {
+ return dataStoreStrategies.get(DataStoreStrategyKey.UNIDENTIFIED);
+ }
+ final SampleIdentifier sampleIdentifier = dataSetInfo.getSampleIdentifier();
+ final ExperimentPE experiment = limsService.getBaseExperiment(sampleIdentifier);
+ if (experiment == null)
+ {
+ notificationLog.error(createNotificationMessage(dataSetInfo, incomingDataSetPath));
+ return dataStoreStrategies.get(DataStoreStrategyKey.UNIDENTIFIED);
+ } else if (experiment.getInvalidation() != null)
+ {
+ notificationLog.error("Data set for sample '" + sampleIdentifier
+ + "' can not be registered because experiment '" + experiment.getCode()
+ + "' has been invalidated.");
+ return dataStoreStrategies.get(DataStoreStrategyKey.UNIDENTIFIED);
+ }
+ dataSetInfo.setExperiment(experiment);
+ final ExperimentIdentifier experimentIdentifier = createExperimentIdentifier(experiment);
+ dataSetInfo.setExperimentIdentifier(experimentIdentifier);
+
+ final SamplePropertyPE[] properties =
+ limsService.getPropertiesOfTopSampleRegisteredFor(sampleIdentifier);
+ if (properties == null)
+ {
+ final PersonPE registrator = experiment.getRegistrator();
+ assert registrator != null : "Registrator must be known";
+ final String message = createInvalidSampleCodeMessage(dataSetInfo);
+ final String recipientMail = registrator.getEmail();
+ if (StringUtils.isNotBlank(recipientMail))
+ {
+ sendEmail(message, experimentIdentifier, recipientMail);
+ } else
+ {
+ notificationLog.error("The registrator '" + registrator
+ + "' has a blank email, sending the following email failed:\n" + message);
+ }
+ operationLog.error(String.format("Incoming data set '%s' claims to "
+ + "belong to experiment '%s' and sample"
+ + " identifier '%s', but according to the openBIS server "
+ + "there is no such sample for this "
+ + "experiment (it has maybe been invalidated?). We thus consider it invalid.",
+ incomingDataSetPath, experimentIdentifier, sampleIdentifier));
+ return dataStoreStrategies.get(DataStoreStrategyKey.INVALID);
+ }
+ dataSetInfo.setProperties(properties);
+
+ if (operationLog.isInfoEnabled())
+ {
+ operationLog.info("Identified that database knows experiment '" + experimentIdentifier
+ + "' and sample '" + sampleIdentifier + "'.");
+ }
+ return dataStoreStrategies.get(DataStoreStrategyKey.IDENTIFIED);
+ }
+
+ private void sendEmail(final String message, final ExperimentIdentifier experimentIdentifier,
+ final String recipientMail)
+ {
+ final String subject = String.format(SUBJECT_FORMAT, experimentIdentifier);
+ try
+ {
+ mailClient.sendMessage(subject, message, null, recipientMail);
+ } catch (final EnvironmentFailureException ex)
+ {
+ operationLog.error(ex.getMessage());
+ }
+ }
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/DefaultDataSetInfoExtractor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/DefaultDataSetInfoExtractor.java
new file mode 100644
index 00000000000..fea627bdb96
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/DefaultDataSetInfoExtractor.java
@@ -0,0 +1,246 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.io.File;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Properties;
+
+import ch.rinn.restrictions.Private;
+import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.common.exceptions.UserFailureException;
+import ch.systemsx.cisd.common.utilities.PropertyUtils;
+
+/**
+ * Default implementation which assumes that the information can be extracted from the file name.
+ * Following information can be extracted:
+ * <ul>
+ * <li>Sample code
+ * <li>Parent data set code
+ * <li>Data producer code
+ * <li>Data production date
+ * </ul>
+ * The name is split into entities separated by the property {@link #ENTITY_SEPARATOR_PROPERTY_NAME}
+ * . It is assumed that each of the above-mentioned pieces of information is one of these entities.
+ * The extractor can be configured by the following optional properties:
+ * <table border="1" * cellspacing="0" cellpadding="5">
+ * <tr>
+ * <th>Property</th>
+ * <th>Default value</th>
+ * <th>Description</th>
+ * </tr>
+ * <tr>
+ * <td><code>entity-separator</code></td>
+ * <td><code>.</code></td>
+ * <td>Character which separates entities in the file name. Whitespace characters are not allowed.</td>
+ * </tr>
+ * <tr>
+ * <td><code>index-of-sample-code</code></td>
+ * <td><code>-1</code></td>
+ * <td>Index of the entity which is interpreted as the sample code.</td>
+ * </tr>
+ * <tr>
+ * <td><code>index-of-parent-data-set-code</code></td>
+ * <td> </td>
+ * <td>Index of the entity which is interpreted as the parent data set code. If not specified no
+ * parent data set code will be extracted.</td>
+ * </tr>
+ * <tr>
+ * <td><code>index-of-data-producer-code</code></td>
+ * <td> </td>
+ * <td>Index of the entity which is interpreted as the data producer code. If not specified no data
+ * producer code will be extracted.</td>
+ * </tr>
+ * <tr>
+ * <td><code>index-of-data-production-date</code></td>
+ * <td> </td>
+ * <td>Index of the entity which is interpreted as the data production date. If not specified no
+ * data production date will be extracted.</td>
+ * </tr>
+ * <tr>
+ * <td><code>data-production-date-format</code></td>
+ * <td><code>yyyyMMddHHmmss</code></td>
+ * <td>Format of the data production date. For the correct syntax see <a
+ * href="http://java.sun.com/j2se/1.5.0/docs/api/java/text/SimpleDateFormat.html"
+ * >SimpleDateFormat</a>.</td>
+ * </tr>
+ * </table>
+ * The first entity has index 0, the second 1, etc. Using negative numbers one can specify entities
+ * from the end. Thus, -1 means the last entity, -2 the second last entity, etc.
+ *
+ * @author Franz-Josef Elmer
+ */
+public class DefaultDataSetInfoExtractor extends AbstractDataSetInfoExtractor
+{
+
+ /**
+ * Name of the property specifying the index of the entity which should be interpreted as the
+ * sample code.
+ * <p>
+ * Use a negative number to count from the end, e.g. <code>-1</code> to use the last entity as
+ * the sample code.
+ * </p>
+ */
+ @Private
+ static final String INDEX_OF_SAMPLE_CODE = "index-of-sample-code";
+
+ /**
+ * Name of the property specifying the index of the entity which should be interpreted as the
+ * parent data set code.
+ * <p>
+ * Use a negative number to count from the end, e.g. <code>-1</code> to use the last entity as
+ * the sample code.
+ * </p>
+ */
+ @Private
+ static final String INDEX_OF_PARENT_DATA_SET_CODE = "index-of-parent-data-set-code";
+
+ /** Default index of sample code. */
+ private static final int DEFAULT_INDEX_OF_SAMPLE_CODE = -1;
+
+ /**
+ * Name of the property specifying the index of the entity which should be interpreted as the
+ * data producer code.
+ * <p>
+ * Use a negative number to count from the end, e.g. <code>-1</code> to use the last entity as
+ * the data producer code.
+ * </p>
+ */
+ @Private
+ static final String INDEX_OF_DATA_PRODUCER_CODE = "index-of-data-producer-code";
+
+ /**
+ * Name of the property specifying the index of the entity which should be interpreted as the
+ * data production date.
+ * <p>
+ * Use a negative number to count from the end, e.g. <code>-1</code> to use the last entity as
+ * the data production date.
+ * </p>
+ */
+ @Private
+ static final String INDEX_OF_DATA_PRODUCTION_DATE = "index-of-data-production-date";
+
+ /**
+ * Name of the property specifying the format of the data production date.
+ */
+ @Private
+ static final String DATA_PRODUCTION_DATE_FORMAT = "data-production-date-format";
+
+ /** Default data production date format. */
+ private static final String DEFAULT_DATA_PRODUCTION_DATE_FORMAT = "yyyyMMddHHmmss";
+
+ private final int indexOfSampleCode;
+
+ private final boolean noParentDataSetCode;
+
+ private final int indexOfParentDataSetCode;
+
+ private final boolean noDataProducerCode;
+
+ private final int indexOfDataProducerCode;
+
+ private final boolean noDataProductionDate;
+
+ private final int indexOfDataProductionDate;
+
+ private final SimpleDateFormat dateFormat;
+
+ /**
+ * The <var>properties</var> are not used by this constructor but present to fulfill the
+ * contract.
+ */
+ public DefaultDataSetInfoExtractor(final Properties globalProperties)
+ {
+ super(globalProperties);
+ indexOfSampleCode =
+ PropertyUtils
+ .getInt(properties, INDEX_OF_SAMPLE_CODE, DEFAULT_INDEX_OF_SAMPLE_CODE);
+ String indexAsString = properties.getProperty(INDEX_OF_PARENT_DATA_SET_CODE);
+ noParentDataSetCode = indexAsString == null;
+ indexOfParentDataSetCode =
+ PropertyUtils.getInt(properties, INDEX_OF_PARENT_DATA_SET_CODE, 0);
+ indexAsString = properties.getProperty(INDEX_OF_DATA_PRODUCER_CODE);
+ noDataProducerCode = indexAsString == null;
+ indexOfDataProducerCode = PropertyUtils.getInt(properties, INDEX_OF_DATA_PRODUCER_CODE, 0);
+ indexAsString = properties.getProperty(INDEX_OF_DATA_PRODUCTION_DATE);
+ noDataProductionDate = indexAsString == null;
+ indexOfDataProductionDate =
+ PropertyUtils.getInt(properties, INDEX_OF_DATA_PRODUCTION_DATE, 0);
+ dateFormat =
+ new SimpleDateFormat(properties.getProperty(DATA_PRODUCTION_DATE_FORMAT,
+ DEFAULT_DATA_PRODUCTION_DATE_FORMAT));
+ }
+
+ //
+ // ICodeExtractor
+ //
+
+ public DataSetInformation getDataSetInformation(final File incomingDataSetPath)
+ throws EnvironmentFailureException, UserFailureException
+ {
+ assert incomingDataSetPath != null : "Incoming data set path can not be null.";
+ final DataSetNameEntitiesProvider entitiesProvider =
+ new DataSetNameEntitiesProvider(incomingDataSetPath, entitySeparator,
+ stripExtension);
+ final DataSetInformation dataSetInformation = new DataSetInformation();
+ dataSetInformation.setSampleCode(entitiesProvider.getEntity(indexOfSampleCode));
+ dataSetInformation.setParentDataSetCode(tryGetParentDataSetCode(entitiesProvider));
+ dataSetInformation.setProducerCode(tryGetDataProducerCode(entitiesProvider));
+ dataSetInformation.setProductionDate(tryGetDataProductionDate(entitiesProvider));
+ return dataSetInformation;
+ }
+
+ private String tryGetParentDataSetCode(
+ final DataSetNameEntitiesProvider dataSetNameEntitiesProvider)
+ {
+ if (noParentDataSetCode)
+ {
+ return null;
+ }
+ return dataSetNameEntitiesProvider.getEntity(indexOfParentDataSetCode);
+ }
+
+ private String tryGetDataProducerCode(
+ final DataSetNameEntitiesProvider dataSetNameEntitiesProvider)
+ {
+ if (noDataProducerCode)
+ {
+ return null;
+ }
+ return dataSetNameEntitiesProvider.getEntity(indexOfDataProducerCode);
+ }
+
+ private Date tryGetDataProductionDate(
+ final DataSetNameEntitiesProvider dataSetNameEntitiesProvider)
+ {
+ if (noDataProductionDate)
+ {
+ return null;
+ }
+ final String dateString = dataSetNameEntitiesProvider.getEntity(indexOfDataProductionDate);
+ try
+ {
+ return dateFormat.parse(dateString);
+ } catch (final ParseException e)
+ {
+ throw new UserFailureException("Could not parse data production date '" + dateString
+ + "' because it violates the following format: " + dateFormat.toPattern());
+ }
+ }
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/DefaultStorageProcessor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/DefaultStorageProcessor.java
new file mode 100644
index 00000000000..356a50764a1
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/DefaultStorageProcessor.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.io.File;
+import java.util.Properties;
+
+import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.common.mail.IMailClient;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ExperimentPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.StorageFormat;
+
+/**
+ * A default {@link IStorageProcessor} implementation.
+ *
+ * @author Christian Ribeaud
+ */
+public class DefaultStorageProcessor extends AbstractStorageProcessor
+{
+ static final String NO_RENAME = "Couldn't rename '%s' to '%s'.";
+
+ public DefaultStorageProcessor(final Properties properties)
+ {
+ super(properties);
+ }
+
+ private final static File createTargetFile(final File incomingDataSetFile,
+ final File baseDirectory)
+ {
+ return new File(baseDirectory, incomingDataSetFile.getName());
+ }
+
+ //
+ // AbstractStorageProcessor
+ //
+
+ public final File storeData(final ExperimentPE experiment,
+ final DataSetInformation dataSetInformation,
+ final IProcedureAndDataTypeExtractor typeExtractor, final IMailClient mailClient,
+ final File incomingDataSetDirectory, final File rootDir)
+ {
+ checkParameters(incomingDataSetDirectory, rootDir);
+ final File targetFile = createTargetFile(incomingDataSetDirectory, rootDir);
+ if (FileRenamer.renameAndLog(incomingDataSetDirectory, targetFile) == false)
+ {
+ throw new EnvironmentFailureException(String.format(NO_RENAME,
+ incomingDataSetDirectory, targetFile));
+ }
+ return targetFile;
+ }
+
+ public final void unstoreData(final File incomingDataSetDirectory,
+ final File storedDataDirectory)
+ {
+ checkParameters(incomingDataSetDirectory, storedDataDirectory);
+ // Note that this will move back <code>targetPath</code> to its original place but the
+ // directory structure will persist. Right now, we consider this is fine as these empty
+ // directories will not disturb the running application.
+ FileRenamer.renameAndLog(createTargetFile(incomingDataSetDirectory, storedDataDirectory),
+ incomingDataSetDirectory);
+ }
+
+ public final StorageFormat getStorageFormat()
+ {
+ return StorageFormat.PROPRIETARY;
+ }
+
+ public final File tryGetProprietaryData(final File storedDataDirectory)
+ {
+ assert storedDataDirectory != null : "Unspecified stored data directory.";
+ return storedDataDirectory;
+ }
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/ETLServerPlugin.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/ETLServerPlugin.java
new file mode 100644
index 00000000000..8d48eb808d5
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/ETLServerPlugin.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+
+/**
+ * ETL Server plugin as a bean.
+ *
+ * @author Franz-Josef Elmer
+ */
+public class ETLServerPlugin implements IETLServerPlugin
+{
+ private final IDataSetInfoExtractor codeExtractor;
+
+ private final IProcedureAndDataTypeExtractor typeExtractor;
+
+ private final IStorageProcessor storageProcessor;
+
+ /**
+ * Creates an instance with the specified extractors.
+ */
+ public ETLServerPlugin(final IDataSetInfoExtractor codeExtractor,
+ final IProcedureAndDataTypeExtractor typeExtractor,
+ final IStorageProcessor storageProcessor)
+ {
+ assert codeExtractor != null : "Missing code extractor";
+ assert typeExtractor != null : "Missing type extractor";
+ assert storageProcessor != null : "Missing storage processor";
+
+ this.codeExtractor = codeExtractor;
+ this.typeExtractor = typeExtractor;
+ this.storageProcessor = storageProcessor;
+ }
+
+ //
+ // IETLServerPlugin
+ //
+
+ public final IDataSetInfoExtractor getDataSetInfoExtractor()
+ {
+ return codeExtractor;
+ }
+
+ public final IProcedureAndDataTypeExtractor getTypeExtractor()
+ {
+ return typeExtractor;
+ }
+
+ public final IStorageProcessor getStorageProcessor()
+ {
+ return storageProcessor;
+ }
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/EncapsulatedLimsService.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/EncapsulatedLimsService.java
new file mode 100644
index 00000000000..e326821a7eb
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/EncapsulatedLimsService.java
@@ -0,0 +1,211 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import org.apache.log4j.Logger;
+
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.exceptions.InvalidSessionException;
+import ch.systemsx.cisd.common.exceptions.UserFailureException;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.openbis.generic.shared.IETLLIMSService;
+import ch.systemsx.cisd.openbis.generic.shared.dto.DatabaseInstancePE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ExperimentPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ExternalData;
+import ch.systemsx.cisd.openbis.generic.shared.dto.SamplePropertyPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifier;
+
+/**
+ * A class that encapsulates the {@link IETLLIMSService} and handles (re-)authentication automatically
+ * as needed.
+ * <p>
+ * This class is thread safe (otherwise one thread can change the session and cause the other thread
+ * to use the invalid one).
+ * </p>
+ *
+ * @author Bernd Rinn
+ */
+final class EncapsulatedLimsService implements IEncapsulatedLimsService
+{
+
+ private static final Logger operationLog =
+ LogFactory.getLogger(LogCategory.OPERATION, EncapsulatedLimsService.class);
+
+ private final String username;
+
+ private final String password;
+
+ private final IETLLIMSService limsService;
+
+ private String sessionToken; // NOTE: can be changed in parallel by different threads
+
+ private Integer version;
+
+ private DatabaseInstancePE homeDatabaseInstance;
+
+ EncapsulatedLimsService(final IETLLIMSService limsService, final String username,
+ final String password)
+ {
+ assert limsService != null : "Given IETLLIMSService implementation can not be null.";
+ assert username != null : "Given username can not be null.";
+ assert password != null : "Given password can not be null.";
+
+ this.limsService = limsService;
+ this.username = username;
+ this.password = password;
+ }
+
+ private final void authenticate()
+ {
+ if (operationLog.isDebugEnabled())
+ {
+ operationLog.debug("Authenticating to LIMS server as user '" + username + "'.");
+ }
+ sessionToken = limsService.authenticate(username, password);
+ if (sessionToken == null)
+ {
+ final String msg =
+ "Authentication failure to LIMS server. Most probable cause: user or password are invalid.";
+ throw new ConfigurationFailureException(msg);
+ }
+ }
+
+ private final void checkSessionToken()
+ {
+ if (sessionToken == null)
+ {
+ authenticate();
+ }
+ }
+
+ private final ExperimentPE primGetBaseExperiment(final SampleIdentifier sampleIdentifier)
+ {
+ return limsService.tryToGetBaseExperiment(sessionToken, sampleIdentifier);
+ }
+
+ private final void primRegisterDataSet(final DataSetInformation dataSetInformation,
+ final String procedureTypeCode, final ExternalData data)
+ {
+ limsService.registerDataSet(sessionToken, dataSetInformation.getSampleIdentifier(),
+ procedureTypeCode, data);
+ }
+
+ private final SamplePropertyPE[] primGetPropertiesOfSampleRegisteredFor(
+ final SampleIdentifier sampleIdentifier)
+ {
+ return limsService.tryToGetPropertiesOfTopSampleRegisteredFor(sessionToken, sampleIdentifier);
+ }
+
+ private final String primCreateDataSetCode()
+ {
+ return limsService.createDataSetCode(sessionToken);
+ }
+
+ //
+ // IEncapsulatedLimsService
+ //
+
+ synchronized public final ExperimentPE getBaseExperiment(
+ final SampleIdentifier sampleIdentifier)
+ {
+ assert sampleIdentifier != null : "Given sample identifier can not be null.";
+
+ checkSessionToken();
+ try
+ {
+ return primGetBaseExperiment(sampleIdentifier);
+ } catch (final InvalidSessionException ex)
+ {
+ authenticate();
+ return primGetBaseExperiment(sampleIdentifier);
+ }
+ }
+
+ synchronized public final void registerDataSet(final DataSetInformation dataSetInformation,
+ final String procedureTypeCode, final ExternalData data)
+ {
+ assert dataSetInformation != null : "missing sample identifier";
+ assert procedureTypeCode != null : "missing procedure type";
+ assert data != null : "missing data";
+
+ checkSessionToken();
+ try
+ {
+ primRegisterDataSet(dataSetInformation, procedureTypeCode, data);
+ } catch (final InvalidSessionException ex)
+ {
+ authenticate();
+ primRegisterDataSet(dataSetInformation, procedureTypeCode, data);
+ }
+ if (operationLog.isInfoEnabled())
+ {
+ operationLog.info("Registered in openBIS: data set " + dataSetInformation.describe()
+ + " PROCEDURE_TYPE('" + procedureTypeCode + "').");
+ }
+ }
+
+ synchronized public final SamplePropertyPE[] getPropertiesOfTopSampleRegisteredFor(
+ final SampleIdentifier sampleIdentifier) throws UserFailureException
+ {
+ assert sampleIdentifier != null : "Given sample identifier can not be null.";
+
+ checkSessionToken();
+ try
+ {
+ return primGetPropertiesOfSampleRegisteredFor(sampleIdentifier);
+ } catch (final InvalidSessionException ex)
+ {
+ authenticate();
+ return primGetPropertiesOfSampleRegisteredFor(sampleIdentifier);
+ }
+ }
+
+ synchronized public final int getVersion()
+ {
+ checkSessionToken();
+ if (version == null)
+ {
+ version = limsService.getVersion();
+ }
+ return version;
+ }
+
+ synchronized public final DatabaseInstancePE getHomeDatabaseInstance()
+ {
+ checkSessionToken();
+ if (homeDatabaseInstance == null)
+ {
+ homeDatabaseInstance = limsService.getHomeDatabaseInstance(sessionToken);
+ }
+ return homeDatabaseInstance;
+ }
+
+ synchronized public final String createDataSetCode()
+ {
+ checkSessionToken();
+ try
+ {
+ return primCreateDataSetCode();
+ } catch (final InvalidSessionException ex)
+ {
+ authenticate();
+ return primCreateDataSetCode();
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/FileBasedFile.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/FileBasedFile.java
new file mode 100644
index 00000000000..bd025e258b1
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/FileBasedFile.java
@@ -0,0 +1,187 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.commons.io.FileUtils;
+
+import ch.systemsx.cisd.common.exceptions.CheckedExceptionTunnel;
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.common.exceptions.StopException;
+import ch.systemsx.cisd.common.filesystem.FileOperations;
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
+import ch.systemsx.cisd.common.filesystem.IImmutableCopier;
+
+/**
+ * Adapter of {@link File}. Files are copies by creating hard links (if possible) if the parameter
+ * <var>hardLinkInsteadOfCopy</var> of the constructor is set to <code>true</code>. Otherwise files
+ * are always copied.
+ *
+ * @author Franz-Josef Elmer
+ */
+public class FileBasedFile implements IFile
+{
+ private final IImmutableCopier hardLinkCopierOrNull;
+
+ private final File file;
+
+ /**
+ * Creates a new instance for the specified file with the specified copy policy.
+ *
+ * @param file Real file wrapped by this adapter.
+ * @param hardLinkCopierOrNull If specified, will be used instead of the normal file system
+ * copier for copying files and directories.
+ */
+ public FileBasedFile(final File file, final IImmutableCopier hardLinkCopierOrNull)
+ {
+ assert file != null : "Unspecified file.";
+ this.file = file;
+ this.hardLinkCopierOrNull = hardLinkCopierOrNull;
+ }
+
+ public void copyFrom(final File sourceFile)
+ {
+ copy(sourceFile, file);
+ }
+
+ public void copyTo(final File destinationFile)
+ {
+ copy(file, destinationFile);
+ }
+
+ private void copy(final File sourceFile, final File destinationFile)
+ {
+ if (sourceFile.isDirectory())
+ {
+ copyDirectory(sourceFile, destinationFile);
+ } else
+ {
+ copyFile(sourceFile, destinationFile);
+ }
+ }
+
+ private void copyFile(final File sourceFile, final File destinationFile)
+ {
+ if (hardLinkCopierOrNull != null)
+ {
+ final File destinationDirectory = destinationFile.getParentFile();
+ final boolean ok =
+ hardLinkCopierOrNull.copyImmutably(sourceFile, destinationDirectory,
+ destinationFile.getName());
+ if (ok == false)
+ {
+ throw new EnvironmentFailureException("Couldn't copy '"
+ + sourceFile.getAbsolutePath() + "' using hard links to '"
+ + destinationFile.getAbsolutePath()
+ + "'. Maybe the destination already exists?");
+ }
+ } else
+ {
+ try
+ {
+ StopException.check();
+ FileUtils.copyFile(sourceFile, destinationFile, true);
+ } catch (final IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ }
+
+ private void copyDirectory(final File sourceDirectory, final File destinationDirectory)
+ {
+ if (hardLinkCopierOrNull != null)
+ {
+ final File destinationParentDirectory = destinationDirectory.getParentFile();
+ final boolean ok =
+ hardLinkCopierOrNull.copyImmutably(sourceDirectory,
+ destinationParentDirectory, destinationDirectory.getName());
+ if (ok == false)
+ {
+ throw new EnvironmentFailureException("Couldn't copy '"
+ + sourceDirectory.getAbsolutePath() + "' using hard links to '"
+ + destinationDirectory.getAbsolutePath()
+ + "'. Maybe the destination already exists?");
+ }
+ } else
+ {
+ try
+ {
+ StopException.check();
+ FileUtils.copyDirectory(sourceDirectory, destinationDirectory, true);
+ } catch (final IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ }
+
+ public void delete()
+ {
+ if (FileOperations.getMonitoredInstanceForCurrentThread().removeRecursivelyQueueing(file))
+ {
+ throw new EnvironmentFailureException("Can not delete file '"
+ + file.getAbsolutePath() + "'.");
+ }
+ }
+
+ public String getAbsolutePath()
+ {
+ return file.getAbsolutePath();
+ }
+
+ public byte[] read()
+ {
+ try
+ {
+ return FileUtils.readFileToByteArray(file);
+ } catch (final IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ public void write(final byte[] data)
+ {
+ try
+ {
+ FileUtils.writeByteArrayToFile(file, data);
+ } catch (final IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ public final void check() throws EnvironmentFailureException, ConfigurationFailureException
+ {
+ final String response = FileUtilities.checkDirectoryFullyAccessible(file, "");
+ if (response != null)
+ {
+ throw new ConfigurationFailureException(response);
+ }
+ }
+
+ public boolean isRemote()
+ {
+ return false;
+ }
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/FileBasedFileFactory.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/FileBasedFileFactory.java
new file mode 100644
index 00000000000..659b123cedb
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/FileBasedFileFactory.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.io.File;
+
+import org.apache.commons.io.FilenameUtils;
+
+import ch.systemsx.cisd.common.TimingParameters;
+import ch.systemsx.cisd.common.filesystem.FastRecursiveHardLinkMaker;
+import ch.systemsx.cisd.common.filesystem.IImmutableCopier;
+
+/**
+ * File factory based on {@link File}. Files are copies by creating hard links (if possible) if the
+ * parameter <var>hardLinkInsteadOfCopy</var> of the constructor is set to <code>true</code>.
+ * Otherwise files are always copied.
+ *
+ * @author Franz-Josef Elmer
+ */
+public class FileBasedFileFactory implements IFileFactory
+{
+ private final IImmutableCopier hardLinkCopierOrNull;
+
+ /**
+ * Creates a new instance with the specified copy policy. Uses default timing parameters.
+ *
+ * @param hardLinkInsteadOfCopy If <code>true</code> hard links instead of copies are created.
+ */
+ public FileBasedFileFactory(final boolean hardLinkInsteadOfCopy)
+ {
+ this(hardLinkInsteadOfCopy, TimingParameters.getDefaultParameters());
+ }
+
+ /**
+ * Creates a new instance with the specified copy policy.
+ *
+ * @param hardLinkInsteadOfCopy If <code>true</code> hard links instead of copies are created.
+ * @param timingParameters The timing parameters to use for copy oprations.
+ */
+ public FileBasedFileFactory(final boolean hardLinkInsteadOfCopy,
+ final TimingParameters timingParameters)
+ {
+ this.hardLinkCopierOrNull =
+ tryGetHardLinkCopier(hardLinkInsteadOfCopy, timingParameters);
+ }
+
+ private static IImmutableCopier tryGetHardLinkCopier(
+ final boolean hardLinkInsteadOfCopy, final TimingParameters timingParameters)
+ {
+ if (hardLinkInsteadOfCopy)
+ {
+ return FastRecursiveHardLinkMaker.tryCreate(timingParameters);
+ } else
+ {
+ return null;
+ }
+ }
+
+ private final IFile wrap(final File file)
+ {
+ return new FileBasedFile(file, hardLinkCopierOrNull);
+ }
+
+ //
+ // IFileFactory
+ //
+
+ public final IFile create(final String path)
+ {
+ assert path != null : "Unspecified path.";
+ final File file = new File(path);
+ return wrap(file);
+ }
+
+ public final IFile create(final IFile baseDir, final String relativePath)
+ {
+ assert baseDir != null : "Unspecified base directory.";
+ assert relativePath != null : "Unspecified relative pate";
+ assert FilenameUtils.getPrefixLength(relativePath) == 0 : String.format(
+ "Given relative path '%s' is not relative.", relativePath);
+ return wrap(new File(baseDir.getAbsolutePath(), relativePath));
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/FileRenamer.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/FileRenamer.java
new file mode 100644
index 00000000000..f6a435d4236
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/FileRenamer.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.io.File;
+
+import org.apache.log4j.Logger;
+
+import ch.systemsx.cisd.common.filesystem.FileOperations;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+
+/**
+ * A file renamer that logs it's operations.
+ * <p>
+ * Renames and logs the file renaming process.
+ * </p>
+ *
+ * @author Franz-Josef Elmer
+ */
+final class FileRenamer
+{
+ private final static Logger notificationLog =
+ LogFactory.getLogger(LogCategory.NOTIFY, FileRenamer.class);
+
+ final static Logger operationLog =
+ LogFactory.getLogger(LogCategory.OPERATION, FileRenamer.class);
+
+ /**
+ * Renames given <var>sourceFile</var> to given <var>destinationFile</var>.
+ * <p>
+ * Internally uses {@link FileOperations} and notifies the administrator if the process
+ * failed.
+ * </p>
+ */
+ static final boolean renameAndLog(final File sourceFile, final File destinationFile)
+ {
+ final String absoluteTargetPath = destinationFile.getAbsolutePath();
+ if (destinationFile.exists())
+ {
+ notificationLog.error(String
+ .format("Destination file '%s' already exists. Won't overwrite it.",
+ absoluteTargetPath));
+ return false;
+ }
+ boolean renamedOK =
+ FileOperations.getMonitoredInstanceForCurrentThread().rename(sourceFile,
+ destinationFile);
+ if (renamedOK)
+ {
+ if (operationLog.isInfoEnabled())
+ {
+ final String entity = sourceFile.isDirectory() ? "directory" : "file";
+ final String name = sourceFile.getName();
+ final String parent = sourceFile.getParent();
+ final String path = destinationFile.getParent();
+ operationLog.info(String.format("Moving %s '%s' from '%s' to '%s'.", entity, name,
+ parent, path));
+ }
+ return true;
+ } else
+ {
+ notificationLog.error(String.format("Moving '%s' to '%s' failed, giving up.",
+ sourceFile.getAbsolutePath(), absoluteTargetPath));
+ return false;
+ }
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/HCSImageCheckList.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/HCSImageCheckList.java
new file mode 100644
index 00000000000..ec86d0e08a4
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/HCSImageCheckList.java
@@ -0,0 +1,173 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import ch.systemsx.cisd.bds.hcs.Geometry;
+import ch.systemsx.cisd.bds.hcs.Location;
+import ch.systemsx.cisd.common.utilities.AbstractHashable;
+
+/**
+ * Helper class to set the <code>is_complete</code> flag in the <i>BDS</i> library.
+ * <p>
+ * All the possible combinations are computed in the constructor. This class is also able to spot
+ * images which have already been handled.
+ * </p>
+ *
+ * @author Franz-Josef Elmer
+ */
+final class HCSImageCheckList
+{
+
+ private final List<Map<FullLocation, Check>> list;
+
+ HCSImageCheckList(final int numberOfChannels, final Geometry plateGeometry,
+ final Geometry wellGeometry)
+ {
+ if (numberOfChannels < 1)
+ {
+ throw new IllegalArgumentException("Number of channels smaller than one.");
+ }
+ if (plateGeometry == null)
+ {
+ throw new IllegalArgumentException("Unspecified plate geometry.");
+ }
+ if (wellGeometry == null)
+ {
+ throw new IllegalArgumentException("Unspecified well geometry.");
+ }
+ list = new ArrayList<Map<FullLocation, Check>>();
+ for (int i = 0; i < numberOfChannels; i++)
+ {
+ final Map<FullLocation, Check> map = new HashMap<FullLocation, Check>();
+ for (int plateX = 1; plateX <= plateGeometry.getColumns(); plateX++)
+ {
+ for (int plateY = 1; plateY <= plateGeometry.getRows(); plateY++)
+ {
+ final Location wellLocation = new Location(plateX, plateY);
+ for (int wellX = 1; wellX <= wellGeometry.getColumns(); wellX++)
+ {
+ for (int wellY = 1; wellY <= wellGeometry.getRows(); wellY++)
+ {
+ final Location tileLocation = new Location(wellX, wellY);
+ map.put(new FullLocation(wellLocation, tileLocation), new Check());
+ }
+ }
+ }
+ }
+ assert map.size() == plateGeometry.getColumns() * plateGeometry.getRows()
+ * wellGeometry.getColumns() * wellGeometry.getRows() : "Wrong map size";
+ list.add(map);
+ }
+ }
+
+ final void checkOff(final int channel, final Location wellLocation, final Location tileLocation)
+ {
+ assert wellLocation != null : "Unspecified well location.";
+ assert tileLocation != null : "Unspecified tile location.";
+ if (channel < 1)
+ {
+ throw new IllegalArgumentException("Not a positive channel number: " + channel);
+ }
+ if (channel > list.size())
+ {
+ throw new IllegalArgumentException("Channel number to large: " + channel + " > "
+ + list.size());
+ }
+ final Map<FullLocation, Check> map = list.get(channel - 1);
+ final Check check = map.get(new FullLocation(wellLocation, tileLocation));
+ if (check == null)
+ {
+ throw new IllegalArgumentException("Invalid well/tile location: " + wellLocation);
+ }
+ if (check.isCheckedOff())
+ {
+ throw new IllegalArgumentException("Image already handle for channel" + channel
+ + ", well:" + wellLocation + " tile:" + tileLocation);
+ }
+ check.checkOff();
+ }
+
+ final List<FullLocation> getCheckedOnFullLocations()
+ {
+ final List<FullLocation> fullLocations = new ArrayList<FullLocation>();
+ for (final Map<FullLocation, Check> map : list)
+ {
+ for (final Map.Entry<FullLocation, Check> entry : map.entrySet())
+ {
+ if (entry.getValue().isCheckedOff() == false)
+ {
+ fullLocations.add(entry.getKey());
+ }
+ }
+ }
+ return fullLocations;
+ }
+
+ //
+ // Helper classes
+ //
+
+ private static final class Check
+ {
+ private boolean checkedOff;
+
+ final void checkOff()
+ {
+ checkedOff = true;
+ }
+
+ final boolean isCheckedOff()
+ {
+ return checkedOff;
+ }
+ }
+
+ final static class FullLocation extends AbstractHashable
+ {
+
+ final Location wellLocation;
+
+ final Location tileLocation;
+
+ FullLocation(final Location wellLocation, final Location tileLocation)
+ {
+ this.wellLocation = wellLocation;
+ this.tileLocation = tileLocation;
+ }
+
+ private final static String toString(final Location location, final String type)
+ {
+ return type + "=" + location;
+ }
+
+ //
+ // AbstractHashable
+ //
+
+ @Override
+ public final String toString()
+ {
+ return "[" + toString(wellLocation, "well") + "," + toString(tileLocation, "tile")
+ + "]";
+ }
+ }
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/HCSImageFileExtractionResult.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/HCSImageFileExtractionResult.java
new file mode 100644
index 00000000000..4294eaaa295
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/HCSImageFileExtractionResult.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ch.systemsx.cisd.etlserver;
+
+import java.util.List;
+import java.util.Set;
+
+import ch.systemsx.cisd.bds.hcs.Channel;
+import ch.systemsx.cisd.bds.storage.IFile;
+
+/**
+ * Class which contains the extraction process results.
+ */
+public final class HCSImageFileExtractionResult
+{
+
+ /** The duration of the process. */
+ private final long duration;
+
+ /** The total number of files found. */
+ private final int totalFiles;
+
+ /** The invalid files found. */
+ private final List<IFile> invalidFiles;
+
+ /** The channels found. */
+ private final Set<Channel> channels;
+
+ public HCSImageFileExtractionResult(final long duration, final int totalFiles,
+ final List<IFile> invalidFiles, final Set<Channel> channels)
+ {
+ this.duration = duration;
+ this.totalFiles = totalFiles;
+ this.invalidFiles = invalidFiles;
+ this.channels = channels;
+ }
+
+ /**
+ * Returns the duration of the process.
+ */
+ public final long getDuration()
+ {
+ return duration;
+ }
+
+ /**
+ * Returns the total number of files found.
+ */
+ public final int getTotalFiles()
+ {
+ return totalFiles;
+ }
+
+ /**
+ * Returns the invalid files found.
+ */
+ public final List<IFile> getInvalidFiles()
+ {
+ return invalidFiles;
+ }
+
+ /**
+ * Returns the channels found.
+ */
+ public final Set<Channel> getChannels()
+ {
+ return channels;
+ }
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/IDataSetInfoExtractor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IDataSetInfoExtractor.java
new file mode 100644
index 00000000000..bbabefb64ba
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IDataSetInfoExtractor.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.io.File;
+
+import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.common.exceptions.UserFailureException;
+
+/**
+ * A role to extract {@link DataSetInformation} from an incoming data set. Implementations of this
+ * interface are expected to have a constructor taking a {@link java.util.Properties} object as
+ * their only argument. The properties can be used to get further arguments that the extractor
+ * implementation requires to function.
+ * <p>
+ * The usage mode of implementations <var>extractorClassName</var>s is:
+ *
+ * <pre>
+ * Properties props = <get some props from somewhere>
+ * Class clazz = Class.forName(extractorClassName);
+ * IIDExtractor extractor = clazz.getConstructor(new Class[] { Properties.class } ).newInstance(new Object[] { props });
+ * DataSetInformation info = extractor.getDataSetInformation(incomingDataSetPath);
+ * String experimentCode = info.getExperimentCode();
+ * String dataSetCode = info.getSampleCode();
+ * </pre>
+ *
+ * Implementations of this class are expected to be "re-usable". This is, calling the method
+ * {@link #getDataSetInformation(File)} multiple times for different data set on the same instance
+ * is expected to work.
+ *
+ * @author Bernd Rinn
+ */
+public interface IDataSetInfoExtractor
+{
+
+ /** Properties key prefix for the extractor. */
+ public static final String EXTRACTOR_KEY = "data-set-info-extractor";
+
+ /**
+ * Extracts data set information from the specified path of the incoming data set.
+ * <p>
+ * <i>Note that <code>incomingDataSetPath.getParent()</code> is arbitrary and the extracted id
+ * and code must not depend on it!</i>
+ * </p>
+ *
+ * @param incomingDataSetPath The path of the incoming data set. The path may be a file or
+ * directory. The caller needs to ensure that the path exists when this method is
+ * called.
+ * @return The information extracted about this data set. The code extractor <i>can</i>, but
+ * <i>doesn't have to</i> provide an group. If no group has been provided by the
+ * extractor then the one specified for the thread in the
+ * <code>service.properties</code> file (if any) will be taken. Never returns
+ * <code>null</code>.
+ * @throws UserFailureException If the incoming data set does not meet the expectations and thus
+ * the extractor can't extract either the experiment id or the data set code or
+ * both.
+ */
+ public DataSetInformation getDataSetInformation(final File incomingDataSetPath)
+ throws UserFailureException, EnvironmentFailureException;
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/IDataStoreStrategy.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IDataStoreStrategy.java
new file mode 100644
index 00000000000..1ecea806c76
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IDataStoreStrategy.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.io.File;
+
+import ch.systemsx.cisd.openbis.generic.shared.dto.DataSetType;
+
+/**
+ * This interface implements a strategy for storing data in the <i>store</i> directory.
+ *
+ * @author Christian Ribeaud
+ */
+interface IDataStoreStrategy
+{
+
+ /**
+ * Returns the key associated with this <code>IDataStoreStrategy</code>.
+ * <p>
+ * This key uniquely identifies this <code>IDataStoreStrategy</code>.
+ * </p>
+ */
+ public DataStoreStrategyKey getKey();
+
+ /**
+ * Returns the base directory where the data are going to be moved into.
+ */
+ public File getBaseDirectory(final File baseDirectory, final DataSetInformation dataSetInfo,
+ final DataSetType dataSetType);
+
+ /**
+ * Create the target path for given <var>baseDirectory</var> and given <var>incomingDataSetPath</var>.
+ * <p>
+ * Note that each call either produces a new <i>target path</i> or throws an exception if
+ * computed <i>target path</i> already exists.
+ * </p>
+ *
+ * @return The target path.
+ */
+ public File getTargetPath(final File baseDirectory, final File incomingDataSetPath);
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/IDataStrategyStore.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IDataStrategyStore.java
new file mode 100644
index 00000000000..c88817c6ecb
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IDataStrategyStore.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.io.File;
+
+/**
+ * The main purpose of this interface is to return a <code>IDataStoreStrategy</code> for a given
+ * <code>DataSetInformation</code>.
+ * <p>
+ * To perform its job it might use some helpers defined in the constructor.
+ * </p>
+ *
+ * @author Christian Ribeaud
+ */
+interface IDataStrategyStore
+{
+
+ /**
+ * For given <var>dataSetInfo</var> and given <var>incomingDataSetPath</var> returns the
+ * corresponding <code>IDataStoreStrategy</code>. As a side effect sets also the sample
+ * properties, the experiment, and if not already set, the experiment identifier.
+ *
+ * @param dataSetInfo The data set information, gets enriched in the process.
+ * @param incomingDataSetPath mainly used for logging purposes.
+ */
+ public IDataStoreStrategy getDataStoreStrategy(final DataSetInformation dataSetInfo,
+ final File incomingDataSetPath);
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/IETLServerPlugin.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IETLServerPlugin.java
new file mode 100644
index 00000000000..e31d9dfe070
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IETLServerPlugin.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+
+/**
+ * Plugin interface of the ETL Server.
+ *
+ * @author Franz-Josef Elmer
+ */
+public interface IETLServerPlugin
+{
+ /**
+ * Returns the code extractor.
+ */
+ public IDataSetInfoExtractor getDataSetInfoExtractor();
+
+ /**
+ * Returns the procedure and data type extractor.
+ */
+ public IProcedureAndDataTypeExtractor getTypeExtractor();
+
+ /**
+ * Returns the {@link IStorageProcessor} implementation.
+ */
+ public IStorageProcessor getStorageProcessor();
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/IEncapsulatedLimsService.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IEncapsulatedLimsService.java
new file mode 100644
index 00000000000..e61b7a7958c
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IEncapsulatedLimsService.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import ch.systemsx.cisd.common.exceptions.UserFailureException;
+import ch.systemsx.cisd.openbis.generic.shared.IETLLIMSService;
+import ch.systemsx.cisd.openbis.generic.shared.dto.DatabaseInstancePE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ExperimentPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ExternalData;
+import ch.systemsx.cisd.openbis.generic.shared.dto.SamplePropertyPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifier;
+
+/**
+ * This interface is very similar to {@link IETLLIMSService} but <code>sessionToken</code> has
+ * been removed from each method.
+ *
+ * @see IETLLIMSService
+ * @author Christian Ribeaud
+ */
+interface IEncapsulatedLimsService
+{
+ /**
+ * For given <var>dataSetInfo</var> returns the <code>BaseExperiment</code> object.
+ */
+ public ExperimentPE getBaseExperiment(final SampleIdentifier sampleIdentifier)
+ throws UserFailureException;
+
+ /**
+ * Registers the specified data.
+ * <p>
+ * As side effect, sets <i>data set code</i> in {@link DataSetInformation#getExtractableData()}.
+ * </p>
+ */
+ public void registerDataSet(final DataSetInformation dataSetInformation,
+ final String procedureTypeCode, final ExternalData data) throws UserFailureException;
+
+ /**
+ * Tries to return the properties of the top sample (e.g. master plate) registered for the
+ * specified sample identifier.
+ *
+ * @return <code>null</code> if no appropriated sample found. Returns an empty array if a a
+ * sample found with no properties.
+ */
+ public SamplePropertyPE[] getPropertiesOfTopSampleRegisteredFor(
+ final SampleIdentifier sampleIdentifier) throws UserFailureException;
+
+ /**
+ * Creates and returns a unique code for a new data set.
+ */
+ public String createDataSetCode();
+
+ /**
+ * Returns the version of the service.
+ */
+ public int getVersion();
+
+ /**
+ * Returns the home database instance.
+ */
+ public DatabaseInstancePE getHomeDatabaseInstance();
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/IFile.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IFile.java
new file mode 100644
index 00000000000..3e0afa9ee73
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IFile.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.io.File;
+
+import ch.systemsx.cisd.common.utilities.ISelfTestable;
+
+/**
+ * A file abstraction.
+ *
+ * @author Franz-Josef Elmer
+ */
+public interface IFile extends ISelfTestable
+{
+ /**
+ * Returns the absolute path of this file.
+ */
+ public String getAbsolutePath();
+
+ /**
+ * Copies the file denoted by this abstract pathname to given <var>destinationFile</var>.
+ * <p>
+ * Note that, depending on the implementation, it effectively copies this abstract pathname or
+ * makes an hard link of it.
+ * </p>
+ */
+ public void copyTo(File destinationFile);
+
+ /**
+ * Copies given <code>sourceFile</code> to the file denoted by this abstract pathname.
+ * <p>
+ * Note that, depending on the implementation, it effectively copies the given <var>sourceFile</var>
+ * or makes an hard link of it.
+ * </p>
+ */
+ public void copyFrom(File sourceFile);
+
+ public byte[] read();
+
+ public void write(byte[] data);
+
+ public void delete();
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/IFileFactory.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IFileFactory.java
new file mode 100644
index 00000000000..6fe516bddf0
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IFileFactory.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+/**
+ * @author Franz-Josef Elmer
+ */
+public interface IFileFactory
+{
+
+ /** Creates given <var>path</var>. */
+ public IFile create(String path);
+
+ /** Creates given <var>relativePath</var> in given <var>baseDir</var>. */
+ public IFile create(IFile baseDir, String relativePath);
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/IHCSImageFileAccepter.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IHCSImageFileAccepter.java
new file mode 100644
index 00000000000..1dbeac69dc5
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IHCSImageFileAccepter.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ch.systemsx.cisd.etlserver;
+
+import ch.systemsx.cisd.bds.hcs.Location;
+import ch.systemsx.cisd.bds.storage.IFile;
+
+/**
+ * Role that is implemented by <i>ETL Server</i> core system.
+ * <p>
+ * It is called by the <code>IHCSImageFileExtractor</code> implementation as callback to register
+ * an image file at given coordinates.
+ * </p>
+ *
+ * @author Christian Ribeaud
+ */
+public interface IHCSImageFileAccepter
+{
+ /**
+ * Registers given <var>imageFile</var> at given <code>standard</code> coordinates.
+ */
+ public void accept(final int channel, final Location wellLocation, final Location tileLocation,
+ final IFile imageFile);
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/IHCSImageFileExtractor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IHCSImageFileExtractor.java
new file mode 100644
index 00000000000..c412b147add
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IHCSImageFileExtractor.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.util.Properties;
+
+import ch.systemsx.cisd.bds.storage.IDirectory;
+
+/**
+ * This role is supposed to be implemented by classes that can extract HCS image files from an
+ * incoming data set directory. Implementations of this interface need to have a constructor that
+ * takes {@link Properties} to initialize itself.
+ *
+ * @author Christian Ribeaud
+ */
+public interface IHCSImageFileExtractor
+{
+
+ public final static String FILE_EXTRACTOR = "file-extractor";
+
+ /**
+ * Extracts <code>StandardCoordinates</code> in the given <var>incomingDataSetDirectory</var>
+ * and for the given <var>dataSetInfo</var> and hands the image files that it finds over to the
+ * specified <var>accepter</var>.
+ *
+ * @return the extraction result. Must not be <code>null</code>.
+ */
+ public HCSImageFileExtractionResult process(final IDirectory incomingDataSetDirectory,
+ DataSetInformation dataSetInformation, final IHCSImageFileAccepter accepter);
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/IProcedureAndDataTypeExtractor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IProcedureAndDataTypeExtractor.java
new file mode 100644
index 00000000000..b305aeb45f3
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IProcedureAndDataTypeExtractor.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.io.File;
+
+import ch.systemsx.cisd.openbis.generic.shared.dto.DataSetType;
+import ch.systemsx.cisd.openbis.generic.shared.dto.FileFormatType;
+import ch.systemsx.cisd.openbis.generic.shared.dto.LocatorType;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ProcedureType;
+
+/**
+ * Extractor for procedure, data set, file format, and locator type.
+ *
+ * @author Franz-Josef Elmer
+ */
+public interface IProcedureAndDataTypeExtractor
+{
+ /** Properties key prefix for the type extractor. */
+ public static final String TYPE_EXTRACTOR_KEY = "type-extractor";
+
+ /**
+ * Gets the procedure type from the specified path of the incoming data set.
+ */
+ public ProcedureType getProcedureType(File incomingDataSetPath);
+
+ /**
+ * Gets the data set type from the specified path of the incoming data set.
+ */
+ public DataSetType getDataSetType(File incomingDataSetPath);
+
+ /**
+ * Gets the file format type from the specified path of the incoming data set.
+ */
+ public FileFormatType getFileFormatType(File incomingDataSetPath);
+
+ /**
+ * Gets the locator type from the specified path of the incoming data set.
+ */
+ public LocatorType getLocatorType(File incomingDataSetPath);
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/IProcessor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IProcessor.java
new file mode 100644
index 00000000000..662d1a5ca49
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IProcessor.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.io.File;
+
+import ch.systemsx.cisd.openbis.generic.shared.dto.ProcessingInstructionDTO;
+import ch.systemsx.cisd.openbis.generic.shared.dto.StorageFormat;
+
+/**
+ * Interface for processing a data set file or folder after registration.
+ *
+ * @author Franz-Josef Elmer
+ */
+public interface IProcessor
+{
+ /**
+ * Returns the required format of the input data used by this processor.
+ */
+ public StorageFormat getRequiredInputDataFormat();
+
+ /**
+ * Initiates processing the <var>dataSet</var> with the specified processing instructions.
+ */
+ public void initiateProcessing(final ProcessingInstructionDTO instructionOrNull,
+ final DataSetInformation dataSetInformation, final File dataSet);
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/IProcessorFactory.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IProcessorFactory.java
new file mode 100644
index 00000000000..586cddae88d
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IProcessorFactory.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import ch.systemsx.cisd.common.filesystem.PathPrefixPrepender;
+
+/**
+ * Factory for {@link IProcessor} instances.
+ *
+ * @author Franz-Josef Elmer
+ */
+public interface IProcessorFactory
+{
+ /**
+ * Creates a new processor.
+ */
+ public IProcessor createProcessor();
+
+ /**
+ * Returns the {@link PathPrefixPrepender}.
+ */
+ public PathPrefixPrepender getPathPrefixPrepender();
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/IStorageProcessor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IStorageProcessor.java
new file mode 100644
index 00000000000..8d8dc3c4292
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IStorageProcessor.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.io.File;
+import java.util.Properties;
+
+import ch.systemsx.cisd.common.mail.IMailClient;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ExperimentPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.StorageFormat;
+
+/**
+ * Takes care of storing the data in the store root directory.
+ * <p>
+ * Implementations of this interface are expected to have a constructor taking a {@link Properties}
+ * object as their only argument.
+ * </p>
+ *
+ * @author Christian Ribeaud
+ */
+public interface IStorageProcessor extends IStoreRootDirectoryHolder
+{
+ /** Properties key prefix to find the {@link IStorageProcessor} implementation. */
+ public static final String STORAGE_PROCESSOR_KEY = "storage-processor";
+
+ /**
+ * Stores the specified incoming data set file to the specified directory. In general some
+ * processing and/or transformation of the incoming data takes place.
+ * <p>
+ * Do not try/catch exceptions that could occur here. Preferably let the upper layer handle
+ * them.
+ * </p>
+ *
+ * @param experiment information about the related experiment.
+ * @param dataSetInformation Information about the data set.
+ * @param typeExtractor the {@link IProcedureAndDataTypeExtractor} implementation.
+ * @param mailClient mail client.
+ * @param incomingDataSetDirectory folder to store. Do not remove it after the implementation
+ * has finished processing. {@link TransferredDataSetHandler} takes care of this.
+ * @param rootDir directory to whom the data will be stored.
+ * @return folder which contains the stored data. This folder <i>must</i> be below the
+ * <var>rootDir</var>. Never returns <code>null</code> but prefers to throw an
+ * exception in case of unexpected behavior.
+ */
+ public File storeData(final ExperimentPE experiment,
+ final DataSetInformation dataSetInformation,
+ final IProcedureAndDataTypeExtractor typeExtractor, final IMailClient mailClient,
+ final File incomingDataSetDirectory, final File rootDir);
+
+ /**
+ * Performs a rollback of
+ * {@link #storeData(ExperimentPE, DataSetInformation, IProcedureAndDataTypeExtractor, IMailClient, File, File)}
+ * The data created in <code>directory</code> will also be removed.
+ * <p>
+ * Call to this method is safe as implementations should try/catch exceptions that could occur
+ * here.
+ * </p>
+ *
+ * @param incomingDataSetDirectory original folder to be restored.
+ * @param storedDataDirectory directory which contains the data to be restored.
+ */
+ public void unstoreData(final File incomingDataSetDirectory, final File storedDataDirectory);
+
+ /**
+ * Returns the format that this storage processor is storing data sets in.
+ */
+ public StorageFormat getStorageFormat();
+
+ /**
+ * Returns the data set in the original proprietary format (before being processed) if
+ * available, or <code>null</code>, if the original data set is no longer available.
+ * <p>
+ * <strong>Consider the data in the returned file / directory read only!</strong>
+ */
+ public File tryGetProprietaryData(final File storedDataDirectory);
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/IStoreRootDirectoryHolder.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IStoreRootDirectoryHolder.java
new file mode 100644
index 00000000000..701da320a64
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IStoreRootDirectoryHolder.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.io.File;
+
+/**
+ * Implementations of this interface specifies the location of the store root directory.
+ *
+ * @author Christian Ribeaud
+ */
+public interface IStoreRootDirectoryHolder
+{
+
+ /**
+ * Returns the store root directory.
+ * <p>
+ * Note that this method does not call {@link File#mkdirs()} on the returned path.
+ * </p>
+ */
+ public File getStoreRootDirectory();
+
+ /**
+ * Sets the store root directory.
+ */
+ public void setStoreRootDirectory(final File storeRootDirectory);
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/IdentifiedDataStrategy.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IdentifiedDataStrategy.java
new file mode 100644
index 00000000000..98dc811072e
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IdentifiedDataStrategy.java
@@ -0,0 +1,180 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.io.File;
+
+import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.openbis.generic.shared.dto.DataSetType;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.ExperimentIdentifier;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifier;
+
+/**
+ * This <code>IDataStoreStrategy</code> implementation if for data set that has been <i>identified</i>,
+ * meaning that kind of connection to this data set could be found in the database (through the
+ * derived <i>Master Plate</i> or through the experiment specified).
+ *
+ * @author Christian Ribeaud
+ */
+public final class IdentifiedDataStrategy implements IDataStoreStrategy
+{
+ static final String DATA_SET_TYPE_PREFIX = "DataSetType_";
+
+ static final String SAMPLE_PREFIX = "Sample_";
+
+ static final String EXPERIMENT_PREFIX = "Experiment_";
+
+ static final String PROJECT_PREFIX = "Project_";
+
+ static final String GROUP_PREFIX = "Group_";
+
+ static final String INSTANCE_PREFIX = "Instance_";
+
+ public static final String DATASET_PREFIX = "Dataset_";
+
+ static final String UNEXPECTED_PATHS_MSG_FORMAT =
+ "There are unexpected paths '%s' in data store '%s'. I'll proceed anyway.";
+
+ static final String STORAGE_LAYOUT_ERROR_MSG_PREFIX = "Serious error in data store layout: ";
+
+ IdentifiedDataStrategy()
+ {
+
+ }
+
+ private static String createInstanceDirectory(final DataSetInformation dataSetInfo)
+ {
+ final String instanceUUID = dataSetInfo.getInstanceUUID();
+ assert instanceUUID != null : "Instance UUID can not be null.";
+ return INSTANCE_PREFIX + instanceUUID;
+ }
+
+ private static String createGroupDirectory(final DataSetInformation dataSetInfo)
+ {
+ final ExperimentIdentifier identifier = dataSetInfo.getExperimentIdentifier();
+ assert identifier != null : "Identifier can not be null.";
+ final String groupCode = identifier.getGroupCode();
+ assert groupCode != null : "Group code can not be null.";
+ return GROUP_PREFIX + groupCode;
+ }
+
+ private static String createProjectDirectory(final DataSetInformation dataSetInfo)
+ {
+ final ExperimentIdentifier identifier = dataSetInfo.getExperimentIdentifier();
+ assert identifier != null : "Identifier can not be null.";
+ final String projectCode = identifier.getProjectCode();
+ assert projectCode != null : "Project code can not be null.";
+ return PROJECT_PREFIX + projectCode;
+ }
+
+ private static String createExperimentDirectory(final DataSetInformation dataSetInfo)
+ {
+ final ExperimentIdentifier identifier = dataSetInfo.getExperimentIdentifier();
+ assert identifier != null : "Identifier can not be null.";
+ final String experimentCode = identifier.getExperimentCode();
+ assert experimentCode != null : "Experiment code can not be null.";
+ return EXPERIMENT_PREFIX + experimentCode;
+ }
+
+ private final static String createSampleDirectory(final DataSetInformation dataSetInfo)
+ {
+ final SampleIdentifier sampleIdentifier = dataSetInfo.getSampleIdentifier();
+ assert sampleIdentifier != null : "Sample identifier can not be null.";
+ return SAMPLE_PREFIX + sampleIdentifier.getSampleCode();
+ }
+
+ private static String createDatasetDirectory(final DataSetInformation dataSetInfo)
+ {
+ final String dataSetCode = dataSetInfo.getDataSetCode();
+ assert dataSetCode != null : "Dataset code con not be null.";
+ return DATASET_PREFIX + dataSetCode;
+ }
+
+ final static String createDataSetTypeDirectory(final DataSetType dataSetType)
+ {
+ final String dataSetTypeCode = dataSetType.getCode();
+ assert dataSetTypeCode != null : "Data set type code can not be null.";
+ return DATA_SET_TYPE_PREFIX + dataSetTypeCode;
+ }
+
+ /**
+ * Computes the base directory with given <var>baseDir</var> and given <var>dataSetInfo</var>
+ * and returns it as <code>File</code>.
+ * <p>
+ * Note that this method does not call {@link File#mkdirs()} on returned <code>File</code>.
+ * </p>
+ */
+ private final static File createBaseDirectory(final File baseDir,
+ final DataSetInformation dataSetInfo, final DataSetType dataSetType)
+ {
+ final File instanceDir = new File(baseDir, createInstanceDirectory(dataSetInfo));
+ final File groupDir = new File(instanceDir, createGroupDirectory(dataSetInfo));
+ final File projectDir = new File(groupDir, createProjectDirectory(dataSetInfo));
+ final File experimentDir = new File(projectDir, createExperimentDirectory(dataSetInfo));
+ final File dataSetTypeDir =
+ new File(experimentDir, createDataSetTypeDirectory(dataSetType));
+ final File sampleDir = new File(dataSetTypeDir, createSampleDirectory(dataSetInfo));
+ final File datasetDir = new File(sampleDir, createDatasetDirectory(dataSetInfo));
+ return datasetDir;
+
+ }
+
+ //
+ // IDataStoreStrategy
+ //
+
+ public final DataStoreStrategyKey getKey()
+ {
+ return DataStoreStrategyKey.IDENTIFIED;
+ }
+
+ public final File getBaseDirectory(final File storeRoot, final DataSetInformation dataSetInfo,
+ final DataSetType dataSetType)
+ {
+ assert storeRoot != null : "Store root can not be null";
+ assert dataSetInfo != null : "Data set information can not be null";
+ assert dataSetType != null : "Data set type can not be null";
+ final File baseDirectory = createBaseDirectory(storeRoot, dataSetInfo, dataSetType);
+ if (baseDirectory.exists())
+ {
+ throw EnvironmentFailureException.fromTemplate(STORAGE_LAYOUT_ERROR_MSG_PREFIX
+ + "Data set directory '%s' exists but has been designed to be unique.",
+ baseDirectory.getPath());
+ }
+ if (baseDirectory.isFile())
+ {
+ throw EnvironmentFailureException.fromTemplate(STORAGE_LAYOUT_ERROR_MSG_PREFIX
+ + "Base directory '%s' is a file.", baseDirectory);
+ }
+ return baseDirectory;
+ }
+
+ public final File getTargetPath(final File baseDirectory, final File incomingDataSetPath)
+ throws IllegalStateException
+ {
+ assert baseDirectory != null : "Base directory can not be null";
+ assert incomingDataSetPath != null : "Incoming data set can not be null";
+ final File targetPath = new File(baseDirectory, incomingDataSetPath.getName());
+ if (targetPath.exists())
+ {
+ throw new IllegalStateException(String.format(
+ "Target path '%s' of identified incoming data set already exists "
+ + "(which it shouldn't), bailing out.", targetPath.getAbsolutePath()));
+ }
+ return targetPath;
+ }
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/Main.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/Main.java
new file mode 100644
index 00000000000..f573e571829
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/Main.java
@@ -0,0 +1,497 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.lang.Thread.UncaughtExceptionHandler;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Timer;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.filefilter.FileFilterUtils;
+import org.apache.commons.io.filefilter.NameFileFilter;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+import ch.rinn.restrictions.Private;
+import ch.systemsx.cisd.common.Constants;
+import ch.systemsx.cisd.common.TimingParameters;
+import ch.systemsx.cisd.common.concurrent.TimerUtilities;
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.common.exceptions.HighLevelException;
+import ch.systemsx.cisd.common.exceptions.StopException;
+import ch.systemsx.cisd.common.filesystem.DirectoryScanningTimerTask;
+import ch.systemsx.cisd.common.filesystem.FaultyPathDirectoryScanningHandler;
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
+import ch.systemsx.cisd.common.filesystem.IDirectoryScanningHandler;
+import ch.systemsx.cisd.common.filesystem.PathPrefixPrepender;
+import ch.systemsx.cisd.common.filesystem.QueueingPathRemoverService;
+import ch.systemsx.cisd.common.highwatermark.HighwaterMarkDirectoryScanningHandler;
+import ch.systemsx.cisd.common.highwatermark.HighwaterMarkWatcher;
+import ch.systemsx.cisd.common.highwatermark.HostAwareFileWithHighwaterMark;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.logging.LogInitializer;
+import ch.systemsx.cisd.common.spring.HttpInvokerUtils;
+import ch.systemsx.cisd.common.utilities.BuildAndEnvironmentInfo;
+import ch.systemsx.cisd.common.utilities.IExitHandler;
+import ch.systemsx.cisd.common.utilities.ISelfTestable;
+import ch.systemsx.cisd.common.utilities.IStopSignaler;
+import ch.systemsx.cisd.common.utilities.PropertyUtils;
+import ch.systemsx.cisd.common.utilities.SystemExit;
+import ch.systemsx.cisd.openbis.generic.shared.IETLLIMSService;
+import ch.systemsx.cisd.openbis.generic.shared.IWebService;
+import ch.systemsx.cisd.openbis.generic.shared.dto.DatabaseInstancePE;
+
+/**
+ * The main class of the ETL server.
+ *
+ * @author Bernd Rinn
+ */
+public final class Main
+{
+ static final String STOREROOT_DIR_KEY = "storeroot-dir";
+
+ static final String NOTIFY_SUCCESSFUL_REGISTRATION = "notify-successful-registration";
+
+ private static final Logger operationLog =
+ LogFactory.getLogger(LogCategory.OPERATION, Main.class);
+
+ private static final Logger notificationLog =
+ LogFactory.getLogger(LogCategory.NOTIFY, Main.class);
+
+ private static final File shredderQueueFile = new File(".shredder");
+
+ private static final UncaughtExceptionHandler loggingExceptionHandler =
+ new UncaughtExceptionHandler()
+ {
+
+ //
+ // UncaughtExceptionHandler
+ //
+
+ public final void uncaughtException(final Thread t, final Throwable e)
+ {
+ notificationLog.error("An exception has occurred [thread: '" + t.getName()
+ + "'].", e);
+ }
+ };
+
+ @Private
+ static IExitHandler exitHandler = SystemExit.SYSTEM_EXIT;
+
+ private static void initLog()
+ {
+ LogInitializer.init();
+ Thread.setDefaultUncaughtExceptionHandler(loggingExceptionHandler);
+ }
+
+ private static void printInitialLogMessage(final Parameters parameters)
+ {
+ operationLog.info("Etlserver is starting up.");
+ for (final String line : BuildAndEnvironmentInfo.INSTANCE.getEnvironmentInfo())
+ {
+ operationLog.info(line);
+ }
+ parameters.log();
+ }
+
+ private static boolean checkListShredder(final String[] args)
+ {
+ if (args.length > 0 && args[0].equals("--show-shredder"))
+ {
+ final List<File> shredderItems =
+ QueueingPathRemoverService.listShredderItems(shredderQueueFile);
+ if (shredderItems.isEmpty())
+ {
+ System.out.println("Shredder is empty.");
+ } else
+ {
+ System.out.println("Found " + shredderItems.size() + " items in shredder:");
+ for (final File f : shredderItems)
+ {
+ System.out.println(f.getAbsolutePath());
+ }
+ }
+ return true;
+ } else
+ {
+ return false;
+ }
+ }
+
+ private static void selfTest(final File incomingDirectory,
+ final IEncapsulatedLimsService service, final ISelfTestable... selfTestables)
+ {
+ final String msgStart = "Etlserver self test failed:";
+ ISelfTestable currentSelfTestableOrNull = null;
+ try
+ {
+ checkFullyAccesible(incomingDirectory);
+ final int serviceVersion = service.getVersion();
+ if (IWebService.VERSION != serviceVersion)
+ {
+ throw new ConfigurationFailureException(
+ "This client has the wrong service version for the server (client: "
+ + IWebService.VERSION + ", server: " + serviceVersion + ").");
+ }
+ for (final ISelfTestable selfTestableOrNull : selfTestables)
+ {
+ if (selfTestableOrNull != null)
+ {
+ currentSelfTestableOrNull = selfTestableOrNull;
+ selfTestableOrNull.check();
+ }
+ }
+ } catch (final HighLevelException e)
+ {
+ if (currentSelfTestableOrNull != null && currentSelfTestableOrNull.isRemote())
+ {
+ notificationLog.error("Self test on self-testable "
+ + currentSelfTestableOrNull.getClass().getSimpleName()
+ + " failed. This it relies on a remote resource which might become "
+ + "available at at later time, we keep the server running anyway.", e);
+ } else
+ {
+ System.err.printf(msgStart + " [%s: %s]\n", e.getClass().getSimpleName(), e
+ .getMessage());
+ exitHandler.exit(1);
+ }
+ } catch (final RuntimeException e)
+ {
+ System.err.println(msgStart);
+ e.printStackTrace();
+ exitHandler.exit(1);
+ }
+ if (TimerUtilities.isOperational())
+ {
+ if (operationLog.isInfoEnabled())
+ {
+ operationLog.info("Timer task interruption is operational.");
+ }
+ } else
+ {
+ operationLog.warn("Timer task interruption is not operational. "
+ + "No clean up can be performed on extraordinary shutdown.");
+ }
+
+ }
+
+ private static void checkFullyAccesible(final File directory)
+ throws ConfigurationFailureException
+ {
+ if (operationLog.isDebugEnabled())
+ {
+ operationLog.debug("Checking source directory '" + directory.getAbsolutePath() + "'.");
+ }
+ final String errorMessage =
+ FileUtilities.checkDirectoryFullyAccessible(directory, "source");
+ if (errorMessage != null)
+ {
+ throw new ConfigurationFailureException(errorMessage);
+ }
+ }
+
+ private static IETLLIMSService getETLLIMSService(final Parameters parameters)
+ {
+ final String serviceURL = getServiceURL(parameters) + "/rmi-etl";
+ final IETLLIMSService service = HttpInvokerUtils.createServiceStub(IETLLIMSService.class, serviceURL, 5);
+ return service;
+ }
+
+ private static String getServiceURL(final Parameters parameters)
+ {
+ final String serverURL = parameters.getServerURL();
+ if (serverURL == null)
+ {
+ throw new EnvironmentFailureException("Application Server URL is not defined.");
+ }
+ return serverURL;
+ }
+
+ private static void startupServer(final Parameters parameters)
+ {
+ final Map<String, IProcessorFactory> processorFactories =
+ new LinkedHashMap<String, IProcessorFactory>();
+ final Map<String, Properties> processorProperties = parameters.getProcessorProperties();
+ for (final Map.Entry<String, Properties> entry : processorProperties.entrySet())
+ {
+ processorFactories.put(entry.getKey(), StandardProcessorFactory
+ .create(entry.getValue()));
+ }
+ final ThreadParameters[] threads = parameters.getThreads();
+ final IEncapsulatedLimsService authorizedLimsService =
+ createAuthorizedLimsService(parameters);
+ final Properties properties = parameters.getProperties();
+ final boolean notifySuccessfulRegistration = getNotifySuccessfulRegistration(properties);
+ final HighwaterMarkWatcher highwaterMarkWatcher =
+ new HighwaterMarkWatcher(getHighwaterMark(properties));
+ for (final ThreadParameters threadParameters : threads)
+ {
+ createProcessingThread(parameters, threadParameters, authorizedLimsService,
+ processorFactories, highwaterMarkWatcher, notifySuccessfulRegistration);
+ }
+ }
+
+ private static IEncapsulatedLimsService createAuthorizedLimsService(final Parameters parameters)
+ {
+ final String username = parameters.getUsername();
+ final String password = parameters.getPassword();
+
+ final IETLLIMSService limsService = getETLLIMSService(parameters);
+ return new EncapsulatedLimsService(limsService, username, password);
+ }
+
+ private final static File getStoreRootDir(final Properties properties)
+ {
+ return FileUtilities.normalizeFile(new File(PropertyUtils.getMandatoryProperty(properties,
+ STOREROOT_DIR_KEY)));
+ }
+
+ @Private
+ final static void migrateStoreRootDir(final File storeRootDir,
+ final DatabaseInstancePE databaseInstancePE)
+ {
+ final File[] instanceDirs =
+ storeRootDir.listFiles((FilenameFilter) new NameFileFilter(
+ IdentifiedDataStrategy.INSTANCE_PREFIX + databaseInstancePE.getCode()));
+ final int size = instanceDirs.length;
+ assert size == 0 || size == 1 : "Wrong size of instance directories.";
+ final String absolutePath = storeRootDir.getAbsolutePath();
+ if (size == 0)
+ {
+ operationLog.info(String.format("No instance directory has been renamed "
+ + "in store root directory '%s'.", absolutePath));
+ } else
+ {
+ final File instanceDir = instanceDirs[0];
+ final File newName =
+ new File(storeRootDir, IdentifiedDataStrategy.INSTANCE_PREFIX
+ + databaseInstancePE.getUuid());
+ instanceDir.renameTo(newName);
+ operationLog.info(String.format("Following instance directory '%s' has been "
+ + "renamed to '%s' in store root directory '%s'.", instanceDir.getName(),
+ newName.getName(), absolutePath));
+ }
+ }
+
+ @Private
+ final static List<File> findFiles(final File root, String prefix, int maxDepth)
+ {
+ ArrayList<File> files = new ArrayList<File>();
+ if (maxDepth == 0)
+ {
+ if (root.getName().startsWith(prefix))
+ {
+ files.add(root);
+ }
+ } else
+ {
+ if (root.isDirectory())
+ {
+ for (File file : root.listFiles())
+ {
+ files.addAll(findFiles(file, prefix, maxDepth - 1));
+ }
+ }
+ }
+ return files;
+ }
+
+ @Private
+ final static void migrateDataStoreByRenamingObservableTypeToDataSetType(final File root)
+ {
+ final String observableTypeDirPrefix = "ObservableType_";
+ final String dataSetTypeDirPrefix = "DataSetType_";
+ final String observableTypeFilePrefix = "observable_type";
+ final String dataSetTypeFilePrefix = "data_set_type";
+
+ for (File file : findFiles(root, observableTypeDirPrefix, 5))
+ {
+ final File newName =
+ new File(file.getAbsolutePath().replaceFirst(observableTypeDirPrefix,
+ dataSetTypeDirPrefix));
+ file.renameTo(newName);
+ }
+ for (File file : findFiles(root, observableTypeFilePrefix, 10))
+ {
+ final File newName =
+ new File(file.getAbsolutePath().replaceFirst(observableTypeFilePrefix,
+ dataSetTypeFilePrefix));
+ file.renameTo(newName);
+ }
+ }
+
+ private final static long getHighwaterMark(final Properties properties)
+ {
+ return PropertyUtils.getLong(properties,
+ HostAwareFileWithHighwaterMark.HIGHWATER_MARK_PROPERTY_KEY, -1L);
+ }
+
+ private final static boolean getNotifySuccessfulRegistration(final Properties properties)
+ {
+ return PropertyUtils.getBoolean(properties, NOTIFY_SUCCESSFUL_REGISTRATION, false);
+ }
+
+ private final static void createProcessingThread(final Parameters parameters,
+ final ThreadParameters threadParameters,
+ final IEncapsulatedLimsService authorizedLimsService,
+ final Map<String, IProcessorFactory> processorFactories,
+ final HighwaterMarkWatcher highwaterMarkWatcher,
+ final boolean notifySuccessfulRegistration)
+ {
+ final File incomingDataDirectory = threadParameters.getIncomingDataDirectory();
+ final IETLServerPlugin plugin = threadParameters.getPlugin();
+ final Properties properties = parameters.getProperties();
+ final File storeRootDir = getStoreRootDir(properties);
+ migrateStoreRootDir(storeRootDir, authorizedLimsService.getHomeDatabaseInstance());
+ migrateDataStoreByRenamingObservableTypeToDataSetType(storeRootDir);
+ plugin.getStorageProcessor().setStoreRootDirectory(storeRootDir);
+ final Properties mailProperties = parameters.getMailProperties();
+ final TransferredDataSetHandler pathHandler =
+ new TransferredDataSetHandler(threadParameters.tryGetGroupCode(), plugin,
+ authorizedLimsService, mailProperties, highwaterMarkWatcher,
+ notifySuccessfulRegistration);
+ pathHandler.setProcessorFactories(processorFactories);
+ final HighwaterMarkDirectoryScanningHandler directoryScanningHandler =
+ createDirectoryScanningHandler(pathHandler, highwaterMarkWatcher,
+ incomingDataDirectory, storeRootDir, processorFactories.values());
+ final DirectoryScanningTimerTask dataMonitorTask =
+ new DirectoryScanningTimerTask(incomingDataDirectory, FileFilterUtils
+ .prefixFileFilter(Constants.IS_FINISHED_PREFIX), pathHandler,
+ directoryScanningHandler);
+ selfTest(incomingDataDirectory, authorizedLimsService, pathHandler);
+ final String timerThreadName =
+ threadParameters.getThreadName() + " - Incoming Data Monitor";
+ final Timer workerTimer = new Timer(timerThreadName);
+ workerTimer.schedule(dataMonitorTask, 0L, parameters.getCheckIntervalMillis());
+ addShutdownHookForCleanup(workerTimer, pathHandler, parameters.getShutdownTimeOutMillis(),
+ threadParameters.getThreadName());
+ }
+
+ private static void addShutdownHookForCleanup(final Timer workerTimer,
+ final TransferredDataSetHandler mover, final long timeoutMillis, final String threadName)
+ {
+ Runtime.getRuntime().addShutdownHook(new Thread(new Runnable()
+ {
+ public void run()
+ {
+ try
+ {
+ if (operationLog.isInfoEnabled())
+ {
+ operationLog.info("Requesting shutdown lock of thread '" + threadName
+ + "'.");
+ }
+ final long startTimeMillis = System.currentTimeMillis();
+ final boolean lockOK =
+ mover.getRegistrationLock().tryLock(timeoutMillis,
+ TimeUnit.MILLISECONDS);
+ final long timeoutLeftMillis =
+ Math.max(timeoutMillis / 2, timeoutMillis
+ - (System.currentTimeMillis() - startTimeMillis));
+ if (lockOK == false)
+ {
+ operationLog.error("Failed to get lock for shutdown of thread '"
+ + threadName + "'.");
+ }
+ try
+ {
+ if (operationLog.isInfoEnabled())
+ {
+ operationLog.info(String.format("Initiating shutdown sequence "
+ + "[maximal shutdown time: %ds].",
+ 2 * timeoutLeftMillis / 1000));
+ }
+ final boolean shutdownOK =
+ TimerUtilities.tryShutdownTimer(workerTimer, timeoutLeftMillis);
+ operationLog.log(shutdownOK ? Level.INFO : Level.ERROR,
+ "Worker thread shutdown, status="
+ + (shutdownOK ? "OK" : "FAILED"));
+ } finally
+ {
+ if (lockOK)
+ {
+ mover.getRegistrationLock().unlock();
+ }
+ }
+ operationLog.warn("Shutting down shredder(s)");
+ QueueingPathRemoverService.stopAndWait(timeoutMillis);
+ } catch (final InterruptedException ex)
+ {
+ throw new StopException(ex);
+ } finally
+ {
+ if (operationLog.isInfoEnabled())
+ {
+ operationLog.info("Shutting down now.");
+ }
+ }
+ }
+ }, threadName + " - Shutdown Handler"));
+ }
+
+ private final static HighwaterMarkDirectoryScanningHandler createDirectoryScanningHandler(
+ final IStopSignaler stopSignaler, final HighwaterMarkWatcher highwaterMarkWatcher,
+ final File incomingDataDirectory, final File storeRootDir,
+ final Iterable<IProcessorFactory> processorFactories)
+ {
+ final IDirectoryScanningHandler faultyPathHandler =
+ new FaultyPathDirectoryScanningHandler(incomingDataDirectory, stopSignaler);
+ final List<File> list = new ArrayList<File>();
+ list.add(incomingDataDirectory);
+ for (final IProcessorFactory processorFactory : processorFactories)
+ {
+ final PathPrefixPrepender pathPrefixPrepender =
+ processorFactory.getPathPrefixPrepender();
+ File file = pathPrefixPrepender.tryGetDirectoryForAbsolutePaths();
+ if (file != null)
+ {
+ list.add(file);
+ }
+ file = pathPrefixPrepender.tryGetDirectoryForRelativePaths();
+ if (file != null)
+ {
+ list.add(file);
+ }
+ }
+ return new HighwaterMarkDirectoryScanningHandler(faultyPathHandler, highwaterMarkWatcher,
+ list.toArray(new File[0]));
+ }
+
+ public final static void main(final String[] args)
+ {
+ if (checkListShredder(args))
+ {
+ System.exit(0);
+ }
+ initLog();
+ final Parameters parameters = new Parameters(args);
+ TimingParameters.setDefault(parameters.getTimingParameters());
+ QueueingPathRemoverService.start(shredderQueueFile);
+ printInitialLogMessage(parameters);
+ startupServer(parameters);
+ operationLog.info("etlserver ready and waiting for data.");
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/NamedDataStrategy.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/NamedDataStrategy.java
new file mode 100644
index 00000000000..2fb39810001
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/NamedDataStrategy.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.io.File;
+import java.util.regex.Pattern;
+
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
+import ch.systemsx.cisd.openbis.generic.shared.dto.DataSetType;
+
+/**
+ * A <code>IDataStoreStrategy</code> implementation that creates a named directory and put the
+ * candidates (data sets) using numbered subdirectories.
+ *
+ * @author Christian Ribeaud
+ */
+final class NamedDataStrategy implements IDataStoreStrategy
+{
+ /**
+ * A pattern used in {@link FileUtilities#createNextNumberedFile(File, Pattern, String)} to
+ * number the files in a given directory.
+ */
+ private final static Pattern multipleFilesPatterns = Pattern.compile("_\\[([0-9]+)\\]");
+
+ private final DataStoreStrategyKey key;
+
+ final static File createTargetPath(final File targetPath)
+ {
+ assert targetPath != null : "Given target path can not be null.";
+ final String defaultFileName = targetPath.getName() + "_[1]";
+ return FileUtilities.createNextNumberedFile(targetPath, multipleFilesPatterns,
+ defaultFileName);
+ }
+
+ NamedDataStrategy(final DataStoreStrategyKey key)
+ {
+ super();
+ this.key = key;
+ }
+
+ static final String getDirectoryName(final DataStoreStrategyKey key)
+ {
+ return key.name().toLowerCase();
+ }
+
+ private final String getDirectoryName()
+ {
+ return getDirectoryName(key);
+ }
+
+ private final static void assertBaseDirectory(final File baseDirectory)
+ {
+ assert baseDirectory != null : "Missing base directory.";
+ }
+
+ //
+ // IDataStoreStrategy
+ //
+
+ public final DataStoreStrategyKey getKey()
+ {
+ return key;
+ }
+
+ public final File getBaseDirectory(final File baseDirectory,
+ final DataSetInformation dataSetInfo, final DataSetType dataSetType)
+ {
+ assertBaseDirectory(baseDirectory);
+ assert dataSetType != null : "Missing data set type.";
+ return new File(new File(baseDirectory, getDirectoryName()), IdentifiedDataStrategy
+ .createDataSetTypeDirectory(dataSetType));
+ }
+
+ public final File getTargetPath(final File baseDirectory, final File incomingDataSetPath)
+ {
+ assertBaseDirectory(baseDirectory);
+ assert incomingDataSetPath != null : "Missing incoming data set path";
+ return createTargetPath(new File(baseDirectory, incomingDataSetPath.getName()));
+ }
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/Parameters.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/Parameters.java
new file mode 100644
index 00000000000..3deee7fbff6
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/Parameters.java
@@ -0,0 +1,403 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+import org.apache.commons.lang.time.DateUtils;
+import org.apache.log4j.Logger;
+
+import ch.systemsx.cisd.args4j.CmdLineException;
+import ch.systemsx.cisd.args4j.CmdLineParser;
+import ch.systemsx.cisd.args4j.ExampleMode;
+import ch.systemsx.cisd.args4j.Option;
+import ch.systemsx.cisd.common.TimingParameters;
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.exceptions.UserFailureException;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.mail.JavaMailProperties;
+import ch.systemsx.cisd.common.utilities.BuildAndEnvironmentInfo;
+import ch.systemsx.cisd.common.utilities.ExtendedProperties;
+import ch.systemsx.cisd.common.utilities.IExitHandler;
+import ch.systemsx.cisd.common.utilities.PropertyUtils;
+import ch.systemsx.cisd.common.utilities.SystemExit;
+
+/**
+ * The class to process the command line parameters and service properties.
+ *
+ * @author Bernd Rinn
+ */
+public class Parameters
+{
+ private static final String SERVICE_PROPERTIES_FILE = "etc/service.properties";
+
+ private static final Logger operationLog =
+ LogFactory.getLogger(LogCategory.OPERATION, Parameters.class);
+
+ private static final Logger notificationLog =
+ LogFactory.getLogger(LogCategory.NOTIFY, Parameters.class);
+
+ /** property with thread names separated by {@link #ITEMS_DELIMITER} */
+ private static final String INPUT_THREAD_NAMES = "inputs";
+
+ private static final String ITEMS_DELIMITER = ",";
+
+ @Option(name = "s", longName = "server-url", metaVar = "URL", usage = "URL of the server")
+ private String serverURL;
+
+ /**
+ * The interval to wait between to checks for activity (in milliseconds).
+ */
+ @Option(name = "c", longName = "check-interval", usage = "The interval to wait between two checks (in seconds) "
+ + "[default: 120]")
+ private long checkIntervalSeconds;
+
+ /**
+ * The time-out for clean up work in the shutdown sequence (in seconds).
+ * <p>
+ * Note that that the maximal time for the shutdown sequence to complete can be as large as
+ * twice this time.
+ */
+ @Option(name = "t", longName = "shutdown-timeout", usage = "The time-out for clean up work "
+ + "in the shutdown sequence (in seconds) [default: 30]")
+ private long shutdownTimeOutSeconds;
+
+ /**
+ * The username to access the LIMS server with.
+ */
+ @Option(name = "u", longName = "username", usage = "User login name")
+ private String username;
+
+ /**
+ * The password to access the LIMS server with.
+ */
+ @Option(name = "p", longName = "password", usage = "User login password")
+ private String password;
+
+ /** A subset of <code>service.properties</code> that are reserved for the <i>JavaMail API</i>. */
+ private Properties mailProperties;
+
+ /**
+ * The command line parser.
+ */
+ private final CmdLineParser parser = new CmdLineParser(this);
+
+ private TimingParameters timingParameters;
+
+ private Properties serviceProperties;
+
+ private ThreadParameters[] threads;
+
+ private Map<String, Properties> processorProperties;
+
+ @Option(longName = "help", skipForExample = true, usage = "Prints out a description of the options.")
+ void printHelp(final boolean exit)
+ {
+ parser.printHelp("etlserver", "<required options> [option [...]]", "", ExampleMode.ALL);
+ if (exit)
+ {
+ System.exit(0);
+ }
+ }
+
+ @Option(longName = "version", skipForExample = true, usage = "Prints out the version information.")
+ void printVersion(final boolean exit)
+ {
+ System.err
+ .println("etlserver version " + BuildAndEnvironmentInfo.INSTANCE.getFullVersion());
+ if (exit)
+ {
+ System.exit(0);
+ }
+ }
+
+ @Option(longName = "test-notify", skipForExample = true, usage = "Tests the notify log (i.e. that an email is "
+ + "sent out).")
+ void sendTestNotification(final boolean exit)
+ {
+ notificationLog
+ .error("This is a test notification given due to specifying the --test-notify option.");
+ if (exit)
+ {
+ System.exit(0);
+ }
+ }
+
+ Parameters(final String[] args)
+ {
+ this(args, SystemExit.SYSTEM_EXIT);
+ }
+
+ Parameters(final String[] args, final IExitHandler systemExitHandler)
+ {
+ try
+ {
+ initParametersFromProperties();
+
+ parser.parseArgument(args);
+ for (final ThreadParameters thread : threads)
+ {
+ thread.check();
+ }
+ if (serverURL == null)
+ {
+ throw new ConfigurationFailureException("No 'server-url' defined.");
+ }
+ } catch (final Exception ex)
+ {
+ outputException(ex);
+ systemExitHandler.exit(1);
+ // Only reached in unit tests.
+ throw new AssertionError(ex.getMessage());
+ }
+ }
+
+ private void initParametersFromProperties()
+ {
+ serviceProperties = PropertyUtils.loadProperties(SERVICE_PROPERTIES_FILE);
+ PropertyUtils.trimProperties(serviceProperties);
+ processorProperties = extractProcessorProperties(serviceProperties);
+ threads = createThreadParameters(serviceProperties);
+ serverURL = serviceProperties.getProperty("server-url");
+ username = serviceProperties.getProperty("username");
+ password = serviceProperties.getProperty("password");
+ checkIntervalSeconds =
+ Long.parseLong(serviceProperties.getProperty("check-interval", "120"));
+ shutdownTimeOutSeconds =
+ Long.parseLong(serviceProperties.getProperty("shutdown-timeout", "30"));
+ mailProperties = createMailProperties(serviceProperties);
+ timingParameters = TimingParameters.create(serviceProperties);
+ }
+
+ private static Map<String, Properties> extractProcessorProperties(final Properties properties)
+ {
+ final LinkedHashMap<String, Properties> map = new LinkedHashMap<String, Properties>();
+ final String processors = properties.getProperty("processors");
+ if (processors != null)
+ {
+ final String[] procedureTypes = processors.split(ITEMS_DELIMITER);
+ for (final String procedureType : procedureTypes)
+ {
+ final String prefix = "processor." + procedureType + ".";
+ map.put(procedureType, ExtendedProperties.getSubset(properties, prefix, true));
+ }
+ }
+ return map;
+ }
+
+ private static ThreadParameters[] createThreadParameters(final Properties serviceProperties)
+ {
+ final String threadNames = serviceProperties.getProperty(INPUT_THREAD_NAMES);
+ if (threadNames == null)
+ {
+ // backward compatibility mode - no prefixes before thread properties, one thread only
+ return new ThreadParameters[]
+ { new ThreadParameters(serviceProperties, "default") };
+ } else
+ {
+ final String[] names = threadNames.split(ITEMS_DELIMITER);
+ validateThreadNames(names);
+ return createThreadParameters(names, serviceProperties);
+ }
+ }
+
+ private static ThreadParameters[] createThreadParameters(final String[] names,
+ final Properties serviceProperties)
+ {
+ final ThreadParameters[] threadParameters = new ThreadParameters[names.length];
+ final Properties generalProperties =
+ removeThreadSpecificProperties(names, serviceProperties);
+ for (int i = 0; i < names.length; i++)
+ {
+ final String name = names[i].trim();
+ if (operationLog.isInfoEnabled())
+ {
+ operationLog.info("Create parameters for thread '" + name + "'.");
+ }
+ // extract thread specific properties, remove prefix
+ final ExtendedProperties threadProperties =
+ ExtendedProperties.getSubset(serviceProperties, getPropertyPrefix(name), true);
+ threadProperties.putAll(generalProperties); // add all general properties
+ threadParameters[i] = new ThreadParameters(threadProperties, name);
+ }
+ return threadParameters;
+ }
+
+ private static Properties removeThreadSpecificProperties(final String[] names,
+ final Properties properties)
+ {
+ final ExtendedProperties generalProperties = ExtendedProperties.createWith(properties);
+ for (final String name : names)
+ {
+ generalProperties.removeSubset(getPropertyPrefix(name));
+ }
+ return generalProperties;
+ }
+
+ private static String getPropertyPrefix(final String name)
+ {
+ return name + ".";
+ }
+
+ private static void validateThreadNames(final String[] names)
+ {
+ final Set<String> processed = new HashSet<String>();
+ for (final String name : names)
+ {
+ if (processed.contains(name))
+ {
+ throw new ConfigurationFailureException("Duplicated thread name: " + name);
+ }
+ if (name.length() == 0)
+ {
+ throw new ConfigurationFailureException("Thread name:cannot be empty!");
+ }
+ processed.add(name);
+ }
+ }
+
+ private final static Properties createMailProperties(final Properties serviceProperties)
+ {
+ final Properties properties =
+ ExtendedProperties.getSubset(serviceProperties, "mail", false);
+ if (properties.getProperty(JavaMailProperties.MAIL_SMTP_HOST) == null)
+ {
+ properties.setProperty(JavaMailProperties.MAIL_SMTP_HOST, "localhost");
+ }
+ if (properties.getProperty(JavaMailProperties.MAIL_FROM) == null)
+ {
+ properties.setProperty(JavaMailProperties.MAIL_FROM, "etlserver@localhost");
+ }
+ return properties;
+ }
+
+ private void outputException(final Exception ex)
+ {
+ if (ex instanceof UserFailureException || ex instanceof CmdLineException)
+ {
+ System.err.println(ex.getMessage());
+ } else
+ {
+ System.err.println("An exception occurred.");
+ ex.printStackTrace();
+ }
+ if (ex instanceof CmdLineException)
+ {
+ printHelp(false);
+ }
+ }
+
+ /**
+ * Returns The interval to wait between to checks for activity (in milliseconds).
+ */
+ public long getCheckIntervalMillis()
+ {
+ return checkIntervalSeconds * DateUtils.MILLIS_PER_SECOND;
+ }
+
+ /**
+ * Returns the time-out for clean up work in the shutdown sequence (in seconds).
+ * <p>
+ * Note that that the maximal time for the shutdown sequence to complete can be as large as
+ * twice this time.
+ */
+ public long getShutdownTimeOutMillis()
+ {
+ return shutdownTimeOutSeconds * DateUtils.MILLIS_PER_SECOND;
+ }
+
+ /**
+ * Returns the password to access the LIMS server with.
+ */
+ public String getPassword()
+ {
+ return password;
+ }
+
+ /**
+ * Returns the username to access the LIMS server with.
+ */
+ public String getUsername()
+ {
+ return username;
+ }
+
+ /**
+ * Returns all properties.
+ */
+ public final Properties getProperties()
+ {
+ return serviceProperties;
+ }
+
+ /**
+ * Returns a map of all processor properties with the procedure type code as the key.
+ */
+ public final Map<String, Properties> getProcessorProperties()
+ {
+ return processorProperties;
+ }
+
+ /** Returns <code>mailProperties</code>. */
+ public final Properties getMailProperties()
+ {
+ return mailProperties;
+ }
+
+ /**
+ * Returns the timing parameters for monitored operations.
+ */
+ public TimingParameters getTimingParameters()
+ {
+ return timingParameters;
+ }
+
+ /**
+ * Logs the current parameters to the {@link LogCategory#OPERATION} log.
+ */
+ public void log()
+ {
+ if (operationLog.isInfoEnabled())
+ {
+ for (final ThreadParameters threadParameters : threads)
+ {
+ threadParameters.log();
+ }
+ operationLog.info(String.format("Check intervall: %d s.",
+ getCheckIntervalMillis() / 1000));
+ }
+ }
+
+ /**
+ * Returns the URL of the LIMS server.
+ */
+ public String getServerURL()
+ {
+ return serverURL;
+ }
+
+ public ThreadParameters[] getThreads()
+ {
+ return threads;
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/PlateDimension.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/PlateDimension.java
new file mode 100644
index 00000000000..6321c839dd4
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/PlateDimension.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.io.Serializable;
+
+import ch.systemsx.cisd.common.utilities.AbstractHashable;
+import ch.systemsx.cisd.openbis.generic.shared.IWebService;
+
+/**
+ * @author Tomasz Pylak
+ */
+public class PlateDimension extends AbstractHashable implements Serializable
+{
+ private static final long serialVersionUID = IWebService.VERSION;
+
+ private int rowsNum;
+
+ private int colsNum;
+
+ // for internal use only
+ public PlateDimension()
+ {
+ this(0, 0);
+ }
+
+ public PlateDimension(int rowsNum, int colsNum)
+ {
+ super();
+ this.rowsNum = rowsNum;
+ this.colsNum = colsNum;
+ }
+
+ public int getRowsNum()
+ {
+ return rowsNum;
+ }
+
+ public void setRowsNum(int rowsNum)
+ {
+ this.rowsNum = rowsNum;
+ }
+
+ public int getColsNum()
+ {
+ return colsNum;
+ }
+
+ public void setColsNum(int colsNum)
+ {
+ this.colsNum = colsNum;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "(" + rowsNum + ", " + colsNum + ")";
+ }
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/PlateDimensionParser.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/PlateDimensionParser.java
new file mode 100644
index 00000000000..1e237a59cf6
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/PlateDimensionParser.java
@@ -0,0 +1,120 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import ch.systemsx.cisd.openbis.generic.shared.dto.EntityPropertyPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.PropertyTypePE;
+
+/**
+ * Extractor and parser of the plate geometry from an array of properties.
+ *
+ * @author Tomasz Pylak
+ */
+public class PlateDimensionParser
+{
+ public static final String PLATE_GEOMETRY_PROPERTY_NAME = "PLATE_GEOMETRY";
+
+ /**
+ * Returns the plate geometry from the specified properties.
+ *
+ * @throws IllegalArgumentException if either their isn't such a property or it has an invalid
+ * value.
+ */
+ public static PlateDimension getPlateDimension(final EntityPropertyPE[] properties)
+ {
+ final PlateDimension plateDimension = tryToGetPlateDimension(properties);
+ if (plateDimension == null)
+ {
+ throw new IllegalArgumentException("Cannot find property "
+ + PLATE_GEOMETRY_PROPERTY_NAME);
+ }
+ return plateDimension;
+ }
+
+ /**
+ * Tries to get the plate geometry from the specified properties.
+ *
+ * @return <code>null</code> if their isn't such a property.
+ * @throws IllegalArgumentException if the property for the plate geometry has an invalid value.
+ */
+ public static PlateDimension tryToGetPlateDimension(final EntityPropertyPE[] properties)
+ {
+ assert properties != null : "Unspecified properties";
+ final String plateGeometryString =
+ tryFindProperty(properties, PLATE_GEOMETRY_PROPERTY_NAME);
+ if (plateGeometryString == null)
+ {
+ return null;
+ }
+ final PlateDimension dimension = tryParsePlateDimension(plateGeometryString);
+ if (dimension == null)
+ {
+ throw new IllegalArgumentException("Cannot parse plate geometry " + plateGeometryString);
+ }
+ return dimension;
+
+ }
+
+ // parses plate geometry - takes the token after the last "_" sign and assumes that the number
+ // of rows is separated
+ // from number of columns by the 'X' sign, e.g. XXX_YYY_16x24
+ private static PlateDimension tryParsePlateDimension(final String plateGeometryString)
+ {
+ final String[] tokens = plateGeometryString.split("_");
+ final String sizeToken = tokens[tokens.length - 1];
+ final String[] dims = sizeToken.split("X");
+ if (dims.length != 2)
+ {
+ return null;
+ }
+ final Integer rows = tryParseInteger(dims[0]);
+ final Integer cols = tryParseInteger(dims[1]);
+ if (rows == null || cols == null)
+ {
+ return null;
+ }
+ return new PlateDimension(rows, cols);
+ }
+
+ private static Integer tryParseInteger(final String value)
+ {
+ try
+ {
+ return Integer.parseInt(value);
+ } catch (final NumberFormatException e)
+ {
+ return null;
+ }
+ }
+
+ private static String tryFindProperty(final EntityPropertyPE[] properties,
+ final String propertyCode)
+ {
+ for (final EntityPropertyPE property : properties)
+ {
+ final PropertyTypePE propertyType =
+ property.getEntityTypePropertyType().getPropertyType();
+ if (propertyType.getCode().equals(propertyCode))
+ {
+ return property.tryGetUntypedValue();
+ }
+ }
+ return null;
+ }
+
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/PropertiesBasedETLServerPlugin.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/PropertiesBasedETLServerPlugin.java
new file mode 100644
index 00000000000..6feb4ffde9d
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/PropertiesBasedETLServerPlugin.java
@@ -0,0 +1,123 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import static ch.systemsx.cisd.etlserver.IDataSetInfoExtractor.EXTRACTOR_KEY;
+import static ch.systemsx.cisd.etlserver.IProcedureAndDataTypeExtractor.TYPE_EXTRACTOR_KEY;
+import static ch.systemsx.cisd.etlserver.IStorageProcessor.STORAGE_PROCESSOR_KEY;
+
+import java.util.Properties;
+
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.utilities.ClassUtils;
+import ch.systemsx.cisd.common.utilities.ExtendedProperties;
+
+/**
+ * An implementation of {@link IETLServerPlugin} which is based on a <code>Properties</code>
+ * object. The objects delivered by this implementation are created only once. For creation the
+ * properties are used. For each object a specific property has to be defined which specifies the
+ * fully-qualified class name of the object. The class has to implement a specific interface and it
+ * should have a constructor with a single argument of type <code>Properties</code>. The argmunt
+ * is derived from the original properties by extracting all properties where the key starts with
+ * the prefix <code><i><class name key></i> + '.'</code>. The prefix is removed from the
+ * key for the derived properties. The following table shows all class name keys and interfaces:
+ * <table cellspacing="0" cellpadding="5" border="1">
+ * <tr>
+ * <th>Class name key</th>
+ * <th>Interface</th>
+ * </tr>
+ * <tr>
+ * <td><code>code-extractor</code></td>
+ * <td>{@link IDataSetInfoExtractor}</td>
+ * </tr>
+ * <tr>
+ * <td><code>type-extractor</code></td>
+ * <td>{@link IProcedureAndDataTypeExtractor}</td>
+ * </tr>
+ * </table> Example of a properties file:
+ *
+ * <pre><tt>
+ * data-set-info-extractor = ch.systemsx.cisd.etlserver.DefaultDataSetInfoExtractor
+ * data-set-info-extractor.entity-separator = ==
+ *
+ * type-extractor = ch.systemsx.cisd.etlserver.SimpleTypeExtractor
+ * type-extractor.file-format-type = TIFF
+ * type-extractor.locator-type = RELATIVE_LOCATION
+ * type-extractor.data-set-type = HCS_IMAGE
+ * type-extractor.procedure-type = DATA_ACQUISITION
+ * </tt></pre>
+ *
+ * @author Franz-Josef Elmer
+ */
+public class PropertiesBasedETLServerPlugin extends ETLServerPlugin
+{
+
+ private static final Properties EMPTY_PROPERTIES = new Properties();
+
+ private final static <T> T create(final Class<T> superClazz, final Properties properties,
+ final String keyPrefix, final boolean withSubset)
+ {
+ final String className = properties.getProperty(keyPrefix);
+ if (className == null)
+ {
+ throw new ConfigurationFailureException("Missing property '" + keyPrefix + "'.");
+ }
+ try
+ {
+ return ClassUtils.create(superClazz, className, withSubset ? createSubsetProperties(
+ properties, keyPrefix) : properties);
+ } catch (IllegalArgumentException ex)
+ {
+ throw new ConfigurationFailureException(ex.getMessage());
+ }
+ }
+
+ public PropertiesBasedETLServerPlugin(final Properties properties)
+ {
+ super(createDataSetInfoExtractor(properties),
+ createProcedureAndDataTypeExtractor(properties), createStorageProcessor(properties));
+ }
+
+ private final static Properties createSubsetProperties(final Properties properties,
+ final String prefix)
+ {
+ if (prefix == null)
+ {
+ return properties;
+ }
+ return ExtendedProperties.getSubset(properties == null ? EMPTY_PROPERTIES : properties,
+ prefix + '.', true);
+ }
+
+ private final static IStorageProcessor createStorageProcessor(final Properties properties)
+ {
+ return create(IStorageProcessor.class, properties, STORAGE_PROCESSOR_KEY, true);
+ }
+
+ private final static IProcedureAndDataTypeExtractor createProcedureAndDataTypeExtractor(
+ final Properties properties)
+ {
+ return create(IProcedureAndDataTypeExtractor.class, properties, TYPE_EXTRACTOR_KEY, true);
+ }
+
+ private final static IDataSetInfoExtractor createDataSetInfoExtractor(
+ final Properties properties)
+ {
+ return create(IDataSetInfoExtractor.class, properties, EXTRACTOR_KEY, false);
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/SimpleTypeExtractor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/SimpleTypeExtractor.java
new file mode 100644
index 00000000000..ad4867f7896
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/SimpleTypeExtractor.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.io.File;
+import java.util.Properties;
+
+import ch.systemsx.cisd.openbis.generic.shared.dto.DataSetType;
+import ch.systemsx.cisd.openbis.generic.shared.dto.FileFormatType;
+import ch.systemsx.cisd.openbis.generic.shared.dto.LocatorType;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ProcedureType;
+import ch.systemsx.cisd.openbis.generic.shared.dto.types.DataSetTypeCode;
+import ch.systemsx.cisd.openbis.generic.shared.dto.types.ProcedureTypeCode;
+
+/**
+ * Implementation of {@link IProcedureAndDataTypeExtractor} which gets the types from the properties
+ * argument of the constructor.
+ *
+ * @author Franz-Josef Elmer
+ */
+public class SimpleTypeExtractor implements IProcedureAndDataTypeExtractor
+{
+ public static final String FILE_FORMAT_TYPE_KEY = "file-format-type";
+
+ public static final String LOCATOR_TYPE_KEY = "locator-type";
+
+ public static final String DATA_SET_TYPE_KEY = "data-set-type";
+
+ public static final String PROCEDURE_TYPE_KEY = "procedure-type";
+
+ private FileFormatType fileFormatType;
+
+ private LocatorType locatorType;
+
+ private DataSetType dataSetType;
+
+ private ProcedureType procedureType;
+
+ public SimpleTypeExtractor(final Properties properties)
+ {
+ String code =
+ properties.getProperty(FILE_FORMAT_TYPE_KEY,
+ FileFormatType.DEFAULT_FILE_FORMAT_TYPE_CODE);
+ fileFormatType = new FileFormatType(code);
+ code = properties.getProperty(LOCATOR_TYPE_KEY, LocatorType.DEFAULT_LOCATOR_TYPE_CODE);
+ locatorType = new LocatorType(code);
+ code = properties.getProperty(DATA_SET_TYPE_KEY, DataSetTypeCode.HCS_IMAGE.getCode());
+ dataSetType = new DataSetType(code);
+ code =
+ properties.getProperty(PROCEDURE_TYPE_KEY, ProcedureTypeCode.DATA_ACQUISITION
+ .getCode());
+ procedureType = new ProcedureType(code);
+ }
+
+ //
+ // IProcedureAndDataTypeExtractor
+ //
+
+ public final FileFormatType getFileFormatType(final File incomingDataSetPath)
+ {
+ return fileFormatType;
+ }
+
+ public final LocatorType getLocatorType(final File incomingDataSetPath)
+ {
+ return locatorType;
+ }
+
+ public final DataSetType getDataSetType(final File incomingDataSetPath)
+ {
+ return dataSetType;
+ }
+
+ public final ProcedureType getProcedureType(final File incomingDataSetPath)
+ {
+ return procedureType;
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/StandardProcessor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/StandardProcessor.java
new file mode 100644
index 00000000000..20e54118205
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/StandardProcessor.java
@@ -0,0 +1,187 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.io.File;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.time.StopWatch;
+import org.apache.log4j.Logger;
+
+import ch.systemsx.cisd.common.exceptions.StopException;
+import ch.systemsx.cisd.common.filesystem.PathPrefixPrepender;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ProcessingInstructionDTO;
+import ch.systemsx.cisd.openbis.generic.shared.dto.StorageFormat;
+
+/**
+ * Standard implementation of <code>IProcessor</code>.
+ *
+ * @author Christian Ribeaud
+ */
+final class StandardProcessor implements IProcessor
+{
+ private static final Logger operationLog =
+ LogFactory.getLogger(LogCategory.OPERATION, StandardProcessor.class);
+
+ private static final Logger notificationLog =
+ LogFactory.getLogger(LogCategory.NOTIFY, StandardProcessor.class);
+
+ private final IFileFactory fileFactory;
+
+ private final PathPrefixPrepender pathPrefixPrepender;
+
+ private final String parametersFileName;
+
+ private final MessageFormat finishedFileFormat;
+
+ private final StorageFormat inputDataFormat;
+
+ private final String dataSetCodePrefixGlueCharacter;
+
+ public StandardProcessor(final IFileFactory fileFactory, final StorageFormat inputDataFormat,
+ final PathPrefixPrepender pathPrefixPrepender, final String parametersFileName,
+ final String finishedFileNameTemplate, final String dataSetCodePrefixGlueCharacter)
+ {
+ assert fileFactory != null : "Unspecified IFileFactory.";
+ assert inputDataFormat != null : "Unspecified StorageFormat.";
+ assert pathPrefixPrepender != null : "Unspecified PathPrefixPrepender.";
+ assert parametersFileName != null : "Unspecified parameters file name.";
+ assert finishedFileNameTemplate != null : "Unspecified finished file name template.";
+ assert dataSetCodePrefixGlueCharacter != null : "Unspecified data set code prefix glue character.";
+ this.fileFactory = fileFactory;
+ this.inputDataFormat = inputDataFormat;
+ this.pathPrefixPrepender = pathPrefixPrepender;
+ this.parametersFileName = parametersFileName;
+ this.finishedFileFormat = new MessageFormat(finishedFileNameTemplate);
+ this.dataSetCodePrefixGlueCharacter = dataSetCodePrefixGlueCharacter;
+ }
+
+ private void createDataSetForProcessing(final File dataSet, final IFile dataSetForProcessing)
+ {
+ final StopWatch watch = new StopWatch();
+ watch.start();
+ dataSetForProcessing.copyFrom(dataSet);
+ if (operationLog.isInfoEnabled())
+ {
+ operationLog.info("Data set '" + dataSet.getName() + "' copied into '"
+ + dataSetForProcessing.getAbsolutePath() + "', took " + watch + ".");
+ }
+ }
+
+ private void createProcessingParameters(final ProcessingInstructionDTO instruction,
+ final IFile processingDirectory, final List<IFile> itemsToRemoveInCaseOfError)
+ {
+ final byte[] instructionDataOrNull = instruction.getParameters();
+ if (instructionDataOrNull != null)
+ {
+ final IFile parametersFile =
+ fileFactory.create(processingDirectory, parametersFileName);
+ itemsToRemoveInCaseOfError.add(parametersFile);
+ parametersFile.write(instructionDataOrNull);
+ if (operationLog.isInfoEnabled())
+ {
+ operationLog.info("Processing parameters written into '"
+ + parametersFile.getAbsolutePath() + "'.");
+ }
+ }
+ }
+
+ private void createFinishedFile(final IFile processingDirectory, final String dataSetName)
+ {
+ final String finishedFileName = finishedFileFormat.format(new String[]
+ { dataSetName });
+ final IFile finishedFile = fileFactory.create(processingDirectory, finishedFileName);
+ finishedFile.write(new byte[0]);
+ }
+
+ private final String getDataSetName(final DataSetInformation dataSetInformation,
+ final File dataSet)
+ {
+ final String dataSetName = dataSet.getName();
+ final String parentDataSetCode = dataSetInformation.getDataSetCode();
+ if (StringUtils.isNotEmpty(parentDataSetCode))
+ {
+ return parentDataSetCode + dataSetCodePrefixGlueCharacter + dataSetName;
+ }
+ return dataSetName;
+ }
+
+ //
+ // IProcessor
+ //
+
+ public final StorageFormat getRequiredInputDataFormat()
+ {
+ return inputDataFormat;
+ }
+
+ public final void initiateProcessing(final ProcessingInstructionDTO instruction,
+ final DataSetInformation dataSetInformation, final File dataSet)
+ {
+ assert instruction != null : "Unspecified instruction.";
+ assert dataSet != null : "Unspecified data set.";
+ assert dataSetInformation != null : "Unspecified data set information.";
+ if (operationLog.isInfoEnabled())
+ {
+ operationLog.info("Start initialization of processing.");
+ }
+ final String processingPath = pathPrefixPrepender.addPrefixTo(instruction.getPath());
+ final IFile processingDirectory = fileFactory.create(processingPath);
+ processingDirectory.check();
+ final String dataSetName = getDataSetName(dataSetInformation, dataSet);
+ final IFile dataSetForProcessing = fileFactory.create(processingDirectory, dataSetName);
+ final List<IFile> itemsToRemoveInCaseOfError = new ArrayList<IFile>(2);
+ itemsToRemoveInCaseOfError.add(dataSetForProcessing);
+ try
+ {
+ StopException.check();
+ createDataSetForProcessing(dataSet, dataSetForProcessing);
+ StopException.check();
+ createProcessingParameters(instruction, processingDirectory, itemsToRemoveInCaseOfError);
+ StopException.check();
+ createFinishedFile(processingDirectory, dataSetName);
+ if (operationLog.isInfoEnabled())
+ {
+ operationLog.info("Processing initiated.");
+ }
+ } catch (final Exception ex)
+ {
+ for (final IFile item : itemsToRemoveInCaseOfError)
+ {
+ item.delete();
+ }
+ if (ex instanceof StopException)
+ {
+ operationLog
+ .warn(String
+ .format(
+ "Requested to stop initiation of processing, rolled back: [data set: '%s'].",
+ dataSetForProcessing.getAbsolutePath()));
+ } else
+ {
+ notificationLog.error(String.format(
+ "Error when initiating processing, rolled back: [data set: '%s'].",
+ dataSetForProcessing.getAbsolutePath()), ex);
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/StandardProcessorFactory.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/StandardProcessorFactory.java
new file mode 100644
index 00000000000..a6c38d0a634
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/StandardProcessorFactory.java
@@ -0,0 +1,135 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.util.Properties;
+
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.filesystem.PathPrefixPrepender;
+import ch.systemsx.cisd.common.utilities.PropertyUtils;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ProcessingInstructionDTO;
+import ch.systemsx.cisd.openbis.generic.shared.dto.StorageFormat;
+
+/**
+ * Factory for standard post processors. A standard post processor does the following:
+ * <ol>
+ * <li>Copies the data set file or folder to the processing directory. The path of the processing
+ * directory is a combination of the processing code (i.e. {@link ProcessingInstructionDTO#getPath()})
+ * and the mandatory property <code>root-directory</code>. This copy action is done during the
+ * preparation step.
+ * <li>Stores the processing parameters (i.e. {@link ProcessingInstructionDTO#getParameters()}) in a
+ * file in the processing directory who's name is specified by the mandatory property
+ * <code>paramaters-file</code>.
+ * <li>Creates an empty file in the processing directory who's name is specified by the mandatory
+ * property <code>finished-file</code>.
+ * </ol>
+ *
+ * @author Franz-Josef Elmer
+ */
+public class StandardProcessorFactory implements IProcessorFactory
+{
+ private static final String HARD_LINK_INSTEAD_OF_COPY_KEY = "hard-link-instead-of-copy";
+
+ private static final String PATH_PREFIX_ABSOLUTE_KEY = "prefix-for-absolute-paths";
+
+ private static final String PATH_PREFIX_RELATIVE_KEY = "prefix-for-relative-paths";
+
+ private static final String PARAMETERS_FILE_KEY = "parameters-file";
+
+ private static final String INPUT_STORAGE_FORMAT_KEY = "input-storage-format";
+
+ private static final String DATA_SET_CODE_PREFIX_GLUE_KEY = "data-set-code-prefix-glue";
+
+ private static final String FINISHED_FILE_TEMPLATE_KEY = "finished-file-template";
+
+ private final String parametersFileName;
+
+ private final String finishedFileNameTemplate;
+
+ private final IFileFactory fileFactory;
+
+ private final StorageFormat inputStorageFormat;
+
+ final PathPrefixPrepender pathPrefixPrepender;
+
+ private final String dataSetCodePrefixGlueCharacter;
+
+ /**
+ * Creates a new instances for the specified properties. Uses the default timing parameters.
+ *
+ * @throws ConfigurationFailureException if one of the mandatory properties is missing or the
+ * property <code>root-directory</code> is not the path of an existing directory.
+ */
+ public static StandardProcessorFactory create(final Properties properties)
+ throws ConfigurationFailureException
+ {
+ final FileBasedFileFactory fileBasedFileFactory = createFileFactory(properties);
+ return new StandardProcessorFactory(properties, fileBasedFileFactory);
+ }
+
+ private static FileBasedFileFactory createFileFactory(final Properties properties)
+ throws ConfigurationFailureException
+ {
+ final boolean useHardLinks =
+ PropertyUtils.getBoolean(properties, HARD_LINK_INSTEAD_OF_COPY_KEY, true);
+ final FileBasedFileFactory fileBasedFileFactory =
+ new FileBasedFileFactory(useHardLinks);
+ return fileBasedFileFactory;
+ }
+
+ private StandardProcessorFactory(final Properties properties, final IFileFactory fileFactory)
+ throws ConfigurationFailureException
+ {
+ this.fileFactory = fileFactory;
+ assert properties != null : "Undefined properties.";
+ final String prefixForAbsolutePathsOrNull =
+ properties.getProperty(PATH_PREFIX_ABSOLUTE_KEY);
+ final String prefixForRelativePathsOrNull =
+ properties.getProperty(PATH_PREFIX_RELATIVE_KEY);
+ final String inputDataFormatCode =
+ PropertyUtils.getMandatoryProperty(properties, INPUT_STORAGE_FORMAT_KEY);
+ inputStorageFormat = StorageFormat.tryGetFromCode(inputDataFormatCode);
+ if (inputStorageFormat == null)
+ {
+ throw ConfigurationFailureException.fromTemplate(INPUT_STORAGE_FORMAT_KEY
+ + " property has illegal value '%s'.", inputDataFormatCode);
+ }
+ pathPrefixPrepender =
+ new PathPrefixPrepender(prefixForAbsolutePathsOrNull, prefixForRelativePathsOrNull);
+ parametersFileName = PropertyUtils.getMandatoryProperty(properties, PARAMETERS_FILE_KEY);
+ finishedFileNameTemplate =
+ PropertyUtils.getMandatoryProperty(properties, FINISHED_FILE_TEMPLATE_KEY);
+ dataSetCodePrefixGlueCharacter =
+ PropertyUtils.getMandatoryProperty(properties, DATA_SET_CODE_PREFIX_GLUE_KEY);
+ }
+
+ //
+ // IProcessorFactory
+ //
+
+ public final PathPrefixPrepender getPathPrefixPrepender()
+ {
+ return pathPrefixPrepender;
+ }
+
+ public final IProcessor createProcessor()
+ {
+ return new StandardProcessor(fileFactory, inputStorageFormat, pathPrefixPrepender,
+ parametersFileName, finishedFileNameTemplate, dataSetCodePrefixGlueCharacter);
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/ThreadParameters.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/ThreadParameters.java
new file mode 100644
index 00000000000..6caa7f5af41
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/ThreadParameters.java
@@ -0,0 +1,146 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.io.File;
+import java.util.Properties;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
+
+import ch.rinn.restrictions.Private;
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.utilities.PropertyUtils;
+
+/**
+ * <i>ETL</i> thread specific parameters.
+ *
+ * @author Tomasz Pylak
+ */
+public final class ThreadParameters
+{
+ @Private
+ static final String GROUP_CODE_KEY = "group-code";
+
+ private static final Logger operationLog =
+ LogFactory.getLogger(LogCategory.OPERATION, ThreadParameters.class);
+
+ private static final String INCOMING_DIR = "incoming-dir";
+
+ /**
+ * The (local) directory to monitor for new files and directories to move to the remote side.
+ * The directory where data to be processed by the ETL server become available.
+ */
+ private final File incomingDataDirectory;
+
+ private final IETLServerPlugin plugin;
+
+ private final String threadName;
+
+ private final String groupCode;
+
+ /**
+ * @param threadProperties parameters for one processing thread together with general
+ * parameters.
+ */
+ public ThreadParameters(final Properties threadProperties, final String threadName)
+ {
+ this.incomingDataDirectory = extractIncomingDataDir(threadProperties);
+ this.plugin = new PropertiesBasedETLServerPlugin(threadProperties);
+ groupCode = tryGetGroupCode(threadProperties);
+ this.threadName = threadName;
+ }
+
+ final void check()
+ {
+ if (incomingDataDirectory.isDirectory() == false)
+ {
+ throw new ConfigurationFailureException("Incoming directory '" + incomingDataDirectory
+ + "' is not a directory.");
+ }
+ }
+
+ @Private
+ static File extractIncomingDataDir(final Properties threadProperties)
+ {
+ final String incomingDir = threadProperties.getProperty(INCOMING_DIR);
+ if (StringUtils.isNotBlank(incomingDir))
+ {
+ return FileUtilities.normalizeFile(new File(incomingDir));
+ } else
+ {
+ throw new ConfigurationFailureException("No '" + INCOMING_DIR + "' defined.");
+ }
+ }
+
+ @Private
+ static final String tryGetGroupCode(final Properties properties)
+ {
+ return StringUtils.defaultIfEmpty(PropertyUtils.getProperty(properties, GROUP_CODE_KEY),
+ null);
+ }
+
+ /**
+ * Returns the <code>group-code</code> property specified for this thread.
+ */
+ final String tryGetGroupCode()
+ {
+ return groupCode;
+ }
+
+ /**
+ * Returns The directory to monitor for incoming data.
+ */
+ final File getIncomingDataDirectory()
+ {
+ return incomingDataDirectory;
+ }
+
+ final IETLServerPlugin getPlugin()
+ {
+ return plugin;
+ }
+
+ /**
+ * Logs the current parameters to the {@link LogCategory#OPERATION} log.
+ */
+ final void log()
+ {
+ if (operationLog.isInfoEnabled())
+ {
+ operationLog.info(String.format("[%s] Code extractor: '%s'", threadName, plugin
+ .getDataSetInfoExtractor().getClass().getName()));
+ operationLog.info(String.format("[%s] Type extractor: '%s'", threadName, plugin
+ .getTypeExtractor().getClass().getName()));
+ operationLog.info(String.format("[%s] Incoming data directory: '%s'.", threadName,
+ getIncomingDataDirectory().getAbsolutePath()));
+ if (groupCode != null)
+ {
+ operationLog.info(String.format("[%s] Group code: '%s'.", threadName, groupCode));
+ }
+ }
+ }
+
+ public String getThreadName()
+ {
+ return threadName;
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandler.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandler.java
new file mode 100644
index 00000000000..29057ff6381
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandler.java
@@ -0,0 +1,743 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import static ch.systemsx.cisd.common.Constants.IS_FINISHED_PREFIX;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.time.StopWatch;
+import org.apache.log4j.Logger;
+
+import ch.rinn.restrictions.Private;
+import ch.systemsx.cisd.common.Constants;
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.common.exceptions.HighLevelException;
+import ch.systemsx.cisd.common.exceptions.StopException;
+import ch.systemsx.cisd.common.exceptions.WrappedIOException;
+import ch.systemsx.cisd.common.filesystem.FileOperations;
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
+import ch.systemsx.cisd.common.filesystem.IFileOperations;
+import ch.systemsx.cisd.common.filesystem.IPathHandler;
+import ch.systemsx.cisd.common.highwatermark.HighwaterMarkWatcher;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.mail.IMailClient;
+import ch.systemsx.cisd.common.mail.MailClient;
+import ch.systemsx.cisd.common.types.BooleanOrUnknown;
+import ch.systemsx.cisd.common.utilities.BeanUtils;
+import ch.systemsx.cisd.common.utilities.ISelfTestable;
+import ch.systemsx.cisd.common.utilities.OSUtilities;
+import ch.systemsx.cisd.openbis.generic.shared.dto.DataSetType;
+import ch.systemsx.cisd.openbis.generic.shared.dto.DatabaseInstancePE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ExperimentPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ExternalData;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ExtractableData;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ProcessingInstructionDTO;
+import ch.systemsx.cisd.openbis.generic.shared.dto.StorageFormat;
+
+/**
+ * The class that handles the incoming data set.
+ *
+ * @author Bernd Rinn
+ */
+public final class TransferredDataSetHandler implements IPathHandler, ISelfTestable
+{
+
+ private static final String TARGET_NOT_RELATIVE_TO_STORE_ROOT =
+ "Target path '%s' is not relative to store root directory '%s'.";
+
+ @Private
+ static final String DATA_SET_STORAGE_FAILURE_TEMPLATE = "Storing data set '%s' failed.";
+
+ @Private
+ static final String DATA_SET_REGISTRATION_FAILURE_TEMPLATE =
+ "Registration of data set '%s' failed.";
+
+ @Private
+ static final String SUCCESSFULLY_REGISTERED_TEMPLATE =
+ "Successfully registered data set '%s' for sample '%s', data set type '%s', "
+ + "experiment '%s' with openBIS service.";
+
+ @Private
+ static final String EMAIL_SUBJECT_TEMPLATE = "Success: data set for experiment '%s";
+
+ private static final Logger notificationLog =
+ LogFactory.getLogger(LogCategory.NOTIFY, TransferredDataSetHandler.class);
+
+ private static final Logger operationLog =
+ LogFactory.getLogger(LogCategory.OPERATION, TransferredDataSetHandler.class);
+
+ private static final NamedDataStrategy ERROR_DATA_STRATEGY =
+ new NamedDataStrategy(DataStoreStrategyKey.ERROR);
+
+ private final IStoreRootDirectoryHolder storeRootDirectoryHolder;
+
+ private final IEncapsulatedLimsService limsService;
+
+ private final IDataStrategyStore dataStrategyStore;
+
+ private final IDataSetInfoExtractor dataSetInfoExtractor;
+
+ private final IFileOperations fileOperations;
+
+ private final Lock registrationLock;
+
+ private final IProcedureAndDataTypeExtractor typeExtractor;
+
+ private final IStorageProcessor storageProcessor;
+
+ private final IMailClient mailClient;
+
+ private final String groupCode;
+
+ private final boolean notifySuccessfulRegistration;
+
+ private boolean stopped = false;
+
+ private Map<String, IProcessorFactory> processorFactories =
+ Collections.<String, IProcessorFactory> emptyMap();
+
+ private DatabaseInstancePE homeDatabaseInstance;
+
+ public TransferredDataSetHandler(final String groupCode, final IETLServerPlugin plugin,
+ final IEncapsulatedLimsService limsService, final Properties mailProperties,
+ final HighwaterMarkWatcher highwaterMarkWatcher,
+ final boolean notifySuccessfulRegistration)
+
+ {
+ this(groupCode, plugin.getStorageProcessor(), plugin, limsService, new MailClient(
+ mailProperties), notifySuccessfulRegistration);
+ }
+
+ TransferredDataSetHandler(final String groupCode,
+ final IStoreRootDirectoryHolder storeRootDirectoryHolder,
+ final IETLServerPlugin plugin, final IEncapsulatedLimsService limsService,
+ final IMailClient mailClient, final boolean notifySuccessfulRegistration)
+
+ {
+ assert storeRootDirectoryHolder != null : "Given store root directory holder can not be null.";
+ assert plugin != null : "IETLServerPlugin implementation can not be null.";
+ assert limsService != null : "IEncapsulatedLimsService implementation can not be null.";
+ assert mailClient != null : "IMailClient implementation can not be null.";
+
+ this.groupCode = groupCode;
+ this.storeRootDirectoryHolder = storeRootDirectoryHolder;
+ this.dataSetInfoExtractor = plugin.getDataSetInfoExtractor();
+ this.typeExtractor = plugin.getTypeExtractor();
+ this.storageProcessor = plugin.getStorageProcessor();
+ this.limsService = limsService;
+ this.mailClient = mailClient;
+ this.dataStrategyStore = new DataStrategyStore(this.limsService, mailClient);
+ this.notifySuccessfulRegistration = notifySuccessfulRegistration;
+ this.registrationLock = new ReentrantLock();
+ this.fileOperations = FileOperations.getMonitoredInstanceForCurrentThread();
+ }
+
+ public final void setProcessorFactories(final Map<String, IProcessorFactory> processorFactories)
+ {
+ assert processorFactories != null : "Unspecified processor factory map.";
+ this.processorFactories = processorFactories;
+ }
+
+ /**
+ * Returns the lock one needs to hold before one interrupts a data set registration.
+ */
+ public Lock getRegistrationLock()
+ {
+ return registrationLock;
+ }
+
+ //
+ // IPathHandler
+ //
+
+ public final void handle(final File isFinishedFile)
+ {
+ if (stopped)
+ {
+ return;
+ }
+ final RegistrationHelper registrationHelper = new RegistrationHelper(isFinishedFile);
+ registrationHelper.prepare();
+ if (registrationHelper.hasDataSetBeenIdentified())
+ {
+ registrationHelper.registerDataSet();
+ } else
+ {
+ registrationHelper.moveDataSet();
+ }
+ }
+
+ public boolean isStopped()
+ {
+ return stopped;
+ }
+
+ //
+ // ISelfTestable
+ //
+
+ public final void check() throws ConfigurationFailureException, EnvironmentFailureException
+ {
+ final File storeRootDirectory = storeRootDirectoryHolder.getStoreRootDirectory();
+ storeRootDirectory.mkdirs();
+ if (operationLog.isDebugEnabled())
+ {
+ operationLog.debug("Checking store root directory '"
+ + storeRootDirectory.getAbsolutePath() + "'.");
+ }
+ final String errorMessage =
+ fileOperations.checkDirectoryFullyAccessible(storeRootDirectory, "store root");
+ if (errorMessage != null)
+ {
+ if (fileOperations.exists(storeRootDirectory) == false)
+ {
+ throw EnvironmentFailureException.fromTemplate(
+ "Store root directory '%s' does not exist.", storeRootDirectory
+ .getAbsolutePath());
+ } else
+ {
+ throw new ConfigurationFailureException(errorMessage);
+ }
+ }
+ }
+
+ public boolean isRemote()
+ {
+ return true;
+ }
+
+ private DatabaseInstancePE getHomeDatabaseInstance()
+ {
+ if (homeDatabaseInstance == null)
+ {
+ homeDatabaseInstance = limsService.getHomeDatabaseInstance();
+ }
+ return homeDatabaseInstance;
+ }
+
+ //
+ // Helper class
+ //
+
+ private final class RegistrationHelper
+ {
+ private final File isFinishedFile;
+
+ private final File incomingDataSetFile;
+
+ private final DataSetInformation dataSetInformation;
+
+ private final IDataStoreStrategy dataStoreStrategy;
+
+ private final DataSetType dataSetType;
+
+ private final File storeRoot;
+
+ private BaseDirectoryHolder baseDirectoryHolder;
+
+ private String errorMessageTemplate;
+
+ RegistrationHelper(final File isFinishedFile)
+ {
+ assert isFinishedFile != null : "Unspecified is-finished file.";
+ final String name = isFinishedFile.getName();
+ assert name.startsWith(IS_FINISHED_PREFIX) : "A finished file must starts with '"
+ + IS_FINISHED_PREFIX + "'.";
+ errorMessageTemplate = DATA_SET_STORAGE_FAILURE_TEMPLATE;
+ this.isFinishedFile = isFinishedFile;
+ incomingDataSetFile = getIncomingDataSetPath(isFinishedFile);
+ dataSetInformation = extractDataSetInformation(incomingDataSetFile);
+ if (dataSetInformation.getDataSetCode() == null)
+ {
+ // Extractor didn't extract an externally generated data set code, so request one
+ // from the openBIS server.
+ dataSetInformation.setDataSetCode(limsService.createDataSetCode());
+ }
+ dataStoreStrategy =
+ dataStrategyStore.getDataStoreStrategy(dataSetInformation, incomingDataSetFile);
+ dataSetType = typeExtractor.getDataSetType(incomingDataSetFile);
+ storeRoot = storageProcessor.getStoreRootDirectory();
+ }
+
+ final void prepare()
+ {
+ final File baseDirectory =
+ createBaseDirectory(dataStoreStrategy, storeRoot, dataSetInformation);
+ baseDirectoryHolder =
+ new BaseDirectoryHolder(dataStoreStrategy, baseDirectory, incomingDataSetFile);
+ }
+
+ final boolean hasDataSetBeenIdentified()
+ {
+ return dataStoreStrategy.getKey() == DataStoreStrategyKey.IDENTIFIED;
+ }
+
+ /**
+ * This method is only ever called for identified data sets.
+ */
+ final void registerDataSet()
+ {
+ final ExperimentPE experiment = dataSetInformation.getExperiment();
+ final String procedureTypeCode =
+ typeExtractor.getProcedureType(incomingDataSetFile).getCode();
+ final IProcessor processorOrNull = tryCreateProcessor(procedureTypeCode);
+ try
+ {
+ registerDataSetAndInitiateProcessing(experiment, procedureTypeCode, processorOrNull);
+ logAndNotifySuccessfulRegistration(experiment.getRegistrator().getEmail());
+ if (fileOperations.exists(incomingDataSetFile)
+ && fileOperations.removeRecursivelyQueueing(incomingDataSetFile) == false)
+ {
+ operationLog.error("Cannot delete '" + incomingDataSetFile.getAbsolutePath()
+ + "'.");
+ }
+ deleteAndLogIsFinishedFile();
+ } catch (final Throwable throwable)
+ {
+ rollback(throwable);
+ }
+ }
+
+ private void rollback(final Throwable throwable) throws Error
+ {
+ stopped |= throwable instanceof StopException;
+ if (stopped)
+ {
+ Thread.interrupted(); // Ensure the thread's interrupted state is cleared.
+ operationLog.warn(String.format("Requested to stop registration of data set '%s'",
+ dataSetInformation));
+ } else
+ {
+ notificationLog.error(String.format(errorMessageTemplate, dataSetInformation),
+ throwable);
+ }
+ // Errors which are not AssertionErrors leave the system in a state that we don't
+ // know and can't trust. Thus we will not perform any operations any more in this
+ // case.
+ if (throwable instanceof Error && throwable instanceof AssertionError == false)
+ {
+ throw (Error) throwable;
+ }
+ storageProcessor.unstoreData(incomingDataSetFile, baseDirectoryHolder
+ .getBaseDirectory());
+ if (stopped == false)
+ {
+ final File baseDirectory =
+ createBaseDirectory(ERROR_DATA_STRATEGY, storeRoot, dataSetInformation);
+ baseDirectoryHolder =
+ new BaseDirectoryHolder(ERROR_DATA_STRATEGY, baseDirectory,
+ incomingDataSetFile);
+ boolean moveInCaseOfErrorOk =
+ FileRenamer.renameAndLog(incomingDataSetFile, baseDirectoryHolder
+ .getTargetFile());
+ writeThrowable(throwable);
+ if (moveInCaseOfErrorOk)
+ {
+ deleteAndLogIsFinishedFile();
+ }
+ }
+ }
+
+ /**
+ * Registers the data set and, if possible, initiates the processing.
+ */
+ private void registerDataSetAndInitiateProcessing(final ExperimentPE experiment,
+ final String procedureTypeCode, final IProcessor processorOrNull)
+ {
+ final File markerFile = createProcessingMarkerFile();
+ try
+ {
+ if (operationLog.isInfoEnabled())
+ {
+ operationLog.info("Start storing data set for sample '"
+ + dataSetInformation.getSampleIdentifier() + "'.");
+ }
+ final StopWatch watch = new StopWatch();
+ watch.start();
+ File dataFile =
+ storageProcessor.storeData(experiment, dataSetInformation, typeExtractor,
+ mailClient, incomingDataSetFile, baseDirectoryHolder
+ .getBaseDirectory());
+ if (operationLog.isInfoEnabled())
+ {
+ operationLog.info("Finished storing data set for sample '"
+ + dataSetInformation.getSampleIdentifier() + "', took " + watch);
+ }
+ assert dataFile != null : "The folder that contains the stored data should not be null.";
+ final String relativePath = FileUtilities.getRelativeFile(storeRoot, dataFile);
+ assert relativePath != null : String.format(TARGET_NOT_RELATIVE_TO_STORE_ROOT,
+ dataFile.getAbsolutePath(), storeRoot.getAbsolutePath());
+ final StorageFormat availableFormat = storageProcessor.getStorageFormat();
+ final BooleanOrUnknown isCompleteFlag = dataSetInformation.getIsCompleteFlag();
+ // Ensure that we either register the data set and initiate the processing copy or
+ // do none of both.
+ getRegistrationLock().lock();
+ try
+ {
+ errorMessageTemplate = DATA_SET_REGISTRATION_FAILURE_TEMPLATE;
+ plainRegisterDataSet(relativePath, procedureTypeCode, availableFormat,
+ isCompleteFlag);
+ deleteAndLogIsFinishedFile();
+ deleteAndLogIsFinishedFile();
+ if (processorOrNull == null)
+ {
+ return;
+ }
+ final StorageFormat requiredFormat =
+ processorOrNull.getRequiredInputDataFormat();
+ boolean canInitiateProcessing = requiredFormat.equals(availableFormat);
+ if (canInitiateProcessing == false
+ && availableFormatMayContainRequiredFormat(availableFormat,
+ requiredFormat))
+ {
+ // Special case: Check whether we can actually get back the original data.
+ dataFile = storageProcessor.tryGetProprietaryData(dataFile);
+ if (dataFile != null)
+ {
+ canInitiateProcessing = true;
+ }
+ }
+ if (canInitiateProcessing == false)
+ {
+ operationLog.error(String.format(
+ "Configuration Error: mismatch in data set format for data set '%s' between storage "
+ + "processor and processor (storage processor:"
+ + " %s, processor: %s) -> No processing initiated.",
+ dataSetInformation, availableFormat, requiredFormat));
+ notificationLog.error(String.format(
+ "Configuration Error: no processing initiated for data set '%s'",
+ dataSetInformation));
+ return;
+ }
+ final ProcessingInstructionDTO processingInstructionOrNull =
+ tryToGetAppropriateProcessingInstruction(experiment
+ .getProcessingInstructions(), procedureTypeCode);
+ if (processingInstructionOrNull != null)
+ {
+ try
+ {
+ processorOrNull.initiateProcessing(processingInstructionOrNull,
+ dataSetInformation, dataFile);
+ } catch (final RuntimeException e)
+ {
+ operationLog.error(
+ "Exception thrown when initiate processing for data set '"
+ + dataSetInformation + "'.", e);
+ notificationLog
+ .error("Couldn't initiate processing a data set for sample '"
+ + dataSetInformation.getSampleIdentifier()
+ + "' for some reason. For more details see log of the ETL Server.");
+ }
+ }
+ } finally
+ {
+ getRegistrationLock().unlock();
+ }
+ } finally
+ {
+ fileOperations.delete(markerFile);
+ }
+ }
+
+ private final File createProcessingMarkerFile()
+ {
+ final File baseDirectory = baseDirectoryHolder.getBaseDirectory();
+ final File baseParentDirectory = baseDirectory.getParentFile();
+ final String processingDirName = baseDirectory.getName();
+ final File markerFile =
+ new File(baseParentDirectory, Constants.PROCESSING_PREFIX + processingDirName);
+ try
+ {
+ fileOperations.createNewFile(markerFile);
+ } catch (final WrappedIOException ex)
+ {
+ throw EnvironmentFailureException.fromTemplate(ex,
+ "Cannot create marker file '%s'.", markerFile.getPath());
+ }
+ return markerFile;
+ }
+
+ private boolean availableFormatMayContainRequiredFormat(
+ final StorageFormat availableFormat, final StorageFormat requiredFormat)
+ {
+ return StorageFormat.PROPRIETARY.equals(requiredFormat)
+ && StorageFormat.BDS_DIRECTORY.equals(availableFormat);
+ }
+
+ /**
+ * This method is only ever called for unidentified or invalid data sets.
+ */
+ final void moveDataSet()
+ {
+ final boolean ok =
+ FileRenamer.renameAndLog(incomingDataSetFile, baseDirectoryHolder
+ .getTargetFile());
+ if (ok)
+ {
+ deleteAndLogIsFinishedFile();
+ }
+ }
+
+ private final void plainRegisterDataSet(final String relativePath,
+ final String procedureTypeCode, final StorageFormat storageFormat,
+ final BooleanOrUnknown isCompleteFlag)
+ {
+ final ExternalData data =
+ createExternalData(relativePath, storageFormat, isCompleteFlag);
+ // Finally: register the data set in the database.
+ limsService.registerDataSet(dataSetInformation, procedureTypeCode, data);
+ }
+
+ private void logAndNotifySuccessfulRegistration(final String email)
+ {
+ String msg = null;
+ if (operationLog.isInfoEnabled())
+ {
+ msg = getSuccessRegistrationMessage();
+ operationLog.info(msg);
+ }
+ if (notifySuccessfulRegistration)
+ {
+ if (msg == null)
+ {
+ msg = getSuccessRegistrationMessage();
+ }
+ if (notificationLog.isInfoEnabled())
+ {
+ notificationLog.info(msg);
+ }
+ if (StringUtils.isBlank(email) == false)
+ {
+ mailClient.sendMessage(String.format(EMAIL_SUBJECT_TEMPLATE, dataSetInformation
+ .getExperimentIdentifier().getExperimentCode()), msg, null, email);
+ }
+ }
+ }
+
+ private final String getSuccessRegistrationMessage()
+ {
+ final StringBuilder buffer = new StringBuilder();
+ buffer.append(String.format(SUCCESSFULLY_REGISTERED_TEMPLATE, dataSetInformation
+ .getDataSetCode(), dataSetInformation.getSampleIdentifier(), dataSetType
+ .getCode(), dataSetInformation.getExperimentIdentifier()));
+ buffer.append(OSUtilities.LINE_SEPARATOR);
+ buffer.append(OSUtilities.LINE_SEPARATOR);
+ appendNameAndObject(buffer, "Experiment Identifier", dataSetInformation
+ .getExperimentIdentifier());
+ appendNameAndObject(buffer, "Producer Code", dataSetInformation.getProducerCode());
+ appendNameAndObject(buffer, "Production Date", dataSetInformation.getProductionDate());
+ appendNameAndObject(buffer, "Parent Data Set", StringUtils
+ .trimToNull(dataSetInformation.getParentDataSetCode()));
+ appendNameAndObject(buffer, "Is complete", dataSetInformation.getIsCompleteFlag());
+ return buffer.toString();
+ }
+
+ private final void appendNameAndObject(final StringBuilder buffer, final String name,
+ final Object object)
+ {
+ if (object != null)
+ {
+ buffer.append(name).append(":\t").append(object);
+ buffer.append(OSUtilities.LINE_SEPARATOR);
+ }
+ }
+
+ /**
+ * From given <var>isFinishedPath</var> gets the incoming data set path and checks it.
+ *
+ * @return <code>null</code> if a problem has happened. Otherwise a useful and usable
+ * incoming data set path is returned.
+ */
+ private final File getIncomingDataSetPath(final File isFinishedPath)
+ {
+ final File incomingDataSetPath =
+ FileUtilities.removePrefixFromFileName(isFinishedPath, IS_FINISHED_PREFIX);
+ if (operationLog.isDebugEnabled())
+ {
+ operationLog.debug(String.format(
+ "Getting incoming data set path '%s' from is-finished path '%s'",
+ incomingDataSetPath, isFinishedPath));
+ }
+ final String errorMsg =
+ fileOperations.checkPathFullyAccessible(incomingDataSetPath,
+ "incoming data set");
+ if (errorMsg != null)
+ {
+ fileOperations.delete(isFinishedPath);
+ throw EnvironmentFailureException.fromTemplate(String.format(
+ "Error moving path '%s' from '%s' to '%s': %s", incomingDataSetPath
+ .getName(), incomingDataSetPath.getParent(),
+ storeRootDirectoryHolder.getStoreRootDirectory(), errorMsg));
+ }
+ return incomingDataSetPath;
+ }
+
+ /**
+ * From given <var>incomingDataSetPath</var> extracts a <code>DataSetInformation</code>.
+ *
+ * @return never <code>null</code> but prefers to throw an exception.
+ */
+ private final DataSetInformation extractDataSetInformation(final File incomingDataSetPath)
+ {
+ try
+ {
+ final DataSetInformation dataSetInfo =
+ dataSetInfoExtractor.getDataSetInformation(incomingDataSetPath);
+ if (dataSetInfo.getSampleIdentifier() == null)
+ {
+ final String extractorName = dataSetInfoExtractor.getClass().getSimpleName();
+ throw ConfigurationFailureException.fromTemplate(
+ "Data Set Information Extractor '%s' extracted no sample code "
+ + "for incoming data set '%s' (extractor contract violation).",
+ extractorName, incomingDataSetPath);
+ }
+ dataSetInfo.setInstanceCode(getHomeDatabaseInstance().getCode());
+ dataSetInfo.setInstanceUUID(getHomeDatabaseInstance().getUuid());
+ if (dataSetInfo.getGroupCode() == null)
+ {
+ dataSetInfo.setGroupCode(groupCode);
+ }
+ if (operationLog.isDebugEnabled())
+ {
+ operationLog.debug(String.format(
+ "Extracting data set information '%s' from incoming "
+ + "data set path '%s'", dataSetInfo, incomingDataSetPath));
+ }
+ return dataSetInfo;
+ } catch (final HighLevelException e)
+ {
+ throw e;
+ } catch (final RuntimeException ex)
+ {
+ throw new EnvironmentFailureException("Error when trying to identify data set '"
+ + incomingDataSetPath.getAbsolutePath() + "'.", ex);
+ }
+ }
+
+ private final File createBaseDirectory(final IDataStoreStrategy strategy,
+ final File baseDir, final DataSetInformation dataSetInfo)
+ {
+ final File baseDirectory =
+ strategy.getBaseDirectory(baseDir, dataSetInfo, dataSetType);
+ baseDirectory.mkdirs();
+ if (fileOperations.isDirectory(baseDirectory) == false)
+ {
+ throw EnvironmentFailureException.fromTemplate(
+ "Creating data set base directory '%s' for data set '%s' failed.",
+ baseDirectory.getAbsolutePath(), incomingDataSetFile);
+ }
+ return baseDirectory;
+ }
+
+ private IProcessor tryCreateProcessor(final String procedureTypeCode)
+ {
+ final IProcessorFactory processorFactory = processorFactories.get(procedureTypeCode);
+ if (processorFactory == null)
+ {
+ return null;
+ }
+ return processorFactory.createProcessor();
+ }
+
+ private ProcessingInstructionDTO tryToGetAppropriateProcessingInstruction(
+ final ProcessingInstructionDTO[] processingInstructions,
+ final String procedureTypeCode)
+ {
+ if (processingInstructions != null)
+ {
+ for (final ProcessingInstructionDTO instruction : processingInstructions)
+ {
+ if (instruction.getProcedureTypeCode().equals(procedureTypeCode))
+ {
+ return instruction;
+ }
+ }
+ }
+ return null;
+ }
+
+ private final ExternalData createExternalData(final String relativePath,
+ final StorageFormat storageFormat, final BooleanOrUnknown isCompleteFlag)
+ {
+ final ExtractableData extractableData = dataSetInformation.getExtractableData();
+ final ExternalData data = BeanUtils.createBean(ExternalData.class, extractableData);
+ data.setLocation(relativePath);
+ data.setLocatorType(typeExtractor.getLocatorType(incomingDataSetFile));
+ data.setDataSetType(typeExtractor.getDataSetType(incomingDataSetFile));
+ data.setFileFormatType(typeExtractor.getFileFormatType(incomingDataSetFile));
+ data.setStorageFormat(storageFormat);
+ data.setComplete(isCompleteFlag);
+ return data;
+ }
+
+ private final void writeThrowable(final Throwable throwable)
+ {
+ final String fileName = incomingDataSetFile.getName() + ".exception";
+ final File file =
+ new File(baseDirectoryHolder.getTargetFile().getParentFile(), fileName);
+ FileWriter writer = null;
+ try
+ {
+ writer = new FileWriter(file);
+ throwable.printStackTrace(new PrintWriter(writer));
+ } catch (final IOException e)
+ {
+ operationLog.warn(String.format(
+ "Could not write out the exception '%s' in file '%s'.", fileName, file
+ .getAbsolutePath()), e);
+ } finally
+ {
+ IOUtils.closeQuietly(writer);
+ }
+ }
+
+ private boolean deleteAndLogIsFinishedFile()
+ {
+ if (fileOperations.exists(isFinishedFile) == false)
+ {
+ return false;
+ }
+ final boolean ok = fileOperations.delete(isFinishedFile);
+ final String absolutePath = isFinishedFile.getAbsolutePath();
+ if (ok == false)
+ {
+ notificationLog.error(String.format("Removing file '%s' failed.", absolutePath));
+ } else
+ {
+ if (operationLog.isDebugEnabled())
+ {
+ operationLog.debug(String.format("File '%s' has been removed.", absolutePath));
+ }
+ }
+ return ok;
+ }
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/imsb/HCSImageFileExtractor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/imsb/HCSImageFileExtractor.java
new file mode 100644
index 00000000000..6112d16c551
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/imsb/HCSImageFileExtractor.java
@@ -0,0 +1,335 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.imsb;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+
+import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
+
+import ch.rinn.restrictions.Private;
+import ch.systemsx.cisd.bds.hcs.Channel;
+import ch.systemsx.cisd.bds.hcs.Geometry;
+import ch.systemsx.cisd.bds.hcs.Location;
+import ch.systemsx.cisd.bds.hcs.WellGeometry;
+import ch.systemsx.cisd.bds.storage.IDirectory;
+import ch.systemsx.cisd.bds.storage.IFile;
+import ch.systemsx.cisd.bds.storage.INode;
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.exceptions.InvalidExternalDataException;
+import ch.systemsx.cisd.common.exceptions.StopException;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.etlserver.ChannelSetHelper;
+import ch.systemsx.cisd.etlserver.DataSetInformation;
+import ch.systemsx.cisd.etlserver.HCSImageFileExtractionResult;
+import ch.systemsx.cisd.etlserver.IHCSImageFileAccepter;
+import ch.systemsx.cisd.etlserver.IHCSImageFileExtractor;
+
+/**
+ * A <code>IHCSImageFileExtractor</code> implementation suitable for <i>3V</i>.
+ * <p>
+ * This implementation extracts and processes image files having the format
+ *
+ * <code>Screening_<well id>_s<tile number>_w<channel number>_[<some UUID that we can just ignore>].tif</code>
+ * . An example is <code>Screening_H24_s6_w1_[UUID].tif</code>.
+ * </p>
+ *
+ * @author Christian Ribeaud
+ * @author Bernd Rinn
+ */
+// @Final
+public class HCSImageFileExtractor implements IHCSImageFileExtractor
+{
+ private static final String TIFF_SUBDIRECTORY = "TIFF";
+
+ private static final Logger operationLog =
+ LogFactory.getLogger(LogCategory.OPERATION, HCSImageFileExtractor.class);
+
+ static final String IMAGE_FILE_NOT_STANDARDIZABLE =
+ "Image file '%s' could not be standardized given following tokens [plateLocation=%s,wellLocation=%s,channel=%s].";
+
+ static final String IMAGE_FILE_NOT_ENOUGH_ENTITIES =
+ "Image file '%s' does not have enough entities.";
+
+ static final String IMAGE_FILE_BELONGS_TO_WRONG_SAMPLE =
+ "Image file '%s' belongs to the wrong sample [expected=%s,found=%s].";
+
+ static final String IMAGE_FILE_ACCEPTED =
+ "Image file '%s' was accepted for channel %d, plate location %s and well location %s.";
+
+ static final char TOKEN_SEPARATOR = '_';
+
+ private final Geometry wellGeometry;
+
+ public HCSImageFileExtractor(final Properties properties)
+ {
+ assert properties != null : "Given properites should not be null";
+ wellGeometry = getWellGeometry(properties);
+ }
+
+ private final static Geometry getWellGeometry(final Properties properties)
+ {
+ final String property = properties.getProperty(WellGeometry.WELL_GEOMETRY);
+ if (property == null)
+ {
+ throw new ConfigurationFailureException(String.format(
+ "No '%s' property has been specified.", WellGeometry.WELL_GEOMETRY));
+ }
+ final Geometry geometry = WellGeometry.createFromString(property);
+ if (geometry == null)
+ {
+ throw new ConfigurationFailureException(String.format(
+ "Could not create a geometry from property value '%s'.", property));
+ }
+ return geometry;
+ }
+
+ /**
+ * Extracts the well location from given <var>value</var>, following the convention adopted
+ * here.
+ * <p>
+ * Returns <code>null</code> if the operation fails.
+ * </p>
+ */
+ private final Location tryGetWellLocation(final String value)
+ {
+ try
+ {
+ return Location.tryCreateLocationFromPosition(Integer.parseInt(value), wellGeometry);
+ } catch (final NumberFormatException ex)
+ {
+ // Nothing to do here. Rest of the code can handle this.
+ }
+ return null;
+ }
+
+ /**
+ * Extracts the plate location from given <var>value</var>, following the convention adopted
+ * here.
+ * <p>
+ * Returns <code>null</code> if the operation fails.
+ * </p>
+ */
+ private final static Location tryGetPlateLocation(final String value)
+ {
+ return Location.tryCreateLocationFromMatrixCoordinate(value);
+ }
+
+ /**
+ * Extracts the channel from given <var>value</var>, following the convention adopted here.
+ * <p>
+ * Returns <code>0</code> if the operation fails.
+ * </p>
+ */
+ private final int getChannelWavelength(final String value)
+ {
+ final String startsWith = "w";
+ if (value.startsWith(startsWith))
+ {
+ try
+ {
+ return Integer.parseInt(value.substring(startsWith.length()));
+ } catch (final NumberFormatException ex)
+ {
+ // Nothing to do here. Rest of the code can handle this.
+ }
+ }
+ return 0;
+ }
+
+ private static class ImageFileRecord
+ {
+ final IFile imageFile;
+
+ final Location plateLocation;
+
+ final Location wellLocation;
+
+ final int channelWavelength;
+
+ ImageFileRecord(final IFile imageFile, final Location plateLocation,
+ final Location wellLocation, final int channelWavelength)
+ {
+ this.imageFile = imageFile;
+ this.plateLocation = plateLocation;
+ this.wellLocation = wellLocation;
+ this.channelWavelength = channelWavelength;
+ }
+ }
+
+ /** Perform channel wavelength sorting on images. */
+ private static class ChannelWavelengthSortingHCSImageFileAccepterDecorator implements
+ IHCSImageFileAccepter
+ {
+
+ private final IHCSImageFileAccepter accepter;
+
+ private final List<ImageFileRecord> images = new ArrayList<ImageFileRecord>();
+
+ private final ChannelSetHelper helper;
+
+ ChannelWavelengthSortingHCSImageFileAccepterDecorator(final IHCSImageFileAccepter accepter)
+ {
+ this.accepter = accepter;
+ helper = new ChannelSetHelper();
+ }
+
+ /**
+ * Returns the set of <code>Channels</code>.
+ */
+ final Set<Channel> getChannels()
+ {
+ return helper.getChannelSet();
+ }
+
+ /**
+ * Informs that {@link #accept(int, Location, Location, IFile)} will no longer get called.
+ * <p>
+ * We are now ready to construct the channels and to commit the images to the encapsulated
+ * <code>IHCSImageFileAccepter</code>.
+ * </p>
+ */
+ final void commit()
+ {
+ for (final ImageFileRecord image : images)
+ {
+ accepter.accept(helper.getChannelForWavelength(image.channelWavelength)
+ .getCounter(), image.plateLocation, image.wellLocation, image.imageFile);
+ }
+ }
+
+ //
+ // IHCSImageFileAccepter
+ //
+
+ public final void accept(final int channelWavelength, final Location wellLocation,
+ final Location tileLocation, final IFile imageFile)
+ {
+ images
+ .add(new ImageFileRecord(imageFile, wellLocation, tileLocation,
+ channelWavelength));
+ helper.addWavelength(channelWavelength);
+ }
+
+ }
+
+ //
+ // IHCSImageFileExtractor
+ //
+
+ public final HCSImageFileExtractionResult process(final IDirectory incomingDataSetDirectory,
+ final DataSetInformation dataSetInformation, final IHCSImageFileAccepter accepter)
+ {
+ assert incomingDataSetDirectory != null;
+ final List<IFile> imageFiles = listTiffFiles(incomingDataSetDirectory);
+ final long start = System.currentTimeMillis();
+ final List<IFile> invalidFiles = new LinkedList<IFile>();
+ final ChannelWavelengthSortingHCSImageFileAccepterDecorator accepterDecorator =
+ new ChannelWavelengthSortingHCSImageFileAccepterDecorator(accepter);
+ for (final IFile imageFile : imageFiles)
+ {
+ StopException.check();
+ if (operationLog.isDebugEnabled())
+ {
+ operationLog.debug(String.format("Processing image file '%s'", imageFile));
+ }
+ final String baseName = FilenameUtils.getBaseName(imageFile.getPath());
+ final String[] tokens = StringUtils.split(baseName, TOKEN_SEPARATOR);
+ if (tokens.length < 4)
+ {
+ if (operationLog.isDebugEnabled())
+ {
+ operationLog.debug(String.format(IMAGE_FILE_NOT_ENOUGH_ENTITIES, imageFile));
+ }
+ invalidFiles.add(imageFile);
+ continue;
+ }
+ final String sampleCode = tokens[tokens.length - 4];
+ if (dataSetInformation.getSampleIdentifier().getSampleCode().equals(sampleCode) == false)
+ {
+ if (operationLog.isDebugEnabled())
+ {
+ operationLog.debug(String.format(IMAGE_FILE_BELONGS_TO_WRONG_SAMPLE, imageFile,
+ dataSetInformation.getSampleIdentifier(), sampleCode));
+ }
+ invalidFiles.add(imageFile);
+ continue;
+ }
+ final String plateLocationStr = tokens[tokens.length - 3];
+ final Location plateLocation = tryGetPlateLocation(plateLocationStr);
+ final String wellLocationStr = tokens[tokens.length - 2];
+ final Location wellLocation = tryGetWellLocation(wellLocationStr);
+ final String channelStr = tokens[tokens.length - 1];
+ final int channelWavelength = getChannelWavelength(channelStr);
+ if (wellLocation != null && plateLocation != null && channelWavelength > 0)
+ {
+ accepterDecorator.accept(channelWavelength, plateLocation, wellLocation, imageFile);
+ if (operationLog.isDebugEnabled())
+ {
+ operationLog.debug(String.format(IMAGE_FILE_ACCEPTED, imageFile,
+ channelWavelength, plateLocation, wellLocation));
+ }
+ } else
+ {
+ if (operationLog.isDebugEnabled())
+ {
+ operationLog.debug(String.format(IMAGE_FILE_NOT_STANDARDIZABLE, imageFile,
+ plateLocationStr, wellLocationStr, channelStr));
+ }
+ invalidFiles.add(imageFile);
+ }
+ }
+ accepterDecorator.commit();
+ return new HCSImageFileExtractionResult(System.currentTimeMillis() - start, imageFiles
+ .size(), Collections.unmodifiableList(invalidFiles), accepterDecorator
+ .getChannels());
+ }
+
+ private IDirectory getTiffSubDirectory(final IDirectory incomingDataSetDirectory)
+ {
+ final INode tiffSubDirectoryNodeOrNull =
+ incomingDataSetDirectory.tryGetNode(TIFF_SUBDIRECTORY);
+ if (tiffSubDirectoryNodeOrNull == null)
+ {
+ throw InvalidExternalDataException.fromTemplate(
+ "The directory '%s' does not have a sub-directory '%s'.",
+ incomingDataSetDirectory.getPath(), TIFF_SUBDIRECTORY);
+ }
+ final IDirectory tiffSubDirectoryOrNull = tiffSubDirectoryNodeOrNull.tryAsDirectory();
+ if (tiffSubDirectoryOrNull == null)
+ {
+ throw InvalidExternalDataException.fromTemplate("The file '%s/%s' is not a directory.",
+ incomingDataSetDirectory.getPath(), TIFF_SUBDIRECTORY);
+ }
+ return tiffSubDirectoryOrNull;
+ }
+
+ @Private
+ List<IFile> listTiffFiles(final IDirectory directory)
+ {
+ final IDirectory tiffSubDirectory = getTiffSubDirectory(directory);
+ return tiffSubDirectory.listFiles(new String[] { "tif", "tiff" }, true);
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/threev/AbstractDataSetInfoExtractorFor3V.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/threev/AbstractDataSetInfoExtractorFor3V.java
new file mode 100644
index 00000000000..2b04add8cbd
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/threev/AbstractDataSetInfoExtractorFor3V.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.threev;
+
+import java.io.File;
+import java.util.Properties;
+
+import org.apache.commons.lang.StringUtils;
+
+import ch.rinn.restrictions.Private;
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.common.exceptions.UserFailureException;
+import ch.systemsx.cisd.common.utilities.PropertyUtils;
+import ch.systemsx.cisd.etlserver.AbstractDataSetInfoExtractor;
+import ch.systemsx.cisd.etlserver.DataSetInformation;
+import ch.systemsx.cisd.etlserver.DataSetNameEntitiesProvider;
+import ch.systemsx.cisd.etlserver.DefaultDataSetInfoExtractor;
+
+/**
+ * @author Franz-Josef Elmer
+ */
+abstract class AbstractDataSetInfoExtractorFor3V extends AbstractDataSetInfoExtractor
+{
+ /**
+ * Name of the property specifying the character which will be used to concatenate the entities
+ * specifying the data set code.
+ */
+ @Private
+ static final String DATA_SET_CODE_ENTITIES_GLUE = "data-set-code-entities-glue";
+
+ private static final String DEFAULT_DATA_SET_CODE_ENTITIES_GLUE = ".";
+
+ private final DefaultDataSetInfoExtractor codeExtractor;
+
+ private final int[] dataSetCodeIndices;
+
+ private final String dataSetCodeEntitiesGlue;
+
+ public AbstractDataSetInfoExtractorFor3V(final Properties globalProperties,
+ final String indicesPropertyName)
+ {
+ super(globalProperties);
+ codeExtractor = new DefaultDataSetInfoExtractor(globalProperties);
+ final String indicesAsString =
+ PropertyUtils.getMandatoryProperty(properties, indicesPropertyName);
+ final String[] indicesAsStringArray = StringUtils.split(indicesAsString, ", ");
+ dataSetCodeIndices = new int[indicesAsStringArray.length];
+ for (int i = 0; i < indicesAsStringArray.length; i++)
+ {
+ final String index = indicesAsStringArray[i];
+ try
+ {
+ dataSetCodeIndices[i] = Integer.parseInt(index);
+ } catch (final NumberFormatException ex)
+ {
+ throw new ConfigurationFailureException(i + 1 + ". index in property '"
+ + indicesPropertyName + "' isn't a number: " + indicesAsString);
+ }
+ }
+ dataSetCodeEntitiesGlue =
+ properties.getProperty(DATA_SET_CODE_ENTITIES_GLUE,
+ DEFAULT_DATA_SET_CODE_ENTITIES_GLUE);
+ }
+
+ //
+ // AbstractCodeExtractor
+ //
+
+ public DataSetInformation getDataSetInformation(final File incomingDataSetPath)
+ throws UserFailureException, EnvironmentFailureException
+ {
+ final DataSetInformation dataSetInfo =
+ codeExtractor.getDataSetInformation(incomingDataSetPath);
+ final DataSetNameEntitiesProvider entitiesProvider =
+ new DataSetNameEntitiesProvider(incomingDataSetPath, entitySeparator,
+ stripExtension);
+ final StringBuilder builder = new StringBuilder();
+ for (final int index : dataSetCodeIndices)
+ {
+ if (builder.length() > 0)
+ {
+ builder.append(dataSetCodeEntitiesGlue);
+ }
+ builder.append(entitiesProvider.getEntity(index));
+ }
+ final String code = builder.toString();
+ setCodeFor(dataSetInfo, code);
+ return dataSetInfo;
+ }
+
+ protected abstract void setCodeFor(DataSetInformation dataSetInfo, String code);
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/threev/DataSetInfoExtractorForDataAcquisition.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/threev/DataSetInfoExtractorForDataAcquisition.java
new file mode 100644
index 00000000000..283b58a2195
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/threev/DataSetInfoExtractorForDataAcquisition.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.threev;
+
+import java.util.Properties;
+
+import ch.rinn.restrictions.Private;
+import ch.systemsx.cisd.etlserver.DataSetInformation;
+import ch.systemsx.cisd.etlserver.DefaultDataSetInfoExtractor;
+
+/**
+ * Implementation which assumes that the information can be extracted from the file name. Following
+ * information can be extracted:
+ * <ul>
+ * <li>Sample code
+ * <li>Parent data set code
+ * <li>Data producer code
+ * <li>Data production date
+ * <li>Data set code
+ * </ul>
+ * This class uses the same properties as {@link DefaultDataSetInfoExtractor}. In addition the
+ * following properties are used to extract the data set code: <table border="1" cellspacing="0"
+ * cellpadding="5">
+ * <tr>
+ * <th>Property</th>
+ * <th>Default value</th>
+ * <th>Description</th>
+ * </tr>
+ * <tr>
+ * <td><code>indices-of-data-set-code-entities</code></td>
+ * <td> </td>
+ * <td>Space or comma separated list of entity indices which define the data set code uniquely.
+ * This is a mandatory property.</td>
+ * </tr>
+ * <tr>
+ * <td><code>data-set-code-entities-glue</code></td>
+ * <td><code>.</code></td>
+ * <td>Symbol used to concatenate entities defining the data set code.</td>
+ * </tr>
+ * </table> The first entity has index 0, the second 1, etc. Using negative numbers one can specify
+ * entities from the end. Thus, -1 means the last entity, -2 the second last entity, etc.
+ *
+ * @author Franz-Josef Elmer
+ */
+public class DataSetInfoExtractorForDataAcquisition extends AbstractDataSetInfoExtractorFor3V
+{
+ /**
+ * Name of the property specifying the indices of those entities which define uniquely the data
+ * set code.
+ * <p>
+ * Use a negative number to count from the end, e.g. <code>-1</code> indicates the last
+ * entity.
+ * </p>
+ */
+ @Private
+ static final String INDICES_OF_DATA_SET_CODE_ENTITIES = "indices-of-data-set-code-entities";
+
+ public DataSetInfoExtractorForDataAcquisition(final Properties properties)
+ {
+ super(properties, INDICES_OF_DATA_SET_CODE_ENTITIES);
+ }
+
+ @Override
+ protected void setCodeFor(final DataSetInformation dataSetInfo, final String code)
+ {
+ dataSetInfo.setDataSetCode(code);
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/threev/DataSetInfoExtractorForImageAnalysis.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/threev/DataSetInfoExtractorForImageAnalysis.java
new file mode 100644
index 00000000000..211c053e724
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/threev/DataSetInfoExtractorForImageAnalysis.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.threev;
+
+import java.util.Properties;
+
+import ch.rinn.restrictions.Private;
+import ch.systemsx.cisd.etlserver.DataSetInformation;
+import ch.systemsx.cisd.etlserver.DefaultDataSetInfoExtractor;
+
+/**
+ * Implementation which assumes that the information can be extracted from the file name. Following
+ * information can be extracted:
+ * <ul>
+ * <li>Sample code
+ * <li>Parent data set code
+ * <li>Data producer code
+ * <li>Data production date
+ * </ul>
+ * This class uses the same properties as {@link DefaultDataSetInfoExtractor} except
+ * <code>index-of-parent-data-set-code</code>. Instead the following properties are used to
+ * extract the parent data set code: <table border="1" cellspacing="0" cellpadding="5">
+ * <tr>
+ * <th>Property</th>
+ * <th>Default value</th>
+ * <th>Description</th>
+ * </tr>
+ * <tr>
+ * <td><code>indices-of-parent-data-set-code-entities</code></td>
+ * <td> </td>
+ * <td>Space or comma separated list of entity indices which define the parent data set code
+ * uniquely. This is a mandatory property.</td>
+ * </tr>
+ * <tr>
+ * <td><code>data-set-code-entities-glue</code></td>
+ * <td><code>.</code></td>
+ * <td>Symbol used to concatenate entities defining the parent data set code.</td>
+ * </tr>
+ * </table> The first entity has index 0, the second 1, etc. Using negative numbers one can specify
+ * entities from the end. Thus, -1 means the last entity, -2 the second last entity, etc.
+ *
+ * @author Franz-Josef Elmer
+ */
+public class DataSetInfoExtractorForImageAnalysis extends AbstractDataSetInfoExtractorFor3V
+{
+ /**
+ * Name of the property specifying the indices of those entities which define uniquely the
+ * parent data set code.
+ * <p>
+ * Use a negative number to count from the end, e.g. <code>-1</code> indicates the last
+ * entity.
+ * </p>
+ */
+ @Private
+ static final String INDICES_OF_PARENT_DATA_SET_CODE_ENTITIES =
+ "indices-of-parent-data-set-code-entities";
+
+ public DataSetInfoExtractorForImageAnalysis(final Properties properties)
+ {
+ super(properties, INDICES_OF_PARENT_DATA_SET_CODE_ENTITIES);
+ }
+
+ @Override
+ protected void setCodeFor(final DataSetInformation dataSetInfo, final String code)
+ {
+ dataSetInfo.setParentDataSetCode(code);
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/threev/HCSImageFileExtractor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/threev/HCSImageFileExtractor.java
new file mode 100644
index 00000000000..c250dfaebfe
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/threev/HCSImageFileExtractor.java
@@ -0,0 +1,211 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.threev;
+
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
+
+import ch.systemsx.cisd.bds.hcs.Geometry;
+import ch.systemsx.cisd.bds.hcs.Location;
+import ch.systemsx.cisd.bds.hcs.WellGeometry;
+import ch.systemsx.cisd.bds.storage.IDirectory;
+import ch.systemsx.cisd.bds.storage.IFile;
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.exceptions.StopException;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.etlserver.ChannelSetHelper;
+import ch.systemsx.cisd.etlserver.DataSetInformation;
+import ch.systemsx.cisd.etlserver.HCSImageFileExtractionResult;
+import ch.systemsx.cisd.etlserver.IHCSImageFileAccepter;
+import ch.systemsx.cisd.etlserver.IHCSImageFileExtractor;
+
+/**
+ * A <code>IHCSImageFileExtractor</code> implementation suitable for <i>3V</i>.
+ * <p>
+ * This implementation extracts and processes image files having the format
+ *
+ * <code>Screening_<well id>_s<tile number>_w<channel number>_[<some UUID that we can just ignore>].tif</code>
+ * . An example is <code>Screening_H24_s6_w1_[UUID].tif</code>.
+ * </p>
+ *
+ * @author Christian Ribeaud
+ */
+public final class HCSImageFileExtractor implements IHCSImageFileExtractor
+{
+ private static final Logger operationLog =
+ LogFactory.getLogger(LogCategory.OPERATION, HCSImageFileExtractor.class);
+
+ static final String IMAGE_FILE_NOT_STANDARDIZABLE =
+ "Image file '%s' could not be standardized given following tokens [plateLocation=%s,wellLocation=%s,channel=%s].";
+
+ static final String IMAGE_FILE_ACCEPTED =
+ "Image file '%s' was accepted for channel %d, plate location %s and well location %s.";
+
+ static final String FILE_PREFIX = "Screening_";
+
+ static final int TOKEN_NUMBER = 5;
+
+ static final char TOKEN_SEPARATOR = '_';
+
+ private final Geometry wellGeometry;
+
+ public HCSImageFileExtractor(final Properties properties)
+ {
+ assert properties != null : "Given properites should not be null";
+ wellGeometry = getWellGeometry(properties);
+ }
+
+ private final static Geometry getWellGeometry(final Properties properties)
+ {
+ final String property = properties.getProperty(WellGeometry.WELL_GEOMETRY);
+ if (property == null)
+ {
+ throw new ConfigurationFailureException(String.format(
+ "No '%s' property has been specified.", WellGeometry.WELL_GEOMETRY));
+ }
+ final Geometry geometry = WellGeometry.createFromString(property);
+ if (geometry == null)
+ {
+ throw new ConfigurationFailureException(String.format(
+ "Could not create a geometry from property value '%s'.", property));
+ }
+ return geometry;
+ }
+
+ /**
+ * Extracts the well location from given <var>value</var>, following the convention adopted
+ * here.
+ * <p>
+ * Returns <code>null</code> if the operation fails.
+ * </p>
+ */
+ private final Location tryGetWellLocation(final String value)
+ {
+ final String startsWith = "s";
+ if (value.startsWith(startsWith))
+ {
+ final String tileNo = value.substring(startsWith.length());
+ try
+ {
+ return Location.tryCreateLocationFromPosition(Integer.parseInt(tileNo),
+ wellGeometry);
+ } catch (NumberFormatException ex)
+ {
+ // Nothing to do here. Rest of the code can handle this.
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Extracts the plate location from given <var>value</var>, following the convention adopted
+ * here.
+ * <p>
+ * Returns <code>null</code> if the operation fails.
+ * </p>
+ */
+ private final static Location tryGetPlateLocation(final String value)
+ {
+ return Location.tryCreateLocationFromMatrixCoordinate(value);
+ }
+
+ /**
+ * Extracts the wavelength from given <var>value</var>, following the convention adopted here.
+ * <p>
+ * Returns <code>0</code> if the operation fails.
+ * </p>
+ */
+ private final int getWavelength(final String value)
+ {
+ final String startsWith = "w";
+ if (value.startsWith(startsWith))
+ {
+ try
+ {
+ return Integer.parseInt(value.substring(startsWith.length()));
+ } catch (NumberFormatException ex)
+ {
+ // Nothing to do here. Rest of the code can handle this.
+ }
+ }
+ return 0;
+ }
+
+ //
+ // IHCSImageFileExtractor
+ //
+
+ public final HCSImageFileExtractionResult process(final IDirectory incomingDataSetDirectory,
+ DataSetInformation dataSetInformation, final IHCSImageFileAccepter accepter)
+ {
+ assert incomingDataSetDirectory != null;
+ final List<IFile> imageFiles = incomingDataSetDirectory.listFiles(new String[]
+ { "tif", "tiff" }, true);
+ final long start = System.currentTimeMillis();
+ final List<IFile> invalidFiles = new LinkedList<IFile>();
+ final ChannelSetHelper helper = new ChannelSetHelper();
+ for (final IFile imageFile : imageFiles)
+ {
+ StopException.check();
+ if (operationLog.isDebugEnabled())
+ {
+ operationLog.debug(String.format("Processing image file '%s'", imageFile));
+ }
+ final String baseName = FilenameUtils.getBaseName(imageFile.getPath());
+ if (baseName.startsWith(FILE_PREFIX) == false)
+ {
+ continue;
+ }
+ final String[] tokens = StringUtils.split(baseName, TOKEN_SEPARATOR);
+ if (tokens.length != TOKEN_NUMBER)
+ {
+ continue;
+ }
+ final Location plateLocation = tryGetPlateLocation(tokens[1]);
+ final Location wellLocation = tryGetWellLocation(tokens[2]);
+ final int wavelength = getWavelength(tokens[3]);
+ if (wellLocation != null && plateLocation != null && wavelength > 0)
+ {
+ helper.addWavelength(wavelength);
+ accepter.accept(wavelength, plateLocation, wellLocation, imageFile);
+ if (operationLog.isDebugEnabled())
+ {
+ operationLog.debug(String.format(IMAGE_FILE_ACCEPTED, imageFile, wavelength,
+ plateLocation, wellLocation));
+ }
+ } else
+ {
+ if (operationLog.isDebugEnabled())
+ {
+ operationLog.debug(String.format(IMAGE_FILE_NOT_STANDARDIZABLE, imageFile,
+ tokens[0], tokens[1], tokens[2]));
+ }
+ invalidFiles.add(imageFile);
+ }
+ }
+ return new HCSImageFileExtractionResult(System.currentTimeMillis() - start, imageFiles
+ .size(), Collections.unmodifiableList(invalidFiles), helper.getChannelSet());
+ }
+
+}
diff --git a/datastore_server/sourceTest/bash/createFakeDataSet.sh b/datastore_server/sourceTest/bash/createFakeDataSet.sh
new file mode 100755
index 00000000000..0fa1d6fb432
--- /dev/null
+++ b/datastore_server/sourceTest/bash/createFakeDataSet.sh
@@ -0,0 +1,179 @@
+#!/bin/sh
+#
+# @date: 2008-04-18
+# @author: franz-josef.elmer@systemsx.ch
+# @author: basil.neff@systemsx.ch
+#
+# This bash script generates dummy data to test the installation.
+# The output is currently in the Subfolder of the Script: <SAMPLE COLDE>/TIFF
+# The Files are empty and have the following filename:
+# <SAMPLE CODE>_<PLATE ROW (as character)><PLATE COLUMN>_<TILE NUMBER>_w<WAVELENGTH>.tif
+#
+
+###################
+## USED BINARIES ##
+###################
+TOUCH=/usr/bin/touch
+BC=/usr/bin/bc
+
+
+############################
+## DO NOT CROSS THIS LINE ##
+############################
+
+if [ $# -ne 4 ]; then
+ echo "Usage: `basename $0` <sample code> <number of channels> <plate geometry> <well geometry>"
+ exit 1
+fi
+
+SAMPLE=$1
+CHANNELS=$2
+PLATE_GEOMETRY=$3
+WELL_GEOMETRY=$4
+
+
+
+if [ ${CHANNELS} -le 0 ]; then
+ echo "<number of channnels> has to be greater than 0"
+ exit 1
+fi
+
+case ${PLATE_GEOMETRY} in
+ 8x12)
+ PLATE_ROWS=8
+ PLATE_COLUMNS=12
+ ;;
+ 16x24)
+ PLATE_ROWS=16
+ PLATE_COLUMNS=24
+ ;;
+ 32x48)
+ PLATE_ROWS=32
+ PLATE_COLUMNS=48
+ ;;
+ *)
+ echo "Plate geometry has to be '8x12', '16x24', or '32x48'"
+ exit 1
+esac
+
+case ${WELL_GEOMETRY} in
+ 1x1)
+ WELL_ROWS=1
+ WELL_COLUMNS=1
+ TILES=1
+ ;;
+ 2x2)
+ WELL_ROWS=2
+ WELL_COLUMNS=2
+ TILES=4
+ ;;
+ 3x3)
+ WELL_ROWS=3
+ WELL_COLUMNS=3
+ TILES=9
+ ;;
+ *)
+ echo "Well geometry has to be '1x1', '2x2', or '3x3'"
+ exit 1
+esac
+
+###############
+## FUNCTIONS ##
+###############
+
+function getCharacterFromInt {
+ POSITION=$1
+ if [ ${POSITION} -lt 26 ];then
+ characters=(A B C D E F G H I J K L M N O P Q R S T U V W X Y Z)
+ echo ${characters[${POSITION}]}
+ else
+ FIRST_INTEGER=`echo $[(${POSITION}-25)/25]|${BC}`
+ FIRST_CHARACTER=`getCharacterFromInt ${FIRST_INTEGER}`
+ SECOND_CHARACTER=`getCharacterFromInt $[${POSITION}%26]`
+ echo "${FIRST_CHARACTER}${SECOND_CHARACTER}"
+ fi
+}
+
+function getWavelength {
+ WAVELENGTH=$1
+ echo "${WAVELENGTH}42"
+ }
+
+PROGRESS_INFO=1
+function printProgress {
+ PERCENT=`echo $1|${BC}`
+ WHEEL=`getProgressWheel ${PROGRESS_INFO}`
+ echo -ne " ${PERCENT}% [${WHEEL}]\r"
+
+ PROGRESS_INFO=$[$PROGRESS_INFO+1]
+}
+
+function getProgressWheel {
+ PARAMETER=$1
+ modulo_tree=`echo $[${PARAMETER}%3]`
+ progress_array=(\| / - \\)
+ echo ${progress_array[${modulo_tree}]}
+}
+
+############
+## SCRIPT ##
+############
+
+echo "${CHANNELS} channels, plate: ${PLATE_ROWS} x ${PLATE_COLUMNS}, well: ${WELL_ROWS} x ${WELL_COLUMNS}"
+
+if [ ! -d ${SAMPLE} ];then
+ mkdir ${SAMPLE}
+fi
+if [ ! -d ${SAMPLE}/TIFF ];then
+ mkdir ${SAMPLE}/TIFF
+fi
+
+###
+# LOOPS
+###
+
+OVERALL_COUNTER=0
+TOTAL_FILES_TO_GENERATE=$[${CHANNELS}*${PLATE_ROWS}*${PLATE_COLUMNS}*${TILES}]
+# Plate Rows
+plateRowCounter=0
+while [ ${plateRowCounter} -lt ${PLATE_ROWS} ]
+do
+
+ # Plate Columns
+ plateColumsCounter=1
+ while [ ${plateColumsCounter} -le ${PLATE_COLUMNS} ]
+ do
+
+ PLATE_ROW_CHARACTER=`getCharacterFromInt ${plateRowCounter}`
+
+ # Plate Column
+ if [ ${plateColumsCounter} -lt 10 ];then
+ PLATE_COLUMN_INT=0${plateColumsCounter}
+ else
+ PLATE_COLUMN_INT=${plateColumsCounter}
+ fi
+
+ # TILES
+ tileCounter=1
+ while [ ${tileCounter} -le ${TILES} ]
+ do
+ # Channels
+ channelCounter=1
+ while [ ${channelCounter} -le ${CHANNELS} ]
+ do
+ FILECHANEL=`getWavelength ${channelCounter}`
+ ${TOUCH} ${SAMPLE}/TIFF/${SAMPLE}_${PLATE_ROW_CHARACTER}${PLATE_COLUMN_INT}_${tileCounter}_w${FILECHANEL}.tif
+
+ OVERALL_COUNTER=$[${OVERALL_COUNTER}+1]
+ channelCounter=$[${channelCounter}+1]
+ done
+ tileCounter=$[$tileCounter+1]
+ done
+ printProgress $((${OVERALL_COUNTER}*100/${TOTAL_FILES_TO_GENERATE}))
+ plateColumsCounter=$[$plateColumsCounter+1]
+ done
+ plateRowCounter=$[$plateRowCounter+1]
+done
+
+echo ""
+echo "${OVERALL_COUNTER} files generated"
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/BDSStorageProcessorTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/BDSStorageProcessorTest.java
new file mode 100644
index 00000000000..93fed037522
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/BDSStorageProcessorTest.java
@@ -0,0 +1,539 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertFalse;
+import static org.testng.AssertJUnit.assertNotNull;
+import static org.testng.AssertJUnit.assertNull;
+import static org.testng.AssertJUnit.fail;
+
+import java.io.File;
+import java.io.IOException;
+import java.sql.Date;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.log4j.Level;
+import org.jmock.Expectations;
+import org.jmock.Mockery;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.bds.DataSet;
+import ch.systemsx.cisd.bds.DataStructureLoader;
+import ch.systemsx.cisd.bds.ExperimentRegistrator;
+import ch.systemsx.cisd.bds.Format;
+import ch.systemsx.cisd.bds.IDataStructure;
+import ch.systemsx.cisd.bds.Sample;
+import ch.systemsx.cisd.bds.UnknownFormatV1_0;
+import ch.systemsx.cisd.bds.Utilities;
+import ch.systemsx.cisd.bds.Version;
+import ch.systemsx.cisd.bds.Utilities.Boolean;
+import ch.systemsx.cisd.bds.hcs.Channel;
+import ch.systemsx.cisd.bds.hcs.HCSImageFormatV1_0;
+import ch.systemsx.cisd.bds.hcs.Location;
+import ch.systemsx.cisd.bds.hcs.WellGeometry;
+import ch.systemsx.cisd.bds.storage.IDirectory;
+import ch.systemsx.cisd.bds.storage.IFile;
+import ch.systemsx.cisd.bds.v1_1.IDataStructureV1_1;
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.filesystem.AbstractFileSystemTestCase;
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
+import ch.systemsx.cisd.common.filesystem.QueueingPathRemoverService;
+import ch.systemsx.cisd.common.logging.BufferedAppender;
+import ch.systemsx.cisd.common.mail.IMailClient;
+import ch.systemsx.cisd.common.types.BooleanOrUnknown;
+import ch.systemsx.cisd.openbis.generic.shared.dto.DataTypePE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ExperimentPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.GroupPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.PersonPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ProjectPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.PropertyTypePE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.SamplePropertyPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.SampleTypePropertyTypePE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.ExperimentIdentifier;
+import ch.systemsx.cisd.openbis.generic.shared.dto.properties.EntityDataType;
+import ch.systemsx.cisd.openbis.generic.shared.dto.types.SampleTypeCode;
+
+/**
+ * Test cases for corresponding {@link BDSStorageProcessor} class.
+ *
+ * @author Christian Ribeaud
+ */
+public final class BDSStorageProcessorTest extends AbstractFileSystemTestCase
+{
+ private static final String EXAMPLE_EMAIL = "j@d";
+
+ private static final String DATA_SET_CODE = "D";
+
+ private static final String INCOMING_DATA_SET_DIR = "NEMO.EXP1==CP001A-3AB";
+
+ private static final String EXAMPLE_TYPE_DESCRIPTION = "Screening Plate";
+
+ private static final Date REGISTRATION_DATE = new Date(47110000);
+
+ private static final String EXAMPLE_INSTANCE = "I";
+
+ private static final String EXAMPLE_INSTANCE_GLOBAL = "222-333";
+
+ private static final String EXAMPLE_GROUP = "G";
+
+ private static final String DATA_STRUCTURE_NAME = "originalData";
+
+ private static final String EXAMPLE_DATA = "hello world!";
+
+ private static final String ORIGINAL_DATA_TXT = DATA_STRUCTURE_NAME + ".txt";
+
+ private static final String VERSION_PROPERTY_KEY = BDSStorageProcessor.VERSION_KEY;
+
+ private static final String FORMAT_KEY = BDSStorageProcessor.FORMAT_KEY;
+
+ private static final String SAMPLE_TYPE_DESCRIPTION_KEY =
+ BDSStorageProcessor.SAMPLE_TYPE_DESCRIPTION_KEY;
+
+ private static final String SAMPLE_TYPE_CODE_KEY =
+
+ BDSStorageProcessor.SAMPLE_TYPE_CODE_KEY;
+
+ private static final String CHANNEL_COUNT_KEY = HCSImageFormatV1_0.NUMBER_OF_CHANNELS;
+
+ private static final String CONTAINS_ORIGINAL_DATA_KEY =
+ HCSImageFormatV1_0.CONTAINS_ORIGINAL_DATA;
+
+ private static final String WELL_GEOMETRY_KEY = WellGeometry.WELL_GEOMETRY;
+
+ private static final String FILE_EXTRACTOR_KEY = IHCSImageFileExtractor.FILE_EXTRACTOR;
+
+ private final static IProcedureAndDataTypeExtractor TYPE_EXTRACTOR =
+ new DefaultStorageProcessorTest.TestProcedureAndDataTypeExtractor();
+
+ private final static String STORE_ROOT_DIR = "store";
+
+ private BufferedAppender logRecorder;
+
+ private Mockery context;
+
+ private IMailClient mailClient;
+
+ private final static Properties createProperties(final Format format)
+ {
+ final Properties props = createPropertiesWithVersion();
+ props.setProperty(Main.STOREROOT_DIR_KEY, "store");
+ props.setProperty(SAMPLE_TYPE_DESCRIPTION_KEY, EXAMPLE_TYPE_DESCRIPTION);
+ props.setProperty(SAMPLE_TYPE_CODE_KEY, SampleTypeCode.CELL_PLATE.getCode());
+ props.setProperty(FORMAT_KEY, format.getCode() + " " + format.getVersion());
+ props.setProperty(CHANNEL_COUNT_KEY, "1");
+ props.setProperty(CONTAINS_ORIGINAL_DATA_KEY, Utilities.Boolean.TRUE.toString());
+ props.setProperty(WELL_GEOMETRY_KEY, "3x3");
+ props.setProperty(FILE_EXTRACTOR_KEY, TestImageFileExtractor.class.getName());
+ return props;
+ }
+
+ final static DataSetInformation createDataSetInformation()
+ {
+ final DataSetInformation dataSetInformation = new DataSetInformation();
+ dataSetInformation.setInstanceCode(EXAMPLE_INSTANCE);
+ dataSetInformation.setInstanceUUID(EXAMPLE_INSTANCE_GLOBAL);
+ final ExperimentIdentifier experimentIdentifier = new ExperimentIdentifier();
+ experimentIdentifier.setExperimentCode("E");
+ experimentIdentifier.setProjectCode("P");
+ experimentIdentifier.setGroupCode(EXAMPLE_GROUP);
+ dataSetInformation.setExperimentIdentifier(experimentIdentifier);
+ dataSetInformation.setSampleCode("S");
+ dataSetInformation.setDataSetCode(DATA_SET_CODE);
+ final SamplePropertyPE plateGeometry =
+ createSamplePropertyPE(PlateDimensionParser.PLATE_GEOMETRY_PROPERTY_NAME,
+ EntityDataType.VARCHAR, "_16X24");
+ dataSetInformation.setProperties(new SamplePropertyPE[]
+ { plateGeometry });
+ return dataSetInformation;
+ }
+
+ private final static SamplePropertyPE createSamplePropertyPE(final String code,
+ final EntityDataType dataType, final String value)
+ {
+ final SamplePropertyPE propertyPE = new SamplePropertyPE();
+ final SampleTypePropertyTypePE entityTypePropertyTypePE = new SampleTypePropertyTypePE();
+ final PropertyTypePE propertyTypePE = new PropertyTypePE();
+ propertyTypePE.setCode(code);
+ propertyTypePE.setLabel(code);
+ final DataTypePE type = new DataTypePE();
+ type.setCode(dataType);
+ propertyTypePE.setType(type);
+ entityTypePropertyTypePE.setPropertyType(propertyTypePE);
+ propertyPE.setEntityTypePropertyType(entityTypePropertyTypePE);
+ propertyPE.setValue(value);
+ return propertyPE;
+ }
+
+ private final File createOriginalDataInDir() throws IOException
+ {
+ final File incoming = new File(workingDirectory, "incoming");
+ incoming.mkdir();
+ final File dir = new File(incoming, INCOMING_DATA_SET_DIR);
+ dir.mkdir();
+ final File originalData = new File(dir, ORIGINAL_DATA_TXT);
+ FileUtilities.writeToFile(originalData, EXAMPLE_DATA);
+ return dir;
+ }
+
+ static final ExperimentPE createExperiment()
+ {
+ final ExperimentPE baseExperiment = new ExperimentPE();
+ baseExperiment.setRegistrationDate(REGISTRATION_DATE);
+ final PersonPE person = new PersonPE();
+ person.setFirstName("Joe");
+ person.setLastName("Doe");
+ person.setEmail(EXAMPLE_EMAIL);
+ final GroupPE group = new GroupPE();
+ group.setCode(EXAMPLE_GROUP);
+ final ProjectPE project = new ProjectPE();
+ project.setGroup(group);
+ baseExperiment.setProject(project);
+ baseExperiment.setRegistrator(person);
+ return baseExperiment;
+ }
+
+ private final static Properties createPropertiesWithVersion()
+ {
+ final Properties props = new Properties();
+ props.setProperty(VERSION_PROPERTY_KEY, "1.1");
+ return props;
+ }
+
+ @BeforeClass
+ public void startQueueingPathRemover()
+ {
+ if (QueueingPathRemoverService.isRunning() == false)
+ {
+ QueueingPathRemoverService.start();
+ }
+ }
+
+ @Override
+ @BeforeMethod
+ public void setUp() throws IOException
+ {
+ super.setUp();
+ logRecorder = new BufferedAppender("%-5p %c - %m%n", Level.INFO);
+ context = new Mockery();
+ mailClient = context.mock(IMailClient.class);
+ }
+
+ @AfterMethod
+ public void tearDown()
+ {
+ logRecorder.reset();
+ // The following line of code should also be called at the end of each test method.
+ // Otherwise one do not known which test failed.
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public final void testConstructorWithUnspecifiedArgument()
+ {
+ boolean fail = true;
+ try
+ {
+ new BDSStorageProcessor(null);
+ } catch (final AssertionError ex)
+ {
+ fail = false;
+ }
+ assertFalse(fail);
+ }
+
+ @Test
+ public final void testCheckVersionInProperties()
+ {
+ final Properties props = new Properties();
+ props.setProperty(Main.STOREROOT_DIR_KEY, "store");
+ final String version = "ae";
+ props.setProperty(VERSION_PROPERTY_KEY, version);
+ try
+ {
+ new BDSStorageProcessor(props);
+ fail("No '.' in given version.");
+ } catch (final ConfigurationFailureException ex)
+ {
+ assertEquals(String.format(BDSStorageProcessor.NO_VERSION_FORMAT,
+ BDSStorageProcessor.VERSION_KEY, version), ex.getMessage());
+ }
+ }
+
+ @Test
+ public final void testCheckVersionCompatible()
+ {
+ final Properties props = new Properties();
+ props.setProperty(Main.STOREROOT_DIR_KEY, "store");
+ props.setProperty(VERSION_PROPERTY_KEY, "1.2");
+ try
+ {
+ new BDSStorageProcessor(props);
+ fail("ConfigurationFailureException expected");
+ } catch (final ConfigurationFailureException e)
+ {
+ assertEquals("Invalid version: V1.2", e.getMessage());
+ }
+ }
+
+ @Test
+ public final void testMissingFormat() throws Exception
+ {
+ final Properties props = createPropertiesWithVersion();
+ try
+ {
+ new BDSStorageProcessor(props);
+ fail("ConfigurationFailureException expected");
+ } catch (final ConfigurationFailureException e)
+ {
+ assertEquals("Given key 'format' not found in properties '[version]'", e.getMessage());
+ }
+ }
+
+ @Test
+ public final void testMissingSampleTypeDescription() throws Exception
+ {
+ final Properties props = createPropertiesWithVersion();
+ final Format format = UnknownFormatV1_0.UNKNOWN_1_0;
+ props.setProperty(FORMAT_KEY, format.getCode() + " " + format.getVersion());
+ try
+ {
+ new BDSStorageProcessor(props);
+ fail("ConfigurationFailureException expected");
+ } catch (final ConfigurationFailureException e)
+ {
+ assertEquals("Given key 'sampleTypeDescription' not found in properties "
+ + "'[version, format]'", e.getMessage());
+ }
+ }
+
+ @DataProvider(name = "formatProvider")
+ public Object[][] getFormats()
+ {
+ return new Object[][]
+ {
+ { UnknownFormatV1_0.UNKNOWN_1_0 },
+ { HCSImageFormatV1_0.HCS_IMAGE_1_0 } };
+ }
+
+ @Test(dataProvider = "formatProvider")
+ public final void testStoreData(final Format format) throws Exception
+ {
+ final Properties properties = createProperties(format);
+ final BDSStorageProcessor storageProcessor = new BDSStorageProcessor(properties);
+ assertEquals(0, workingDirectory.list().length);
+ final File incomingDataSetDirectory = createOriginalDataInDir();
+ assertEquals(true, incomingDataSetDirectory.exists());
+ final ExperimentPE baseExperiment = createExperiment();
+ final DataSetInformation dataSetInformation = createDataSetInformation();
+ prepareMailClient(format);
+ final File dataFile =
+ storageProcessor.storeData(baseExperiment, dataSetInformation, TYPE_EXTRACTOR,
+ mailClient, incomingDataSetDirectory, new File(workingDirectory,
+ STORE_ROOT_DIR));
+ assertEquals(new File(workingDirectory, STORE_ROOT_DIR).getAbsolutePath(), dataFile
+ .getAbsolutePath());
+ final IDataStructure dataStructure =
+ new DataStructureLoader(workingDirectory).load(STORE_ROOT_DIR);
+ assertEquals(true, dataStructure instanceof IDataStructureV1_1);
+ final IDataStructureV1_1 ds = (IDataStructureV1_1) dataStructure;
+ assertEquals(new Version(1, 1), ds.getVersion());
+ final ch.systemsx.cisd.bds.ExperimentIdentifier eid = ds.getExperimentIdentifier();
+ assertEquals(EXAMPLE_INSTANCE, eid.getInstanceCode());
+ assertEquals(EXAMPLE_GROUP, eid.getGroupCode());
+ assertEquals(dataSetInformation.getExperimentIdentifier().getProjectCode(), eid
+ .getProjectCode());
+ assertEquals(dataSetInformation.getExperimentIdentifier().getExperimentCode(), eid
+ .getExperimentCode());
+ final ExperimentRegistrator registrator = ds.getExperimentRegistrator();
+ assertEquals(baseExperiment.getRegistrator().getFirstName(), registrator.getFirstName());
+ assertEquals(baseExperiment.getRegistrator().getLastName(), registrator.getLastName());
+ assertEquals(baseExperiment.getRegistrator().getEmail(), registrator.getEmail());
+ assertEquals(REGISTRATION_DATE, ds.getExperimentRegistratorTimestamp().getDate());
+ final Sample sample = ds.getSample();
+ assertEquals(EXAMPLE_TYPE_DESCRIPTION, sample.getTypeDescription());
+ assertEquals(dataSetInformation.getSampleIdentifier().getSampleCode(), sample.getCode());
+ final Format f = ds.getFormattedData().getFormat();
+ assertEquals(format, f);
+ final IDirectory directory =
+ (IDirectory) ds.getOriginalData().tryGetNode(INCOMING_DATA_SET_DIR);
+ assertEquals(EXAMPLE_DATA, Utilities.getTrimmedString(directory, ORIGINAL_DATA_TXT));
+ assertEquals(false, incomingDataSetDirectory.exists());
+ // DataSet
+ final DataSet dataSet = ds.getDataSet();
+ assertEquals(DATA_SET_CODE, dataSet.getCode());
+ assertEquals(TYPE_EXTRACTOR.getDataSetType(null).getCode(), dataSet.getDataSetTypeCode());
+ assertEquals(0, dataSet.getParentCodes().size());
+ assertNull(dataSet.getProducerCode());
+ assertNull(dataSet.getProductionTimestamp());
+ assertEquals(Boolean.TRUE, dataSet.isMeasured());
+
+ context.assertIsSatisfied();
+ }
+
+ @Test(dataProvider = "formatProvider")
+ public final void testUnstoreData(final Format format) throws Exception
+ {
+ final Properties properties = createProperties(format);
+ final BDSStorageProcessor storageAdapter = new BDSStorageProcessor(properties);
+ assertEquals(0, workingDirectory.list().length);
+ final File incomingDirectoryData = createOriginalDataInDir();
+ // incoming/NEMO.EXP1==CP001A-3AB in 'workingDirectory'
+ assert incomingDirectoryData.exists();
+ final ExperimentPE baseExperiment = createExperiment();
+ final DataSetInformation dataSetInformation = createDataSetInformation();
+ // NEMO.EXP1==CP001A-3AB in 'workingDirectory'
+ prepareMailClient(format);
+ final File storeRootDir = new File(workingDirectory, STORE_ROOT_DIR);
+ final File dataStore =
+ storageAdapter.storeData(baseExperiment, dataSetInformation, TYPE_EXTRACTOR,
+ mailClient, incomingDirectoryData, storeRootDir);
+ assertEquals(true, dataStore.isDirectory());
+ assertEquals(false, incomingDirectoryData.exists());
+ storageAdapter.unstoreData(incomingDirectoryData, storeRootDir);
+ assertEquals(false, dataStore.exists());
+ assertEquals(true, incomingDirectoryData.isDirectory());
+
+ context.assertIsSatisfied();
+ }
+
+ @Test(dataProvider = "formatProvider")
+ public void testTryToGetOriginalData(final Format format) throws Exception
+ {
+ final Properties properties = createProperties(format);
+ final BDSStorageProcessor storageProcessor = new BDSStorageProcessor(properties);
+ final File incomingDirectoryData = createOriginalDataInDir();
+ final ExperimentPE baseExperiment = createExperiment();
+ final DataSetInformation dataSetInformation = createDataSetInformation();
+ prepareMailClient(format);
+ final File storeData =
+ storageProcessor.storeData(baseExperiment, dataSetInformation, TYPE_EXTRACTOR,
+ mailClient, incomingDirectoryData, workingDirectory);
+ final File originalDataSet = storageProcessor.tryGetProprietaryData(storeData);
+ assertNotNull(originalDataSet);
+ assertEquals(INCOMING_DATA_SET_DIR, originalDataSet.getName());
+ assertEquals(true, originalDataSet.isDirectory());
+ assertEquals(format == UnknownFormatV1_0.UNKNOWN_1_0 ? BooleanOrUnknown.U
+ : BooleanOrUnknown.F, dataSetInformation.getIsCompleteFlag());
+ final File[] files = originalDataSet.listFiles();
+ assertEquals(1, files.length);
+ final File file = files[0];
+ assertEquals(true, file.exists());
+ assertEquals(false, file.isDirectory());
+ assertEquals(ORIGINAL_DATA_TXT, file.getName());
+
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testTryToGetOriginalDataWhichAreNotAvailable() throws Exception
+ {
+ final Properties properties = createProperties(HCSImageFormatV1_0.HCS_IMAGE_1_0);
+ properties.setProperty(CONTAINS_ORIGINAL_DATA_KEY, Utilities.Boolean.FALSE.toString());
+ final BDSStorageProcessor storageProcessor = new BDSStorageProcessor(properties);
+ final File incomingDirectoryData = createOriginalDataInDir();
+ final ExperimentPE baseExperiment = createExperiment();
+ final DataSetInformation dataSetInformation = createDataSetInformation();
+ prepareMailClient(HCSImageFormatV1_0.HCS_IMAGE_1_0);
+ final File storeData =
+ storageProcessor.storeData(baseExperiment, dataSetInformation, TYPE_EXTRACTOR,
+ mailClient, incomingDirectoryData, workingDirectory);
+ logRecorder.resetLogContent();
+ final File originalDataSet = storageProcessor.tryGetProprietaryData(storeData);
+ assertEquals(null, originalDataSet);
+ assertEquals("WARN OPERATION.BDSStorageProcessor - " + "Original data are not available.",
+ logRecorder.getLogContent());
+
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testConstructorWithInvalidFormat() throws Exception
+ {
+ final Properties properties = createProperties(new Format("bla", new Version(1, 2), "v"));
+ try
+ {
+ new BDSStorageProcessor(properties);
+ fail("ConfigurationFailureException expected");
+ } catch (final ConfigurationFailureException e)
+ {
+ assertEquals("Property 'format': no valid and known format could be extracted "
+ + "from text 'bla V1.2'.", e.getMessage());
+ }
+ }
+
+ private void prepareMailClient(final Format format)
+ {
+ if (format != UnknownFormatV1_0.UNKNOWN_1_0)
+ {
+ context.checking(new Expectations()
+ {
+ {
+ one(mailClient)
+ .sendMessage(
+ "Incomplete data set 'NEMO.EXP1==CP001A-3AB'",
+ "Incomplete data set 'NEMO.EXP1==CP001A-3AB': "
+ + "3455 image file(s) are missing (locations: "
+ + "[[well=[x=16,y=1],tile=[x=3,y=3]], [well=[x=16,y=2],tile=[x=2,y=3]], "
+ + "[well=[x=16,y=3],tile=[x=1,y=3]], [well=[x=24,y=10],tile=[x=1,y=3]], "
+ + "[well=[x=24,y=9],tile=[x=2,y=3]], [well=[x=24,y=8],tile=[x=3,y=3]], "
+ + "[well=[x=7,y=6],tile=[x=1,y=2]], [well=[x=7,y=5],tile=[x=2,y=2]], "
+ + "[well=[x=7,y=4],tile=[x=3,y=2]], [well=[x=14,y=6],tile=[x=3,y=1]], "
+ + "... (3445 left)])", null, EXAMPLE_EMAIL);
+ }
+ });
+ }
+ }
+
+ //
+ // Helper classes
+ //
+
+ public final static class TestImageFileExtractor implements IHCSImageFileExtractor
+ {
+
+ public TestImageFileExtractor(final Properties properties)
+ {
+ }
+
+ //
+ // IHCSImageFileExtractor
+ //
+
+ public final HCSImageFileExtractionResult process(
+ final IDirectory incomingDataSetDirectory,
+ final DataSetInformation dataSetInformation, final IHCSImageFileAccepter accepter)
+ {
+ assertEquals(INCOMING_DATA_SET_DIR, incomingDataSetDirectory.getName());
+ final List<IFile> listFiles = incomingDataSetDirectory.listFiles(null, true);
+ assertEquals(1, listFiles.size());
+ final IFile file = listFiles.get(0);
+ assertEquals(ORIGINAL_DATA_TXT, file.getName());
+ accepter.accept(1, new Location(1, 1), new Location(1, 1), file);
+ return new HCSImageFileExtractionResult(0, listFiles.size(), new ArrayList<IFile>(),
+ Collections.singleton(new Channel(1, 123)));
+ }
+ }
+
+}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/ChannelSetHelperTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/ChannelSetHelperTest.java
new file mode 100644
index 00000000000..ce583e2ae41
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/ChannelSetHelperTest.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.fail;
+
+import org.testng.annotations.Test;
+
+/**
+ * Test cases for corresponding {@link ChannelSetHelper} class.
+ *
+ * @author Christian Ribeaud
+ */
+public final class ChannelSetHelperTest
+{
+
+ @Test
+ public final void testAddWavelength()
+ {
+ final int wavelength = 123;
+ final ChannelSetHelper helper = new ChannelSetHelper();
+ helper.addWavelength(wavelength);
+ assertEquals(wavelength, helper.getChannelSet().iterator().next().getWavelength());
+ try
+ {
+ helper.addWavelength(456);
+ fail("ChannelSetHelper is locked.");
+ } catch (final AssertionError e)
+ {
+ // Nothing to do here.
+ }
+ }
+
+ @Test
+ public final void testGetChannelForWavelength()
+ {
+ final ChannelSetHelper helper = new ChannelSetHelper();
+ helper.addWavelength(456);
+ helper.addWavelength(893);
+ helper.addWavelength(1);
+ assertEquals(1, helper.getChannelForWavelength(1).getCounter());
+ assertEquals(2, helper.getChannelForWavelength(456).getCounter());
+ assertEquals(3, helper.getChannelForWavelength(893).getCounter());
+ try
+ {
+ helper.getChannelForWavelength(2);
+ fail("Given wavelength unknown.");
+ } catch (final AssertionError e)
+ {
+ // Nothing to do here.
+ }
+ }
+}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/CodeExtractortTestCase.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/CodeExtractortTestCase.java
new file mode 100644
index 00000000000..c733030a528
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/CodeExtractortTestCase.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+/**
+ * @author Franz-Josef Elmer
+ */
+public abstract class CodeExtractortTestCase
+{
+ protected static final String PREFIX = IDataSetInfoExtractor.EXTRACTOR_KEY + ".";
+
+ protected static final String ENTITY_SEPARATOR =
+ PREFIX + DefaultDataSetInfoExtractor.ENTITY_SEPARATOR_PROPERTY_NAME;
+
+ protected static final String INDEX_OF_SAMPLE_CODE =
+ PREFIX + DefaultDataSetInfoExtractor.INDEX_OF_SAMPLE_CODE;
+
+ protected static final String INDEX_OF_PARENT_DATA_SET_CODE =
+ PREFIX + DefaultDataSetInfoExtractor.INDEX_OF_PARENT_DATA_SET_CODE;
+
+ protected static final String INDEX_OF_DATA_PRODUCER_CODE =
+ PREFIX + DefaultDataSetInfoExtractor.INDEX_OF_DATA_PRODUCER_CODE;
+
+ protected static final String INDEX_OF_DATA_PRODUCTION_DATE =
+ PREFIX + DefaultDataSetInfoExtractor.INDEX_OF_DATA_PRODUCTION_DATE;
+
+ protected static final String DATA_PRODUCTION_DATE_FORMAT =
+ PREFIX + DefaultDataSetInfoExtractor.DATA_PRODUCTION_DATE_FORMAT;
+
+}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/DataSetNameEntitiesProviderTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/DataSetNameEntitiesProviderTest.java
new file mode 100644
index 00000000000..c747491fa75
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/DataSetNameEntitiesProviderTest.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.fail;
+
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.common.exceptions.UserFailureException;
+
+/**
+ * Test cases for corresponding {@link DataSetNameEntitiesProvider} class.
+ *
+ * @author Franz-Josef Elmer
+ */
+public class DataSetNameEntitiesProviderTest
+{
+ private static final String ALPHA = "alpha";
+
+ private static final String BETA = "beta";
+
+ private static final String GAMMA = "gamma";
+
+ private DataSetNameEntitiesProvider provider;
+
+ @BeforeMethod
+ public void setup()
+ {
+ char separator = '.';
+ provider =
+ new DataSetNameEntitiesProvider(ALPHA + separator + BETA + separator + GAMMA,
+ separator, false);
+ }
+
+ @Test
+ public void testGetEntityWithValidIndex()
+ {
+ assertEquals(ALPHA, provider.getEntity(0));
+ assertEquals(BETA, provider.getEntity(1));
+ assertEquals(GAMMA, provider.getEntity(2));
+ }
+
+ @Test
+ public void testGetEntityWithValidNegativeIndex()
+ {
+ assertEquals(ALPHA, provider.getEntity(-3));
+ assertEquals(BETA, provider.getEntity(-2));
+ assertEquals(GAMMA, provider.getEntity(-1));
+ }
+
+ @Test
+ public void testGetEntityWithInvalidPositiveIndex()
+ {
+ try
+ {
+ provider.getEntity(3);
+ fail("UserFailureException expected");
+ } catch (UserFailureException e)
+ {
+ assertEquals("Invalid data set name 'alpha.beta.gamma'. "
+ + "We need 4 entities, separated by '.', but got only 3.", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testGetEntityWithInvalidNegativeIndex()
+ {
+ try
+ {
+ provider.getEntity(-4);
+ fail("UserFailureException expected");
+ } catch (UserFailureException e)
+ {
+ assertEquals("Invalid data set name 'alpha.beta.gamma'. "
+ + "We need 4 entities, separated by '.', but got only 3.", e.getMessage());
+ }
+ }
+}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/DataStrategyStoreTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/DataStrategyStoreTest.java
new file mode 100644
index 00000000000..d2fde9547c7
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/DataStrategyStoreTest.java
@@ -0,0 +1,261 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertTrue;
+
+import java.io.File;
+
+import org.apache.log4j.Level;
+import org.jmock.Expectations;
+import org.jmock.Mockery;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.common.filesystem.AbstractFileSystemTestCase;
+import ch.systemsx.cisd.common.logging.BufferedAppender;
+import ch.systemsx.cisd.common.mail.IMailClient;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ExperimentPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.GroupPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.InvalidationPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.PersonPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ProjectPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.SamplePropertyPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifier;
+
+/**
+ * Test cases for corresponding {@link DataStrategyStore} class.
+ *
+ * @author Christian Ribeaud
+ */
+public final class DataStrategyStoreTest extends AbstractFileSystemTestCase
+{
+ private static final String EXPERIMENT_CODE = "E";
+
+ private static final String PROJECT_CODE = "P";
+
+ private static final String GROUP_CODE = "G";
+
+ private Mockery context;
+
+ private IEncapsulatedLimsService limsService;
+
+ private IMailClient mailClient;
+
+ private DataStrategyStore dataStrategyStore;
+
+ private BufferedAppender logRecorder;
+
+ private final static ExperimentPE createBaseExperiment()
+ {
+ final ExperimentPE baseExperiment = new ExperimentPE();
+ baseExperiment.setCode(EXPERIMENT_CODE);
+ final ProjectPE project = new ProjectPE();
+ project.setCode(PROJECT_CODE);
+ final GroupPE group = new GroupPE();
+ group.setCode(GROUP_CODE);
+ project.setGroup(group);
+ baseExperiment.setProject(project);
+ return baseExperiment;
+ }
+
+ private final File createIncomingDataSetPath()
+ {
+ return new File(workingDirectory, "Twain");
+ }
+
+ @BeforeMethod
+ public void startup()
+ {
+ logRecorder = new BufferedAppender("%-5p %c - %m%n", Level.DEBUG);
+ }
+
+ @AfterMethod
+ public final void tearDown()
+ {
+ logRecorder.reset();
+ // The following line of code should also be called at the end of each test method.
+ // Otherwise one do not known which test failed.
+ context.assertIsSatisfied();
+ }
+
+ @BeforeClass
+ public final void beforeClass()
+ {
+ context = new Mockery();
+ limsService = context.mock(IEncapsulatedLimsService.class);
+ mailClient = context.mock(IMailClient.class);
+ dataStrategyStore = new DataStrategyStore(limsService, mailClient);
+ }
+
+ @Test
+ public final void testEasiestCases()
+ {
+ boolean exceptionThrown = false;
+ try
+ {
+ dataStrategyStore.getDataStoreStrategy(null, null);
+ } catch (final AssertionError ex)
+ {
+ exceptionThrown = true;
+ }
+ assertTrue("Null incoming data set path not permited", exceptionThrown);
+ final File incomingDataSetPath = createIncomingDataSetPath();
+ final IDataStoreStrategy dataStoreStrategy =
+ dataStrategyStore.getDataStoreStrategy(null, incomingDataSetPath);
+ assertEquals(dataStoreStrategy.getKey(), DataStoreStrategyKey.UNIDENTIFIED);
+ }
+
+ @Test
+ public final void testWithFullDataSetInfo()
+ {
+ final File incomingDataSetPath = createIncomingDataSetPath();
+ final DataSetInformation dataSetInfo = IdentifiedDataStrategyTest.createDataSetInfo();
+ final SampleIdentifier sampleIdentifier = dataSetInfo.getSampleIdentifier();
+ final ExperimentPE baseExperiment = createBaseExperiment();
+ context.checking(new Expectations()
+ {
+ {
+ one(limsService).getBaseExperiment(sampleIdentifier);
+ will(returnValue(baseExperiment));
+
+ one(limsService).getPropertiesOfTopSampleRegisteredFor(sampleIdentifier);
+ will(returnValue(new SamplePropertyPE[0]));
+ }
+ });
+ final IDataStoreStrategy dataStoreStrategy =
+ dataStrategyStore.getDataStoreStrategy(dataSetInfo, incomingDataSetPath);
+ assertEquals(dataStoreStrategy.getKey(), DataStoreStrategyKey.IDENTIFIED);
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public final void testWithoutExperimentIdentifier()
+ {
+ final File incomingDataSetPath = createIncomingDataSetPath();
+ final DataSetInformation dataSetInfo = IdentifiedDataStrategyTest.createDataSetInfo();
+ dataSetInfo.setExperimentIdentifier(null);
+ final ExperimentPE baseExperiment = createBaseExperiment();
+ final SampleIdentifier sampleIdentifier = dataSetInfo.getSampleIdentifier();
+ context.checking(new Expectations()
+ {
+ {
+ one(limsService).getBaseExperiment(sampleIdentifier);
+ will(returnValue(baseExperiment));
+
+ one(limsService).getPropertiesOfTopSampleRegisteredFor(sampleIdentifier);
+ will(returnValue(new SamplePropertyPE[0]));
+ }
+ });
+ final IDataStoreStrategy dataStoreStrategy =
+ dataStrategyStore.getDataStoreStrategy(dataSetInfo, incomingDataSetPath);
+ assertEquals(dataStoreStrategy.getKey(), DataStoreStrategyKey.IDENTIFIED);
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public final void testWithNullBaseExperiment()
+ {
+ final File incomingDataSetPath = createIncomingDataSetPath();
+ final DataSetInformation dataSetInfo = IdentifiedDataStrategyTest.createDataSetInfo();
+ context.checking(new Expectations()
+ {
+ {
+ one(limsService).getBaseExperiment(dataSetInfo.getSampleIdentifier());
+ will(returnValue(null));
+ }
+ });
+
+ final IDataStoreStrategy dataStoreStrategy =
+ dataStrategyStore.getDataStoreStrategy(dataSetInfo, incomingDataSetPath);
+
+ assertEquals(dataStoreStrategy.getKey(), DataStoreStrategyKey.UNIDENTIFIED);
+ final String logContent = logRecorder.getLogContent();
+ assertEquals("Unexpected log content: " + logContent, true, logContent
+ .startsWith("ERROR NOTIFY.DataStrategyStore"));
+
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public final void testWithInvalidBaseExperiment()
+ {
+ final File incomingDataSetPath = createIncomingDataSetPath();
+ final DataSetInformation dataSetInfo = IdentifiedDataStrategyTest.createDataSetInfo();
+ final ExperimentPE baseExperiment = createBaseExperiment();
+ baseExperiment.setInvalidation(new InvalidationPE());
+ context.checking(new Expectations()
+ {
+ {
+ one(limsService).getBaseExperiment(dataSetInfo.getSampleIdentifier());
+ will(returnValue(baseExperiment));
+ }
+ });
+ final IDataStoreStrategy dataStoreStrategy =
+ dataStrategyStore.getDataStoreStrategy(dataSetInfo, incomingDataSetPath);
+ assertEquals(dataStoreStrategy.getKey(), DataStoreStrategyKey.UNIDENTIFIED);
+ final String logContent = logRecorder.getLogContent();
+ assertEquals("ERROR NOTIFY.DataStrategyStore - "
+ + "Data set for sample 'MY-INSTANCE:/S' can not be registered "
+ + "because experiment 'E' has been invalidated.", logContent);
+
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public final void testSampleIsNotRegistered()
+ {
+ final File incomingDataSetPath = createIncomingDataSetPath();
+ final DataSetInformation dataSetInfo = IdentifiedDataStrategyTest.createDataSetInfo();
+ final ExperimentPE baseExperiment = createBaseExperiment();
+ final PersonPE person = new PersonPE();
+ final String email = "john.doe@freemail.org";
+ person.setEmail(email);
+ baseExperiment.setRegistrator(person);
+ context.checking(new Expectations()
+ {
+ {
+ one(limsService).getBaseExperiment(dataSetInfo.getSampleIdentifier());
+ will(returnValue(baseExperiment));
+
+ one(limsService).getPropertiesOfTopSampleRegisteredFor(
+ dataSetInfo.getSampleIdentifier());
+ will(returnValue(null));
+
+ String replyTo = null;
+ one(mailClient).sendMessage(
+ with(equal(String.format(DataStrategyStore.SUBJECT_FORMAT, dataSetInfo
+ .getExperimentIdentifier()))), with(any(String.class)),
+ with(equal(replyTo)), with(equal(new String[]
+ { email })));
+ }
+ });
+
+ final IDataStoreStrategy dataStoreStrategy =
+ dataStrategyStore.getDataStoreStrategy(dataSetInfo, incomingDataSetPath);
+
+ assertEquals(dataStoreStrategy.getKey(), DataStoreStrategyKey.INVALID);
+ final String logContent = logRecorder.getLogContent();
+ assertEquals("Unexpected log content: " + logContent, true, logContent
+ .startsWith("ERROR OPERATION.DataStrategyStore"));
+
+ context.assertIsSatisfied();
+ }
+}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/DefaultDataSetInfoExtractorTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/DefaultDataSetInfoExtractorTest.java
new file mode 100644
index 00000000000..985e19f131a
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/DefaultDataSetInfoExtractorTest.java
@@ -0,0 +1,155 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertNull;
+import static org.testng.AssertJUnit.fail;
+
+import java.io.File;
+import java.text.SimpleDateFormat;
+import java.util.Properties;
+
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.common.exceptions.UserFailureException;
+
+/**
+ * Test cases for the {@link DefaultDataSetInfoExtractor}.
+ *
+ * @author Bernd Rinn
+ */
+public final class DefaultDataSetInfoExtractorTest extends CodeExtractortTestCase
+{
+
+ @Test
+ public void testHappyCaseWithDefaultProperties()
+ {
+ final String barcode = "XYZ123";
+ final IDataSetInfoExtractor extractor = new DefaultDataSetInfoExtractor(new Properties());
+
+ final DataSetInformation dsInfo =
+ extractor.getDataSetInformation(new File("bla.bla." + barcode));
+
+ assertNull(dsInfo.getExperimentIdentifier());
+ assertEquals(barcode, dsInfo.getSampleIdentifier().getSampleCode());
+ assertEquals(null, dsInfo.getParentDataSetCode());
+ assertEquals(null, dsInfo.getProducerCode());
+ assertEquals(null, dsInfo.getProductionDate());
+ }
+
+ @Test
+ public void testHappyCaseWithProducerCodeAndProductionDate()
+ {
+ final Properties properties = new Properties();
+ properties.setProperty(INDEX_OF_DATA_PRODUCER_CODE, "-2");
+ properties.setProperty(INDEX_OF_DATA_PRODUCTION_DATE, "0");
+ final IDataSetInfoExtractor extractor = new DefaultDataSetInfoExtractor(properties);
+ final String producerCode = "M1";
+ final String productionDate = "20070903181312";
+ final String barcode = "XYZ123";
+
+ final DataSetInformation dsInfo =
+ extractor.getDataSetInformation(new File(productionDate + ".A.B." + producerCode
+ + "." + barcode));
+
+ assertEquals(barcode, dsInfo.getSampleIdentifier().getSampleCode());
+ assertEquals(producerCode, dsInfo.getProducerCode());
+ final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss");
+ assertEquals(productionDate, dateFormat.format(dsInfo.getProductionDate()));
+ }
+
+ @Test
+ public void testHappyCaseWithAllPropertiesSet()
+ {
+ final Properties properties = new Properties();
+ final String separator = "=";
+ properties.setProperty(ENTITY_SEPARATOR, separator);
+ properties.setProperty(INDEX_OF_SAMPLE_CODE, "0");
+ properties.setProperty(INDEX_OF_PARENT_DATA_SET_CODE, "1");
+ properties.setProperty(INDEX_OF_DATA_PRODUCER_CODE, "-2");
+ properties.setProperty(INDEX_OF_DATA_PRODUCTION_DATE, "-1");
+ final String format = "yyyy-MM-dd HH:mm:ss";
+ properties.setProperty(DATA_PRODUCTION_DATE_FORMAT, format);
+ final IDataSetInfoExtractor extractor = new DefaultDataSetInfoExtractor(properties);
+ final String producerCode = "M1";
+ final String parentDataSetCode = "1234-8";
+ final String productionDate = "2007-09-03 18:03:12";
+ final String barcode = "XYZ-123";
+ final DataSetInformation dsInfo =
+ extractor.getDataSetInformation(new File(barcode + separator + parentDataSetCode
+ + separator + "A" + separator + producerCode + separator + productionDate));
+ assertEquals(barcode, dsInfo.getSampleIdentifier().getSampleCode());
+ assertEquals(parentDataSetCode, dsInfo.getParentDataSetCode());
+ assertEquals(producerCode, dsInfo.getProducerCode());
+ final SimpleDateFormat dateFormat = new SimpleDateFormat(format);
+ assertEquals(productionDate, dateFormat.format(dsInfo.getProductionDate()));
+ }
+
+ @Test
+ public void testWrongProductionDateFormat()
+ {
+ final Properties properties = new Properties();
+ properties.setProperty(INDEX_OF_DATA_PRODUCTION_DATE, "0");
+ final IDataSetInfoExtractor extractor = new DefaultDataSetInfoExtractor(properties);
+ try
+ {
+ extractor.getDataSetInformation(new File("blabla.XYZ-123"));
+ fail("UserFailureException expected");
+ } catch (final UserFailureException e)
+ {
+ assertEquals("Could not parse data production date 'blabla' "
+ + "because it violates the following format: yyyyMMddHHmmss", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testIndexTooLarge()
+ {
+ final Properties properties = new Properties();
+ properties.setProperty(INDEX_OF_SAMPLE_CODE, "1");
+ final IDataSetInfoExtractor extractor = new DefaultDataSetInfoExtractor(properties);
+ try
+ {
+ extractor.getDataSetInformation(new File("XYZ-123"));
+ fail("UserFailureException expected");
+ } catch (final UserFailureException e)
+ {
+ assertEquals("Invalid data set name 'XYZ-123'. We need 2 entities, separated by '.', "
+ + "but got only 1.", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testIndexTooSmall()
+ {
+ final Properties properties = new Properties();
+ properties.setProperty(ENTITY_SEPARATOR, "-");
+ properties.setProperty(INDEX_OF_SAMPLE_CODE, "-3");
+ final IDataSetInfoExtractor extractor = new DefaultDataSetInfoExtractor(properties);
+ try
+ {
+ extractor.getDataSetInformation(new File("XYZ-123"));
+ fail("UserFailureException expected");
+ } catch (final UserFailureException e)
+ {
+ assertEquals("Invalid data set name 'XYZ-123'. We need 3 entities, separated by '-', "
+ + "but got only 2.", e.getMessage());
+ }
+ }
+
+}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/DefaultStorageProcessorTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/DefaultStorageProcessorTest.java
new file mode 100644
index 00000000000..c0c42d7f3be
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/DefaultStorageProcessorTest.java
@@ -0,0 +1,157 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.fail;
+
+import java.io.File;
+import java.util.Properties;
+
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.common.filesystem.AbstractFileSystemTestCase;
+import ch.systemsx.cisd.openbis.generic.shared.dto.DataSetType;
+import ch.systemsx.cisd.openbis.generic.shared.dto.FileFormatType;
+import ch.systemsx.cisd.openbis.generic.shared.dto.LocatorType;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ProcedureType;
+import ch.systemsx.cisd.openbis.generic.shared.dto.types.ProcedureTypeCode;
+
+/**
+ * Test cases for corresponding {@link DefaultStorageProcessor} class.
+ *
+ * @author Christian Ribeaud
+ */
+public final class DefaultStorageProcessorTest extends AbstractFileSystemTestCase
+{
+
+ private final static IProcedureAndDataTypeExtractor TYPE_EXTRACTOR =
+ new TestProcedureAndDataTypeExtractor();
+
+ private final DefaultStorageProcessor createStorageProcessor()
+ {
+ final Properties properties = new Properties();
+ final DefaultStorageProcessor storageProcessor = new DefaultStorageProcessor(properties);
+ storageProcessor.setStoreRootDirectory(workingDirectory);
+ return storageProcessor;
+ }
+
+ private File createDirectory(final String directoryName)
+ {
+ final File file = new File(workingDirectory, directoryName);
+ file.mkdir();
+ assertEquals(true, file.isDirectory());
+ return file;
+ }
+
+ @Test
+ public final void testStoreData()
+ {
+ final DefaultStorageProcessor storageProcessor = createStorageProcessor();
+ try
+ {
+ storageProcessor.storeData(null, null, null, null, null, null);
+ fail("Null values not accepted");
+ } catch (final AssertionError e)
+ {
+ // Nothing to do here.
+ }
+ final File incomingDataSetDirectory = createDirectory("incoming");
+ final File rootDir = createDirectory("root");
+ final File storeData =
+ storageProcessor.storeData(null, null, TYPE_EXTRACTOR, null,
+ incomingDataSetDirectory, rootDir);
+ assertEquals(false, incomingDataSetDirectory.exists());
+ assertEquals(true, storeData.isDirectory());
+ assertEquals(new File(new File(workingDirectory, "root"), "incoming").getAbsolutePath(),
+ storeData.getAbsolutePath());
+ }
+
+ @Test
+ public final void testGetStoreRootDirectory()
+ {
+ DefaultStorageProcessor storageProcessor = createStorageProcessor();
+ File storeRootDirectory = storageProcessor.getStoreRootDirectory();
+ assertEquals(workingDirectory.getAbsolutePath(), storeRootDirectory.getAbsolutePath());
+ }
+
+ @Test
+ public final void testUnstoreData()
+ {
+ final DefaultStorageProcessor storageProcessor = createStorageProcessor();
+ try
+ {
+ storageProcessor.unstoreData(null, null);
+ fail("Null values not accepted");
+ } catch (final AssertionError e)
+ {
+ // Nothing to do here.
+ }
+ final File root = createDirectory("root");
+ final File incomingDataSetDirectory = createDirectory("incoming");
+ final File storeData =
+ storageProcessor.storeData(null, null, TYPE_EXTRACTOR, null,
+ incomingDataSetDirectory, root);
+ assertEquals(true, storeData.exists());
+ assertEquals(false, incomingDataSetDirectory.exists());
+ storageProcessor.unstoreData(incomingDataSetDirectory, root);
+ assertEquals(false, storeData.exists());
+ assertEquals(true, incomingDataSetDirectory.exists());
+ }
+
+ //
+ // Helper classes
+ //
+
+ final static class TestProcedureAndDataTypeExtractor implements IProcedureAndDataTypeExtractor
+ {
+
+ static final String PROCEDURE_TYPE = ProcedureTypeCode.DATA_ACQUISITION.getCode();
+
+ static final String DATA_SET_TYPE = "dataSetType";
+
+ static final String LOCATOR_TYPE = "locatorType";
+
+ static final String FILE_FORMAT_TYPE = "fileFormatType";
+
+ //
+ // IProcedureAndDataTypeExtractor
+ //
+
+ public final FileFormatType getFileFormatType(final File incomingDataSetPath)
+ {
+ return new FileFormatType(FILE_FORMAT_TYPE);
+ }
+
+ public final LocatorType getLocatorType(final File incomingDataSetPath)
+ {
+ return new LocatorType(LOCATOR_TYPE);
+ }
+
+ public final DataSetType getDataSetType(final File incomingDataSetPath)
+ {
+ return new DataSetType(DATA_SET_TYPE);
+ }
+
+ public final ProcedureType getProcedureType(final File incomingDataSetPath)
+ {
+ final ProcedureType procedureType = new ProcedureType(PROCEDURE_TYPE);
+ procedureType.setDataAcquisition(true);
+ return procedureType;
+ }
+ }
+}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/EncapsulatedLimsServiceTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/EncapsulatedLimsServiceTest.java
new file mode 100644
index 00000000000..a29896c3df1
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/EncapsulatedLimsServiceTest.java
@@ -0,0 +1,143 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import org.jmock.Expectations;
+import org.jmock.Mockery;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.common.exceptions.InvalidSessionException;
+import ch.systemsx.cisd.openbis.generic.shared.IETLLIMSService;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ExternalData;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifier;
+import ch.systemsx.cisd.openbis.generic.shared.dto.types.ProcedureTypeCode;
+
+/**
+ * Test cases for corresponding {@link EncapsulatedLimsService} class.
+ *
+ * @author Basil Neff
+ */
+public class EncapsulatedLimsServiceTest
+{
+ private Mockery context;
+
+ private IETLLIMSService limsService;
+
+ private IEncapsulatedLimsService encapsulatedLimsService;
+
+ private static final String LIMS_USER = "testuser";
+
+ private static final String LIMS_PASSWORD = "testpassword";
+
+ private final DataSetInformation createDataSetInformation()
+ {
+ final DataSetInformation dataSetInformation = new DataSetInformation();
+ dataSetInformation.setSampleCode("S1");
+ return dataSetInformation;
+ }
+
+ private void prepareCallGetBaseExperiment(final Expectations exp,
+ final DataSetInformation dataSetInformation)
+ {
+ exp.one(limsService).authenticate(LIMS_USER, LIMS_PASSWORD);
+ exp.one(limsService).tryToGetBaseExperiment("", dataSetInformation.getSampleIdentifier());
+ }
+
+ @BeforeMethod
+ public void setUp()
+ {
+ context = new Mockery();
+ limsService = context.mock(IETLLIMSService.class);
+ encapsulatedLimsService =
+ new EncapsulatedLimsService(limsService, LIMS_USER, LIMS_PASSWORD);
+ }
+
+ @AfterMethod
+ public void tearDown()
+ {
+ // To following line of code should also be called at the end of each test method.
+ // Otherwise one do not known which test failed.
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public final void testGetBaseExperimentReauthentificate()
+ {
+ final DataSetInformation dataSetInformation = createDataSetInformation();
+ context.checking(new Expectations()
+ {
+ {
+ prepareCallGetBaseExperiment(this, dataSetInformation);
+ will(throwException(new InvalidSessionException("error")));
+ prepareCallGetBaseExperiment(this, dataSetInformation);
+ }
+ });
+ encapsulatedLimsService.getBaseExperiment(dataSetInformation.getSampleIdentifier());
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public final void testGetBaseExperiment()
+ {
+ final DataSetInformation dataSetInformation = createDataSetInformation();
+ context.checking(new Expectations()
+ {
+ {
+ prepareCallGetBaseExperiment(this, dataSetInformation);
+ }
+ });
+ encapsulatedLimsService.getBaseExperiment(dataSetInformation.getSampleIdentifier());
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public final void testRegisterDataSet()
+ {
+ final DataSetInformation dataSetInfo = createDataSetInformation();
+ final ProcedureTypeCode procedureTypeCode = ProcedureTypeCode.DATA_ACQUISITION;
+ final ExternalData data = new ExternalData();
+ context.checking(new Expectations()
+ {
+ {
+ one(limsService).authenticate(LIMS_USER, LIMS_PASSWORD);
+ one(limsService).registerDataSet("", dataSetInfo.getSampleIdentifier(),
+ procedureTypeCode.getCode(), data);
+ }
+ });
+ encapsulatedLimsService.registerDataSet(dataSetInfo, procedureTypeCode.getCode(), data);
+
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public final void testIsSampleRegisteredForDataSet()
+ {
+ final SampleIdentifier sampleIdentifier = SampleIdentifier.createHomeGroup("");
+ context.checking(new Expectations()
+ {
+ {
+ one(limsService).authenticate(LIMS_USER, LIMS_PASSWORD);
+ one(limsService).tryToGetPropertiesOfTopSampleRegisteredFor("",
+ sampleIdentifier);
+ }
+ });
+ encapsulatedLimsService.getPropertiesOfTopSampleRegisteredFor(sampleIdentifier);
+ context.assertIsSatisfied();
+ }
+}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/FileBasedFileTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/FileBasedFileTest.java
new file mode 100644
index 00000000000..e034e39b564
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/FileBasedFileTest.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import static org.testng.AssertJUnit.*;
+
+import java.io.File;
+
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.common.TimingParameters;
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
+import ch.systemsx.cisd.common.logging.LogInitializer;
+
+/**
+ * Test cases for corresponding {@link FileBasedFile} class.
+ *
+ * @author Franz-Josef Elmer
+ */
+public class FileBasedFileTest
+{
+ private static final File WORKING_DIRECTORY =
+ new File("targets/unit-test-wd/FileBasedFileTest");
+
+ private static final File DESTINATION = new File(WORKING_DIRECTORY, "destination");
+
+ @BeforeTest
+ public void setUp()
+ {
+ LogInitializer.init();
+ FileUtilities.deleteRecursively(WORKING_DIRECTORY);
+ assertTrue(WORKING_DIRECTORY.mkdirs());
+ assertTrue(DESTINATION.mkdirs());
+ }
+
+ @Test
+ public void copyFileUsingHardLinks()
+ {
+ File file = new File(WORKING_DIRECTORY, "test.txt");
+ FileUtilities.writeToFile(file, "hello world!");
+ File destFile = new File(DESTINATION, "copy_of_test.txt");
+ IFile destinationFile =
+ new FileBasedFileFactory(true, TimingParameters.getNoTimeoutNoRetriesParameters())
+ .create(destFile.getPath());
+
+ destinationFile.copyFrom(file);
+
+ assertEquals("hello world!", FileUtilities.loadToString(destFile).trim());
+ }
+
+ @Test
+ public void copyDirectoryUsingHardLinks()
+ {
+ File folder = new File(WORKING_DIRECTORY, "folder");
+ assertTrue(folder.mkdir());
+ File file1 = new File(folder, "file1.txt");
+ FileUtilities.writeToFile(file1, "hello file1");
+ File file2 = new File(folder, "file2.txt");
+ FileUtilities.writeToFile(file2, "hello file2");
+ File destFolder = new File(DESTINATION, "copy_folder");
+ IFile destinationFolder =
+ new FileBasedFileFactory(true, TimingParameters.createNoRetries(2000L))
+ .create(destFolder.getPath());
+
+ destinationFolder.copyFrom(folder);
+
+ assertEquals("hello file1", FileUtilities.loadToString(
+ new File(DESTINATION, "copy_folder/file1.txt")).trim());
+ assertEquals("hello file2", FileUtilities.loadToString(
+ new File(DESTINATION, "copy_folder/file2.txt")).trim());
+ }
+}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/HCSImageCheckListTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/HCSImageCheckListTest.java
new file mode 100644
index 00000000000..dbb413dd5b9
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/HCSImageCheckListTest.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.fail;
+
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.bds.hcs.Location;
+import ch.systemsx.cisd.bds.hcs.PlateGeometry;
+import ch.systemsx.cisd.bds.hcs.WellGeometry;
+
+/**
+ * Test cases for corresponding {@link HCSImageCheckList} class.
+ *
+ * @author Christian Ribeaud
+ */
+public final class HCSImageCheckListTest
+{
+
+ private static final WellGeometry WELL_GEOMETRY = new WellGeometry(1, 2);
+
+ private static final PlateGeometry PLATE_GEOMETRY = new PlateGeometry(2, 1);
+
+ private final static HCSImageCheckList createImageCheckList()
+ {
+ return new HCSImageCheckList(1, PLATE_GEOMETRY, WELL_GEOMETRY);
+ }
+
+ @Test
+ public final void testConstructor()
+ {
+ try
+ {
+ new HCSImageCheckList(0, null, null);
+ fail("IllegalArgumentException expected");
+ } catch (final IllegalArgumentException e)
+ {
+ assertEquals("Number of channels smaller than one.", e.getMessage());
+ }
+ try
+ {
+ new HCSImageCheckList(1, null, null);
+ fail("IllegalArgumentException expected");
+ } catch (final IllegalArgumentException e)
+ {
+ assertEquals("Unspecified plate geometry.", e.getMessage());
+ }
+ try
+ {
+ new HCSImageCheckList(1, PLATE_GEOMETRY, null);
+ fail("IllegalArgumentException expected");
+ } catch (final IllegalArgumentException e)
+ {
+ assertEquals("Unspecified well geometry.", e.getMessage());
+ }
+ new HCSImageCheckList(1, PLATE_GEOMETRY, WELL_GEOMETRY);
+ }
+
+ @Test
+ public final void testCheckOff()
+ {
+ final HCSImageCheckList checkList = createImageCheckList();
+ assertEquals(4, checkList.getCheckedOnFullLocations().size());
+ try
+ {
+ checkList.checkOff(1, new Location(2, 1), new Location(1, 1));
+ fail("Wrong well location.");
+ } catch (final IllegalArgumentException ex)
+ {
+ }
+ try
+ {
+ checkList.checkOff(1, new Location(1, 2), new Location(1, 2));
+ fail("Wrong tile location.");
+ } catch (final IllegalArgumentException ex)
+ {
+ }
+ checkList.checkOff(1, new Location(1, 2), new Location(2, 1));
+ assertEquals(3, checkList.getCheckedOnFullLocations().size());
+ try
+ {
+ checkList.checkOff(1, new Location(1, 2), new Location(2, 1));
+ fail("Image already handled.");
+ } catch (IllegalArgumentException ex)
+ {
+ }
+ checkList.checkOff(1, new Location(1, 1), new Location(1, 1));
+ checkList.checkOff(1, new Location(1, 1), new Location(2, 1));
+ checkList.checkOff(1, new Location(1, 2), new Location(1, 1));
+ assertEquals(0, checkList.getCheckedOnFullLocations().size());
+ }
+}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/IdentifiedDataStrategyTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/IdentifiedDataStrategyTest.java
new file mode 100644
index 00000000000..2e6aa25956a
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/IdentifiedDataStrategyTest.java
@@ -0,0 +1,141 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertNotNull;
+import static org.testng.AssertJUnit.assertTrue;
+import static org.testng.AssertJUnit.fail;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.commons.io.FileUtils;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.common.filesystem.AbstractFileSystemTestCase;
+import ch.systemsx.cisd.openbis.generic.shared.dto.DataSetType;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.ExperimentIdentifier;
+import ch.systemsx.cisd.openbis.generic.shared.dto.types.DataSetTypeCode;
+
+/**
+ * Test cases for corresponding {@link IdentifiedDataStrategy} class.
+ *
+ * @author Christian Ribeaud
+ */
+public class IdentifiedDataStrategyTest extends AbstractFileSystemTestCase
+{
+ private static final String EXAMPLE_PROJECT_CODE = "P";
+
+ private static final String EXAMPLE_EXPERIMENT_CODE = "E";
+
+ private final static String FILE_NAME = "AX14";
+
+ private final static IdentifiedDataStrategy strategy = new IdentifiedDataStrategy();
+
+ private final static DataSetType dataSetType =
+ new DataSetType(DataSetTypeCode.HCS_IMAGE.getCode());
+
+ private static final String EXAMPLE_GROUP_CODE = "G";
+
+ final static DataSetInformation createDataSetInfo()
+ {
+ final DataSetInformation dataSetInfo = new DataSetInformation();
+ final ExperimentIdentifier experimentIdentifier = new ExperimentIdentifier();
+ experimentIdentifier.setExperimentCode(EXAMPLE_EXPERIMENT_CODE);
+ experimentIdentifier.setProjectCode(EXAMPLE_PROJECT_CODE);
+ experimentIdentifier.setGroupCode(EXAMPLE_GROUP_CODE);
+ dataSetInfo.setExperimentIdentifier(experimentIdentifier);
+ dataSetInfo.setSampleCode("S");
+ dataSetInfo.setInstanceCode("my-instance");
+ dataSetInfo.setInstanceUUID("1111-2222");
+ dataSetInfo.setDataSetCode("data-set-code");
+ return dataSetInfo;
+ }
+
+ //
+ // AbstractFileSystemTestCase
+ //
+
+ @Override
+ @BeforeMethod
+ public void setUp() throws IOException
+ {
+ super.setUp();
+ }
+
+ @Test
+ public final void testGetBaseDirectory() throws IOException
+ {
+ boolean exceptionThrown = false;
+ try
+ {
+ strategy.getBaseDirectory(null, null, null);
+ } catch (final AssertionError ex)
+ {
+ exceptionThrown = true;
+ }
+ assertTrue("Null values not permited here", exceptionThrown);
+ final DataSetInformation dataSetInfo = createDataSetInfo();
+ File baseDirectory = strategy.getBaseDirectory(workingDirectory, dataSetInfo, dataSetType);
+ final File file =
+ new File(workingDirectory, "Instance_1111-2222/Group_G/Project_P/Experiment_E/"
+ + "DataSetType_HCS_IMAGE/Sample_S/Dataset_data-set-code");
+ assertEquals(file, baseDirectory);
+ assertTrue(baseDirectory.exists() == false);
+ // Create a file instead of a directory
+ FileUtils.touch(file);
+ try
+ {
+ strategy.getBaseDirectory(workingDirectory, dataSetInfo, dataSetType);
+ fail("illegal storage layout not detected");
+ } catch (final EnvironmentFailureException ex)
+ {
+ assertTrue("Unexpected exception message: " + ex.getMessage(), ex.getMessage()
+ .startsWith(IdentifiedDataStrategy.STORAGE_LAYOUT_ERROR_MSG_PREFIX));
+ }
+ FileUtils.forceDelete(file);
+ assert file.exists() == false;
+ // Create base directory
+ baseDirectory.mkdirs();
+ assertTrue(baseDirectory.exists() && baseDirectory.isDirectory());
+ try
+ {
+ baseDirectory = strategy.getBaseDirectory(workingDirectory, dataSetInfo, dataSetType);
+ fail("illegal storage layout not detected");
+ } catch (final EnvironmentFailureException ex)
+ {
+ assertTrue("Unexpected exception message: " + ex.getMessage(), ex.getMessage()
+ .startsWith(IdentifiedDataStrategy.STORAGE_LAYOUT_ERROR_MSG_PREFIX));
+ }
+ }
+
+ @Test
+ public final void testGetTargetPath()
+ {
+ File file = new File(FILE_NAME);
+ File targetPath = strategy.getTargetPath(workingDirectory, file);
+ assertEquals(new File(workingDirectory, FILE_NAME), targetPath);
+ final String property = System.getProperty("java.io.tmpdir");
+ assertNotNull(property);
+ file = new File(property, FILE_NAME);
+ targetPath = strategy.getTargetPath(workingDirectory, file);
+ assertEquals(new File(workingDirectory, FILE_NAME), targetPath);
+ }
+}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/MainTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/MainTest.java
new file mode 100644
index 00000000000..1b2fc09fbb7
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/MainTest.java
@@ -0,0 +1,136 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import static org.testng.AssertJUnit.assertFalse;
+import static org.testng.AssertJUnit.assertTrue;
+
+import java.io.File;
+
+import org.testng.AssertJUnit;
+import org.testng.annotations.Test;
+
+import ch.rinn.restrictions.Friend;
+import ch.systemsx.cisd.common.filesystem.AbstractFileSystemTestCase;
+import ch.systemsx.cisd.openbis.generic.shared.dto.DatabaseInstancePE;
+
+/**
+ * Test cases for corresponding {@link Main} class.
+ *
+ * @author Christian Ribeaud
+ */
+@Friend(toClasses = Main.class)
+public final class MainTest extends AbstractFileSystemTestCase
+{
+
+ private final static DatabaseInstancePE createDatabaseInstance()
+ {
+ final DatabaseInstancePE databaseInstancePE = new DatabaseInstancePE();
+ databaseInstancePE.setCode("XXX");
+ databaseInstancePE.setUuid("1111-2222");
+ return databaseInstancePE;
+ }
+
+ @Test
+ public final void testMigrateStoreRootDir()
+ {
+ final File instanceDir =
+ new File(
+ new File(workingDirectory, IdentifiedDataStrategy.INSTANCE_PREFIX + "CISD"),
+ IdentifiedDataStrategy.GROUP_PREFIX + "CISD");
+ instanceDir.mkdirs();
+ assertTrue(instanceDir.exists());
+ final DatabaseInstancePE databaseInstancePE = createDatabaseInstance();
+ // Not same code
+ Main.migrateStoreRootDir(workingDirectory, databaseInstancePE);
+ assertTrue(instanceDir.exists());
+ databaseInstancePE.setCode("CISD");
+ // Same code
+ Main.migrateStoreRootDir(workingDirectory, databaseInstancePE);
+ assertFalse(instanceDir.exists());
+ assertTrue(new File(workingDirectory, IdentifiedDataStrategy.INSTANCE_PREFIX
+ + databaseInstancePE.getUuid()).exists());
+ // Trying again does not change anything
+ Main.migrateStoreRootDir(workingDirectory, databaseInstancePE);
+ assertFalse(instanceDir.exists());
+ assertTrue(new File(workingDirectory, IdentifiedDataStrategy.INSTANCE_PREFIX
+ + databaseInstancePE.getUuid()).exists());
+ }
+
+ @Test
+ public void testMigrateDataStoreByRenamingObservableTypeToDataSetType() throws Exception
+ {
+ String observableTypeValue = "DST1";
+ String observableTypeDirPrefix = "ObservableType_";
+ File instanceDir =
+ new File(workingDirectory, IdentifiedDataStrategy.INSTANCE_PREFIX + "I1");
+ File groupDir = new File(instanceDir, IdentifiedDataStrategy.GROUP_PREFIX + "G1");
+ File projectDir = new File(groupDir, IdentifiedDataStrategy.PROJECT_PREFIX + "P1");
+ File experimentDir = new File(projectDir, IdentifiedDataStrategy.EXPERIMENT_PREFIX + "E1");
+ File observableTypeDir =
+ new File(experimentDir, observableTypeDirPrefix + observableTypeValue);
+ File sampleDir = new File(observableTypeDir, IdentifiedDataStrategy.SAMPLE_PREFIX + "S1");
+ File dataSetDir = new File(sampleDir, IdentifiedDataStrategy.DATASET_PREFIX + "D1");
+ File metadataDir = new File(dataSetDir, "metadata");
+ File metadataDataSetDir = new File(metadataDir, "data_set");
+
+ //
+ // Don't break when directory does not exist
+ //
+
+ Main.migrateDataStoreByRenamingObservableTypeToDataSetType(workingDirectory);
+
+ //
+ // Rename ObservableType_<> directory and observable_type file
+ //
+
+ // create directories
+ metadataDataSetDir.mkdirs();
+ assertTrue(metadataDataSetDir.exists());
+ assertTrue(observableTypeDir.getName()
+ .equals(observableTypeDirPrefix + observableTypeValue));
+
+ // create files
+ String observableTypeFileName = "observable_type";
+ File observableTypeFile = new File(metadataDataSetDir, observableTypeFileName);
+ observableTypeFile.createNewFile();
+ assertTrue(observableTypeFile.exists());
+ AssertJUnit.assertEquals(observableTypeFileName, metadataDataSetDir.listFiles()[0]
+ .getName());
+ assertTrue(observableTypeFile.getName().equals(observableTypeFileName));
+
+ // Do the migration
+ Main.migrateDataStoreByRenamingObservableTypeToDataSetType(workingDirectory);
+
+ // check directory renamed
+ AssertJUnit.assertEquals(IdentifiedDataStrategy.DATA_SET_TYPE_PREFIX + observableTypeValue,
+ experimentDir.listFiles()[0].getName());
+
+ // update variables
+ observableTypeDir =
+ new File(experimentDir, IdentifiedDataStrategy.DATA_SET_TYPE_PREFIX
+ + observableTypeValue);
+ sampleDir = new File(observableTypeDir, IdentifiedDataStrategy.SAMPLE_PREFIX + "S1");
+ dataSetDir = new File(sampleDir, IdentifiedDataStrategy.DATASET_PREFIX + "D1");
+ metadataDir = new File(dataSetDir, "metadata");
+ metadataDataSetDir = new File(metadataDir, "data_set");
+
+ // check file renamed
+ AssertJUnit.assertEquals("data_set_type", metadataDataSetDir.listFiles()[0].getName());
+ }
+
+}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/NamedDataStrategyTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/NamedDataStrategyTest.java
new file mode 100644
index 00000000000..d4750df591a
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/NamedDataStrategyTest.java
@@ -0,0 +1,127 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.commons.io.FileUtils;
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.common.filesystem.AbstractFileSystemTestCase;
+import ch.systemsx.cisd.openbis.generic.shared.dto.DataSetType;
+
+/**
+ * Test cases for corresponding {@link NamedDataStrategy} class.
+ *
+ * @author Christian Ribeaud
+ */
+public final class NamedDataStrategyTest extends AbstractFileSystemTestCase
+{
+ private static final DataStoreStrategyKey UNIDENTIFIED = DataStoreStrategyKey.UNIDENTIFIED;
+
+ private final static String FILE_NAME = "AX14";
+
+ private final static String TEST_FILENAME = "test";
+
+ private final static NamedDataStrategy strategy = new NamedDataStrategy(UNIDENTIFIED);
+
+ private final void createSomeFiles() throws IOException
+ {
+ createNumberedFiles(workingDirectory);
+ assert workingDirectory.list().length == 3;
+ assert new File(workingDirectory, FILE_NAME + "_[1]").exists();
+ }
+
+ private final static void createNumberedFiles(final File dir) throws IOException
+ {
+ for (int i = 0; i < 3; i++)
+ {
+ FileUtils.touch(new File(dir, FILE_NAME + "_[" + (i + 1) + "]"));
+ }
+ }
+
+ @Test
+ public void testCreateTargetPathNoFileExists() throws IOException
+ {
+ assertEquals(TEST_FILENAME, NamedDataStrategy.createTargetPath(
+ new File(workingDirectory, TEST_FILENAME)).getName());
+ }
+
+ @Test
+ public void testCreateTargetPathOriginalFileExists() throws IOException
+ {
+ FileUtils.touch(new File(workingDirectory, TEST_FILENAME));
+ assertEquals(TEST_FILENAME + "_[1]", NamedDataStrategy.createTargetPath(
+ new File(workingDirectory, TEST_FILENAME)).getName());
+ }
+
+ @Test
+ public void testCreateTargetPathSomeFilesExist() throws IOException
+ {
+ FileUtils.touch(new File(workingDirectory, TEST_FILENAME));
+ FileUtils.touch(new File(workingDirectory, TEST_FILENAME + "_[1]"));
+ FileUtils.touch(new File(workingDirectory, TEST_FILENAME + "_[2]"));
+ FileUtils.touch(new File(workingDirectory, TEST_FILENAME + "_[3]"));
+ assertEquals(TEST_FILENAME + "_[4]", NamedDataStrategy.createTargetPath(
+ new File(workingDirectory, TEST_FILENAME)).getName());
+ }
+
+ @Test
+ public final void testGetBaseDirectory() throws IOException
+ {
+ createSomeFiles();
+ boolean exceptionThrown = false;
+ try
+ {
+ strategy.getBaseDirectory(null, null, null);
+ } catch (AssertionError e)
+ {
+ exceptionThrown = true;
+ }
+ assertTrue("Base directory can not be null", exceptionThrown);
+ final DataSetType dataSetType = new DataSetType("DataSet");
+ final File baseDirectory =
+ strategy.getBaseDirectory(workingDirectory, null, dataSetType);
+ assertEquals(new File(new File(workingDirectory, NamedDataStrategy
+ .getDirectoryName(UNIDENTIFIED)), IdentifiedDataStrategy
+ .createDataSetTypeDirectory(dataSetType)), baseDirectory);
+ }
+
+ @Test(dependsOnMethods = "testGetBaseDirectory")
+ public final void testGetTargetPath() throws IOException
+ {
+ createSomeFiles();
+ boolean exceptionThrown = false;
+ try
+ {
+ strategy.getTargetPath(null, null);
+ } catch (AssertionError e)
+ {
+ exceptionThrown = true;
+ }
+ assertTrue("Base directory and incoming data set can not be null", exceptionThrown);
+ File targetPath = strategy.getTargetPath(workingDirectory, new File(FILE_NAME));
+ assertEquals(new File(workingDirectory, FILE_NAME), targetPath);
+ FileUtils.touch(targetPath);
+ targetPath = strategy.getTargetPath(workingDirectory, new File(FILE_NAME));
+ assertEquals(new File(workingDirectory, FILE_NAME + "_[4]"), targetPath);
+ }
+}
\ No newline at end of file
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/SimpleTypeExtractorTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/SimpleTypeExtractorTest.java
new file mode 100644
index 00000000000..3fd7bc8b613
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/SimpleTypeExtractorTest.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import static org.testng.AssertJUnit.assertEquals;
+
+import java.util.Properties;
+
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.openbis.generic.shared.dto.FileFormatType;
+import ch.systemsx.cisd.openbis.generic.shared.dto.LocatorType;
+import ch.systemsx.cisd.openbis.generic.shared.dto.types.DataSetTypeCode;
+import ch.systemsx.cisd.openbis.generic.shared.dto.types.ProcedureTypeCode;
+
+/**
+ * Test cases for corresponding {@link SimpleTypeExtractor} class.
+ *
+ * @author Christian Ribeaud
+ */
+public class SimpleTypeExtractorTest
+{
+
+ private final static Properties createProperties()
+ {
+ Properties props = new Properties();
+ props.put(SimpleTypeExtractor.FILE_FORMAT_TYPE_KEY, "F");
+ props.put(SimpleTypeExtractor.LOCATOR_TYPE_KEY, "L");
+ props.put(SimpleTypeExtractor.DATA_SET_TYPE_KEY, "O");
+ props.put(SimpleTypeExtractor.PROCEDURE_TYPE_KEY, ProcedureTypeCode.DATA_ACQUISITION
+ .getCode());
+ return props;
+ }
+
+ @Test
+ public final void testConstructor()
+ {
+ SimpleTypeExtractor extractor = new SimpleTypeExtractor(new Properties());
+ assertEquals(extractor.getFileFormatType(null).getCode(),
+ FileFormatType.DEFAULT_FILE_FORMAT_TYPE_CODE);
+ assertEquals(extractor.getLocatorType(null).getCode(),
+ LocatorType.DEFAULT_LOCATOR_TYPE_CODE);
+ assertEquals(extractor.getDataSetType(null).getCode(), DataSetTypeCode.HCS_IMAGE
+ .getCode());
+ assertEquals(extractor.getProcedureType(null).getCode(), ProcedureTypeCode.DATA_ACQUISITION
+ .getCode());
+ extractor = new SimpleTypeExtractor(createProperties());
+ assertEquals("F", extractor.getFileFormatType(null).getCode());
+ assertEquals("L", extractor.getLocatorType(null).getCode());
+ assertEquals("O", extractor.getDataSetType(null).getCode());
+ assertEquals(ProcedureTypeCode.DATA_ACQUISITION.getCode(), extractor.getProcedureType(null)
+ .getCode());
+ }
+}
\ No newline at end of file
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/StandardProcessingFactoryTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/StandardProcessingFactoryTest.java
new file mode 100644
index 00000000000..e91f484cf16
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/StandardProcessingFactoryTest.java
@@ -0,0 +1,127 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.fail;
+
+import java.util.Properties;
+
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+
+/**
+ * Test cases for the {@link StandardProcessorFactory}.
+ *
+ * @author Bernd Rinn
+ */
+public class StandardProcessingFactoryTest
+{
+
+ @Test
+ public void testCreateStandardProcessingFactoryWithMissingInputDataSetFormat()
+ {
+ try
+ {
+ final Properties props = new Properties();
+ props.put("parameters-file", "parameters.dat");
+ props.put("finished-file-template", ".finished_{0}");
+ StandardProcessorFactory.create(props);
+ fail("Missing property not detected");
+ } catch (ConfigurationFailureException ex)
+ {
+ assertEquals("Given key 'input-storage-format' not found in properties"
+ + " '[parameters-file, finished-file-template]'", ex.getMessage());
+ }
+ }
+
+ @Test
+ public void testCreateStandardProcessingFactoryWithIllegalInputDataSetFormat()
+ {
+ try
+ {
+ final Properties props = new Properties();
+ props.put("input-storage-format", "ILLEGAL");
+ props.put("parameters-file", "parameters.dat");
+ props.put("finished-file-template", ".finished_{0}");
+ StandardProcessorFactory.create(props);
+ fail("Illegal property value not detected");
+ } catch (ConfigurationFailureException ex)
+ {
+ assertEquals("input-storage-format property has illegal value 'ILLEGAL'.", ex
+ .getMessage());
+ }
+ }
+
+ @Test
+ public void testCreateStandardProcessingFactoryWithInputDataSetFormatProprietary()
+ {
+ final Properties props = new Properties();
+ props.put("input-storage-format", "PROPRIETARY");
+ props.put("parameters-file", "parameters.dat");
+ props.put("finished-file-template", ".finished_{0}");
+ props.put("data-set-code-prefix-glue", "_");
+ StandardProcessorFactory.create(props);
+ }
+
+ @Test
+ public void testCreateStandardProcessingFactoryWithInputDataSetFormatBdsDirectory()
+ {
+ final Properties props = new Properties();
+ props.put("input-storage-format", "BDS_DIRECTORY");
+ props.put("parameters-file", "parameters.dat");
+ props.put("finished-file-template", ".finished_{0}");
+ props.put("data-set-code-prefix-glue", "_");
+ StandardProcessorFactory.create(props);
+ }
+
+ @Test
+ public void testCreateStandardProcessingFactoryWithMissingParametersFileProperty()
+ {
+ try
+ {
+ final Properties props = new Properties();
+ props.put("input-storage-format", "BDS_DIRECTORY");
+ props.put("finished-file-template", ".finished_{0}");
+ StandardProcessorFactory.create(props);
+ fail("Missing property not detected");
+ } catch (ConfigurationFailureException ex)
+ {
+ assertEquals("Given key 'parameters-file' not found in properties"
+ + " '[input-storage-format, finished-file-template]'", ex.getMessage());
+ }
+ }
+
+ @Test
+ public void testCreateStandardProcessingFactoryWithMissingFinishedFileTemplateProperty()
+ {
+ try
+ {
+ final Properties props = new Properties();
+ props.put("input-storage-format", "BDS_DIRECTORY");
+ props.put("parameters-file", "parameters.dat");
+ StandardProcessorFactory.create(props);
+ fail("Missing property not detected");
+ } catch (ConfigurationFailureException ex)
+ {
+ assertEquals("Given key 'finished-file-template' not found in properties"
+ + " '[parameters-file, input-storage-format]'", ex.getMessage());
+ }
+ }
+
+}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/StandardProcessorTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/StandardProcessorTest.java
new file mode 100644
index 00000000000..a1ff51e182e
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/StandardProcessorTest.java
@@ -0,0 +1,209 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertTrue;
+import static org.testng.AssertJUnit.fail;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.Description;
+import org.jmock.Expectations;
+import org.jmock.Mockery;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.filesystem.AbstractFileSystemTestCase;
+import ch.systemsx.cisd.common.filesystem.PathPrefixPrepender;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ProcessingInstructionDTO;
+import ch.systemsx.cisd.openbis.generic.shared.dto.StorageFormat;
+
+/**
+ * Test cases for the {@link StandardProcessor}.
+ *
+ * @author Christian Ribeaud
+ */
+public final class StandardProcessorTest extends AbstractFileSystemTestCase
+{
+ private final static class AbsolutPathMatcher extends BaseMatcher<String>
+ {
+ private final String absolutePath;
+
+ AbsolutPathMatcher(final String absolutePath)
+ {
+ this.absolutePath = absolutePath;
+ }
+
+ //
+ // BaseMatcher
+ //
+
+ public final void describeTo(final Description description)
+ {
+ description.appendText(absolutePath);
+ }
+
+ public final boolean matches(final Object item)
+ {
+ if (item instanceof String == false)
+ {
+ return false;
+ }
+ final String path = (String) item;
+ return path.replace('\\', '/').equals(absolutePath.replace('\\', '/'));
+ }
+ }
+
+ private static final String PROCESSING_PATH = "processing";
+
+ private static final String PREFIX_FOR_RELATIVE_PATH = "rel";
+
+ private static final String PREFIX_FOR_ABSOLUTE_PATH = null;
+
+ private static final String FINISHED_FILE_NAME_TEMPLATE = ".MARKER_is_finished_{0}";
+
+ private static final String PARAMETERS_FILE_NAME = "parameters";
+
+ private IProcessor processor;
+
+ private Mockery context;
+
+ private IFileFactory fileFactory;
+
+ private PathPrefixPrepender pathPrefixPrepender;
+
+ private IFile iFile;
+
+ @Override
+ @BeforeMethod
+ public final void setUp() throws IOException
+ {
+ super.setUp();
+ context = new Mockery();
+ fileFactory = context.mock(IFileFactory.class);
+ iFile = context.mock(IFile.class);
+ pathPrefixPrepender = createPathPrefixPrepender();
+ processor = createStandardProcessor();
+ }
+
+ @AfterMethod
+ public void tearDown()
+ {
+ // The following line of code should also be called at the end of each test method.
+ // Otherwise one do not known which test failed.
+ context.assertIsSatisfied();
+ }
+
+ private final PathPrefixPrepender createPathPrefixPrepender()
+ {
+ final File file = new File(workingDirectory, PREFIX_FOR_RELATIVE_PATH);
+ assertEquals(true, file.mkdir());
+ assertTrue(file.exists());
+ assertTrue(file.isDirectory());
+ return new PathPrefixPrepender(PREFIX_FOR_ABSOLUTE_PATH, file.getAbsolutePath());
+ }
+
+ private final IProcessor createStandardProcessor()
+ {
+ return new StandardProcessor(fileFactory, StorageFormat.PROPRIETARY, pathPrefixPrepender,
+ PARAMETERS_FILE_NAME, FINISHED_FILE_NAME_TEMPLATE, "_");
+ }
+
+ private final ProcessingInstructionDTO createProcessingInstruction()
+ {
+ final ProcessingInstructionDTO processingInstruction = new ProcessingInstructionDTO();
+ processingInstruction.setPath(PROCESSING_PATH);
+ return processingInstruction;
+ }
+
+ @Test
+ public final void testInitiateProcessingWithNullParameters()
+ {
+ try
+ {
+ processor.initiateProcessing(null, null, null);
+ fail("Null parameters not allowed here.");
+ } catch (final AssertionError ex)
+ {
+ // Nothing to do here.
+ }
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public final void testInitiateProcessingWithNotSuitableDirectory()
+ {
+ final File dataSet = new File("dataSet");
+ final String absolutePath =
+ new File(new File(workingDirectory, PREFIX_FOR_RELATIVE_PATH), PROCESSING_PATH)
+ .getAbsolutePath();
+ context.checking(new Expectations()
+ {
+ {
+ one(fileFactory).create(with(new AbsolutPathMatcher(absolutePath)));
+ will(returnValue(iFile));
+
+ one(iFile).check();
+ will(throwException(new ConfigurationFailureException("")));
+ }
+ });
+ try
+ {
+ processor.initiateProcessing(createProcessingInstruction(), new DataSetInformation(),
+ dataSet);
+ fail("Configuration failure exception.");
+ } catch (final ConfigurationFailureException ex)
+ {
+ // Nothing to do here.
+ }
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public final void testInitiateProcessing()
+ {
+ final String dataSetName = "dataSet";
+ final File dataSet = new File(dataSetName);
+ final String absolutePath =
+ new File(new File(workingDirectory, PREFIX_FOR_RELATIVE_PATH), PROCESSING_PATH)
+ .getAbsolutePath();
+ final DataSetInformation dataSetInformation = new DataSetInformation();
+ final String dataSetCode = "data-set-code";
+ dataSetInformation.setDataSetCode(dataSetCode);
+ final String dataSetFullName = dataSetCode + "_" + dataSetName;
+ context.checking(new Expectations()
+ {
+ {
+ one(fileFactory).create(with(new AbsolutPathMatcher(absolutePath)));
+ will(returnValue(iFile));
+
+ one(iFile).check();
+
+ one(fileFactory).create(iFile, dataSetFullName);
+ one(fileFactory).create(iFile, ".MARKER_is_finished_" + dataSetFullName);
+ }
+ });
+ processor.initiateProcessing(createProcessingInstruction(), dataSetInformation, dataSet);
+ context.assertIsSatisfied();
+ }
+
+}
\ No newline at end of file
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/ThreadParametersTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/ThreadParametersTest.java
new file mode 100644
index 00000000000..f36cd5c7625
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/ThreadParametersTest.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertNull;
+
+import java.util.Properties;
+
+import org.testng.annotations.Test;
+
+import ch.rinn.restrictions.Friend;
+
+/**
+ * Test cases for the {@link ThreadParameters}.
+ *
+ * @author Christian Ribeaud
+ */
+@Friend(toClasses = ThreadParameters.class)
+public final class ThreadParametersTest
+{
+
+ @Test
+ public final void testTryGetGroupCode()
+ {
+ final Properties properties = new Properties();
+ assertNull(ThreadParameters.tryGetGroupCode(properties));
+ properties.setProperty(ThreadParameters.GROUP_CODE_KEY, "");
+ assertNull(ThreadParameters.tryGetGroupCode(properties));
+ final String groupCode = "G1";
+ properties.setProperty(ThreadParameters.GROUP_CODE_KEY, groupCode);
+ assertEquals(groupCode, ThreadParameters.tryGetGroupCode(properties));
+ }
+}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandlerTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandlerTest.java
new file mode 100644
index 00000000000..24f2d60a65d
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandlerTest.java
@@ -0,0 +1,905 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertTrue;
+import static org.testng.AssertJUnit.fail;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Level;
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.Description;
+import org.jmock.Expectations;
+import org.jmock.Mockery;
+import org.jmock.api.ExpectationError;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.AfterTest;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+import ch.rinn.restrictions.Friend;
+import ch.systemsx.cisd.common.Constants;
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.common.exceptions.UserFailureException;
+import ch.systemsx.cisd.common.filesystem.AbstractFileSystemTestCase;
+import ch.systemsx.cisd.common.filesystem.QueueingPathRemoverService;
+import ch.systemsx.cisd.common.logging.BufferedAppender;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogInitializer;
+import ch.systemsx.cisd.common.mail.IMailClient;
+import ch.systemsx.cisd.common.mail.JavaMailProperties;
+import ch.systemsx.cisd.common.test.LogMonitoringAppender;
+import ch.systemsx.cisd.common.utilities.OSUtilities;
+import ch.systemsx.cisd.openbis.generic.shared.IETLLIMSService;
+import ch.systemsx.cisd.openbis.generic.shared.dto.DataSetType;
+import ch.systemsx.cisd.openbis.generic.shared.dto.DatabaseInstancePE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ExperimentPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ExternalData;
+import ch.systemsx.cisd.openbis.generic.shared.dto.FileFormatType;
+import ch.systemsx.cisd.openbis.generic.shared.dto.GroupPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.LocatorType;
+import ch.systemsx.cisd.openbis.generic.shared.dto.PersonPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ProcedureType;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ProcessingInstructionDTO;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ProjectPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.SamplePropertyPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.StorageFormat;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.ExperimentIdentifier;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifier;
+import ch.systemsx.cisd.openbis.generic.shared.dto.types.ProcedureTypeCode;
+
+/**
+ * Test cases for corresponding {@link TransferredDataSetHandler} class.
+ *
+ * @author Franz-Josef Elmer
+ */
+@Friend(toClasses = TransferredDataSetHandler.class)
+public final class TransferredDataSetHandlerTest extends AbstractFileSystemTestCase
+{
+
+ private static final String SAMPLE_CODE = "sample1";
+
+ private static final String LOG_MSG_OF_DATA_FORMAT_MISMATCH =
+ "Configuration Error: no processing initiated for data set";
+
+ private static final String FOLDER_NAME = "folder";
+
+ private static final String DATA2_NAME = "data2";
+
+ private static final String DATA1_NAME = "data1";
+
+ private static final String SESSION_TOKEN = "sessionToken";
+
+ private static final String DATA_SET_CODE = "4711-42";
+
+ private static final String PARENT_DATA_SET_CODE = "4711-1";
+
+ private static final ProcedureType PROCEDURE_TYPE =
+ new ProcedureType(ProcedureTypeCode.DATA_ACQUISITION.getCode());
+
+ private static final LocatorType LOCATOR_TYPE = new LocatorType("L1");
+
+ private static final DataSetType DATA_SET_TYPE = new DataSetType("O1");
+
+ private static final FileFormatType FILE_FORMAT_TYPE = new FileFormatType("FF1");
+
+ private static final String DATA_PRODUCER_CODE = "microscope";
+
+ private static final Date DATA_PRODUCTION_DATE = new Date(2001);
+
+ private static final String EXAMPLE_PROCEDURE_TYPE_CODE =
+ ProcedureTypeCode.DATA_ACQUISITION.getCode();
+
+ private static final class ExternalDataMatcher extends BaseMatcher<ExternalData>
+ {
+ private final ExternalData expectedData;
+
+ public ExternalDataMatcher(final ExternalData externalData)
+ {
+ this.expectedData = externalData;
+ }
+
+ public void describeTo(final Description description)
+ {
+ description.appendValue(expectedData);
+ }
+
+ public boolean matches(final Object item)
+ {
+ if (item instanceof ExternalData == false)
+ {
+ return false;
+ }
+ final ExternalData data = (ExternalData) item;
+ assertEquals(expectedData.getCode(), data.getCode());
+ assertEquals(expectedData.getDataProducerCode(), data.getDataProducerCode());
+ assertEquals(expectedData.getLocation(), data.getLocation());
+ assertEquals(expectedData.getLocatorType(), data.getLocatorType());
+ assertEquals(expectedData.getFileFormatType(), data.getFileFormatType());
+ assertEquals(expectedData.getDataSetType(), data.getDataSetType());
+ assertEquals(expectedData.getParentDataSetCode(), data.getParentDataSetCode());
+ assertEquals(expectedData.getProductionDate(), data.getProductionDate());
+ assertEquals(expectedData.getStorageFormat(), data.getStorageFormat());
+ return true;
+ }
+
+ // This method throws an ExpectationError instead of the usual AssertionError.
+ // Reason: TransferredDataSetHandler catches AssertionError.
+ private void assertEquals(final Object expected, final Object actual)
+ {
+ if (expected == null ? expected != actual : expected.equals(actual) == false)
+ {
+ throw new ExpectationError("Expecting <" + expected + "> but got <" + actual + ">",
+ this);
+ }
+ }
+ }
+
+ /**
+ * This wrapper is needed because the class name of the wrapped class is not known.
+ */
+ private static final class MockDataSetInfoExtractor implements IDataSetInfoExtractor
+ {
+ private final IDataSetInfoExtractor codeExtractor;
+
+ MockDataSetInfoExtractor(final IDataSetInfoExtractor codeExtractor)
+ {
+ this.codeExtractor = codeExtractor;
+ }
+
+ public DataSetInformation getDataSetInformation(final File incomingDataSetPath)
+ throws UserFailureException, EnvironmentFailureException
+ {
+ return codeExtractor.getDataSetInformation(incomingDataSetPath);
+ }
+ }
+
+ private Mockery context;
+
+ private IDataSetInfoExtractor dataSetInfoExtractor;
+
+ private IProcedureAndDataTypeExtractor typeExtractor;
+
+ private IStorageProcessor storageProcessor;
+
+ private IETLLIMSService limsService;
+
+ private TransferredDataSetHandler handler;
+
+ private File data1;
+
+ private File isFinishedData1;
+
+ private DataSetInformation dataSetInformation;
+
+ private String relativeTargetFolder;
+
+ private ExternalData targetData1;
+
+ private File folder;
+
+ private File isFinishedFolder;
+
+ private File data2;
+
+ private IMailClient mailClient;
+
+ private IProcessorFactory processorFactory;
+
+ private IProcessor processor;
+
+ private BufferedAppender logRecorder;
+
+ private DatabaseInstancePE homeDatabaseInstance;
+
+ @BeforeTest
+ public void init()
+ {
+ QueueingPathRemoverService.start();
+ }
+
+ @AfterTest
+ public void finish()
+ {
+ QueueingPathRemoverService.stop();
+ }
+
+ @Override
+ @BeforeMethod
+ public final void setUp() throws IOException
+ {
+ super.setUp();
+ LogInitializer.init();
+ data1 = new File(workingDirectory, DATA1_NAME);
+ FileUtils.touch(data1);
+ isFinishedData1 =
+ new File(workingDirectory, Constants.IS_FINISHED_PREFIX + data1.getName());
+ FileUtils.touch(isFinishedData1);
+
+ folder = new File(workingDirectory, FOLDER_NAME);
+ folder.mkdir();
+ data2 = new File(folder, DATA2_NAME);
+ FileUtils.touch(data2);
+ isFinishedFolder =
+ new File(workingDirectory, Constants.IS_FINISHED_PREFIX + folder.getName());
+ FileUtils.touch(isFinishedFolder);
+
+ context = new Mockery();
+ dataSetInfoExtractor = context.mock(IDataSetInfoExtractor.class);
+ typeExtractor = context.mock(IProcedureAndDataTypeExtractor.class);
+ final Properties properties = new Properties();
+ properties.setProperty(JavaMailProperties.MAIL_SMTP_HOST, "host");
+ properties.setProperty(JavaMailProperties.MAIL_FROM, "me");
+ properties.setProperty(Main.STOREROOT_DIR_KEY, workingDirectory.getPath());
+ storageProcessor = context.mock(IStorageProcessor.class);
+ limsService = context.mock(IETLLIMSService.class);
+ mailClient = context.mock(IMailClient.class);
+ processorFactory = context.mock(IProcessorFactory.class);
+ processor = context.mock(IProcessor.class);
+ final Map<String, IProcessorFactory> map = new HashMap<String, IProcessorFactory>();
+ map.put(EXAMPLE_PROCEDURE_TYPE_CODE, processorFactory);
+ final IETLServerPlugin plugin =
+ new ETLServerPlugin(new MockDataSetInfoExtractor(dataSetInfoExtractor),
+ typeExtractor, storageProcessor);
+ final IEncapsulatedLimsService authorizedLimsService =
+ new EncapsulatedLimsService(limsService, "u", "p");
+ handler =
+ new TransferredDataSetHandler(null, storageProcessor, plugin,
+ authorizedLimsService, mailClient, true);
+
+ handler.setProcessorFactories(map);
+ dataSetInformation = new DataSetInformation();
+ final ExperimentIdentifier experimentIdentifier = new ExperimentIdentifier();
+ experimentIdentifier.setExperimentCode("experiment1".toUpperCase());
+ experimentIdentifier.setProjectCode("project1".toUpperCase());
+ experimentIdentifier.setGroupCode("group1".toUpperCase());
+ homeDatabaseInstance = new DatabaseInstancePE();
+ homeDatabaseInstance.setCode("my-instance");
+ homeDatabaseInstance.setUuid("1111-2222");
+ dataSetInformation.setInstanceCode(homeDatabaseInstance.getCode());
+ dataSetInformation.setInstanceUUID(homeDatabaseInstance.getUuid());
+ dataSetInformation.setExperimentIdentifier(experimentIdentifier);
+ dataSetInformation.setSampleCode(SAMPLE_CODE);
+ dataSetInformation.setProducerCode(DATA_PRODUCER_CODE);
+ dataSetInformation.setProductionDate(DATA_PRODUCTION_DATE);
+ dataSetInformation.setDataSetCode(DATA_SET_CODE);
+ dataSetInformation.setParentDataSetCode(PARENT_DATA_SET_CODE);
+ relativeTargetFolder =
+ "Instance_1111-2222" + File.separator + "Group_"
+ + experimentIdentifier.getGroupCode() + File.separator + "Project_"
+ + experimentIdentifier.getProjectCode() + File.separator + "Experiment_"
+ + experimentIdentifier.getExperimentCode() + File.separator
+ + "DataSetType_" + DATA_SET_TYPE.getCode() + File.separator + "Sample_"
+ + dataSetInformation.getSampleIdentifier().getSampleCode() + File.separator
+ + "Dataset_" + DATA_SET_CODE;
+ targetData1 = createTargetData(data1);
+ logRecorder = new BufferedAppender("%-5p %c - %m%n", Level.INFO);
+ }
+
+ private final String createLogMsgOfSuccess(final ExperimentIdentifier identifier,
+ final SampleIdentifier sampleIdentifier)
+ {
+ return String.format(TransferredDataSetHandler.SUCCESSFULLY_REGISTERED_TEMPLATE,
+ DATA_SET_CODE, sampleIdentifier, DATA_SET_TYPE.getCode(), identifier);
+ }
+
+ private final void assertLog(final String expectedLog)
+ {
+ assertEquals(expectedLog, normalize(logRecorder.getLogContent()));
+ }
+
+ private final String normalize(final String message)
+ {
+ return message.replace(workingDirectory.getAbsolutePath(), "/<wd>").replace(
+ workingDirectory.getPath(), "<wd>").replace('\\', '/');
+ }
+
+ private final ExternalData createTargetData(final File dataSet)
+ {
+ final ExternalData data = new ExternalData();
+ data.setLocation(relativeTargetFolder + File.separator + dataSet.getName());
+ data.setLocatorType(LOCATOR_TYPE);
+ data.setDataSetType(DATA_SET_TYPE);
+ data.setFileFormatType(FILE_FORMAT_TYPE);
+ data.setStorageFormat(StorageFormat.BDS_DIRECTORY);
+ data.setDataProducerCode(DATA_PRODUCER_CODE);
+ data.setProductionDate(DATA_PRODUCTION_DATE);
+ data.setCode(DATA_SET_CODE);
+ data.setParentDataSetCode(PARENT_DATA_SET_CODE);
+ return data;
+ }
+
+ private final static ExperimentPE createBaseExperiment(
+ final DataSetInformation dataSetInformation)
+ {
+ final ExperimentPE baseExperiment = new ExperimentPE();
+ final ExperimentIdentifier experimentIdentifier =
+ dataSetInformation.getExperimentIdentifier();
+ baseExperiment.setCode(experimentIdentifier.getExperimentCode());
+ final GroupPE group = new GroupPE();
+ group.setCode(experimentIdentifier.getGroupCode());
+ final ProjectPE project = new ProjectPE();
+ project.setCode(experimentIdentifier.getProjectCode());
+ project.setGroup(group);
+ baseExperiment.setProject(project);
+ final PersonPE person = new PersonPE();
+ person.setEmail("john.doe@somewhere.com");
+ baseExperiment.setRegistrator(person);
+ baseExperiment.setProcessingInstructions(new ProcessingInstructionDTO[]
+ { create() });
+ return baseExperiment;
+ }
+
+ private final static ProcessingInstructionDTO create()
+ {
+ final ProcessingInstructionDTO processingInstruction = new ProcessingInstructionDTO();
+ processingInstruction.setProcedureTypeCode(EXAMPLE_PROCEDURE_TYPE_CODE);
+ return processingInstruction;
+ }
+
+ @AfterMethod
+ public void tearDown()
+ {
+ logRecorder.reset();
+ // The following line of code should also be called at the end of each test method.
+ // Otherwise one do not known which test failed.
+ context.assertIsSatisfied();
+ }
+
+ private final void prepareForStrategy(final File dataSet, final ExperimentPE baseExperiment)
+ {
+ context.checking(new Expectations()
+ {
+ {
+ one(dataSetInfoExtractor).getDataSetInformation(dataSet);
+ will(returnValue(dataSetInformation));
+
+ one(limsService).authenticate("u", "p");
+ will(returnValue(SESSION_TOKEN));
+
+ one(limsService).getHomeDatabaseInstance(SESSION_TOKEN);
+ will(returnValue(homeDatabaseInstance));
+
+ one(storageProcessor).getStoreRootDirectory();
+ will(returnValue(workingDirectory));
+
+ atLeast(1).of(limsService).tryToGetBaseExperiment(SESSION_TOKEN,
+ dataSetInformation.getSampleIdentifier());
+ will(returnValue(baseExperiment));
+
+ allowing(typeExtractor).getDataSetType(dataSet);
+ will(returnValue(DATA_SET_TYPE));
+ }
+ });
+ }
+
+ private final void prepareForStrategyIDENTIFIED(final File dataSet,
+ final ExternalData targetData, final ExperimentPE baseExperiment)
+ {
+ prepareForStrategy(dataSet, baseExperiment);
+ context.checking(new Expectations()
+ {
+ {
+ one(limsService).tryToGetPropertiesOfTopSampleRegisteredFor(SESSION_TOKEN,
+ dataSetInformation.getSampleIdentifier());
+ will(returnValue(new SamplePropertyPE[0]));
+ }
+ });
+ }
+
+ private final void prepareForRegistration(final File dataSet)
+ {
+ context.checking(new Expectations()
+ {
+ {
+ one(typeExtractor).getLocatorType(dataSet);
+ will(returnValue(LOCATOR_TYPE));
+
+ one(typeExtractor).getFileFormatType(dataSet);
+ will(returnValue(FILE_FORMAT_TYPE));
+
+ one(typeExtractor).getProcedureType(dataSet);
+ will(returnValue(PROCEDURE_TYPE));
+ }
+ });
+ }
+
+ private final String getNotificationEmailContent(final DataSetInformation dataset,
+ final String dataSetCode)
+ {
+ final StringBuffer stringBuffer = new StringBuffer();
+ stringBuffer.append(createLogMsgOfSuccess(dataset.getExperimentIdentifier(), dataset
+ .getSampleIdentifier())
+ + OSUtilities.LINE_SEPARATOR + OSUtilities.LINE_SEPARATOR);
+ stringBuffer.append("Experiment Identifier:\t"
+ + dataSetInformation.getExperimentIdentifier() + OSUtilities.LINE_SEPARATOR);
+ stringBuffer.append("Producer Code:\t" + dataset.getProducerCode()
+ + OSUtilities.LINE_SEPARATOR);
+ if (dataset.getProductionDate() != null)
+ {
+ stringBuffer.append("Production Date:\t" + dataset.getProductionDate()
+ + OSUtilities.LINE_SEPARATOR);
+ }
+ if (StringUtils.isNotBlank(dataset.getParentDataSetCode()))
+ {
+ stringBuffer.append("Parent Data Set:\t" + dataset.getParentDataSetCode()
+ + OSUtilities.LINE_SEPARATOR);
+ }
+ stringBuffer.append("Is complete:\t" + dataset.getIsCompleteFlag()
+ + OSUtilities.LINE_SEPARATOR);
+
+ return stringBuffer.toString();
+ }
+
+ private final void checkSuccessEmailNotification(final Expectations expectations,
+ final DataSetInformation dataSet, final String dataSetCode, final String recipient)
+ {
+ expectations.one(mailClient).sendMessage(
+ String.format(TransferredDataSetHandler.EMAIL_SUBJECT_TEMPLATE, dataSet
+ .getExperimentIdentifier().getExperimentCode()),
+ getNotificationEmailContent(dataSet, dataSetCode), null, recipient);
+ }
+
+ @Test
+ public final void testDataSetFileIsReadOnly()
+ {
+ data1.setReadOnly();
+ context.checking(new Expectations()
+ {
+ {
+ one(storageProcessor).getStoreRootDirectory();
+ will(returnValue(workingDirectory));
+ }
+ });
+
+ try
+ {
+ handler.handle(isFinishedData1);
+ fail("EnvironmentFailureException expected");
+ } catch (final EnvironmentFailureException e)
+ {
+ final String normalizedMessage = normalize(e.getMessage());
+ assertEquals("Error moving path 'data1' from '<wd>' to '<wd>': "
+ + "Incoming data set directory '<wd>/data1' is not writable.",
+ normalizedMessage);
+ }
+
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public final void testNoDataSetInfoCouldBeExtractedFromDataSetFileName()
+ {
+ context.checking(new Expectations()
+ {
+ {
+ one(dataSetInfoExtractor).getDataSetInformation(data1);
+ will(returnValue(new DataSetInformation()));
+ }
+ });
+
+ try
+ {
+ handler.handle(isFinishedData1);
+ fail("ConfigurationFailureException expected.");
+ } catch (final ConfigurationFailureException e)
+ {
+ final String normalizedMessage = normalize(e.getMessage());
+ assertEquals("Data Set Information Extractor 'MockDataSetInfoExtractor' extracted "
+ + "no sample code for incoming data set '<wd>/data1' "
+ + "(extractor contract violation).", normalizedMessage);
+ }
+
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public final void testBaseDirectoryCouldNotBeCreated() throws IOException
+ {
+ FileUtils.touch(new File(workingDirectory,
+ "Instance_1111-2222/Group_GROUP1/Project_PROJECT1/Experiment_EXPERIMENT1/"
+ + "DataSetType_O1/Sample_" + SAMPLE_CODE + "/Dataset_" + DATA_SET_CODE));
+ prepareForStrategyIDENTIFIED(data1, null, createBaseExperiment(dataSetInformation));
+ try
+ {
+ handler.handle(isFinishedData1);
+ fail("Base directory could not be created because"
+ + " there is already a file with the same name.");
+ } catch (final EnvironmentFailureException ex)
+ {
+ assertTrue(ex.getMessage().indexOf(
+ IdentifiedDataStrategy.STORAGE_LAYOUT_ERROR_MSG_PREFIX) > -1);
+ }
+ assertLog("INFO OPERATION.DataStrategyStore - "
+ + "Identified that database knows experiment '/GROUP1/PROJECT1/EXPERIMENT1' "
+ + "and sample 'MY-INSTANCE:/sample1'.");
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public final void testMoveIdentifiedDataSetFile()
+ {
+ final ExperimentPE baseExperiment = createBaseExperiment(dataSetInformation);
+ baseExperiment.setProcessingInstructions(new ProcessingInstructionDTO[]
+ { create() });
+ final File baseDir = new File(workingDirectory, relativeTargetFolder);
+ prepareForStrategyIDENTIFIED(data1, targetData1, baseExperiment);
+ prepareForRegistration(data1);
+ context.checking(new Expectations()
+ {
+ {
+ one(limsService).registerDataSet(with(equal(SESSION_TOKEN)),
+ with(equal(dataSetInformation.getSampleIdentifier())),
+ with(equal(PROCEDURE_TYPE.getCode())),
+ with(new ExternalDataMatcher(targetData1)));
+
+ checkSuccessEmailNotification(this, dataSetInformation, DATA_SET_CODE,
+ baseExperiment.getRegistrator().getEmail());
+
+ allowing(storageProcessor).getStorageFormat();
+ will(returnValue(StorageFormat.BDS_DIRECTORY));
+ one(storageProcessor).storeData(baseExperiment, dataSetInformation,
+ typeExtractor, mailClient, data1, baseDir);
+ final File finalDataSetPath = new File(baseDir, DATA1_NAME);
+ will(returnValue(finalDataSetPath));
+
+ one(processorFactory).createProcessor();
+ will(returnValue(processor));
+ allowing(processor).getRequiredInputDataFormat();
+ will(returnValue(StorageFormat.BDS_DIRECTORY));
+ one(processor).initiateProcessing(
+ baseExperiment.getProcessingInstructions()[0], dataSetInformation,
+ finalDataSetPath);
+ }
+ });
+ final LogMonitoringAppender appender =
+ LogMonitoringAppender.addAppender(LogCategory.OPERATION, String
+ .format(createLogMsgOfSuccess(dataSetInformation.getExperimentIdentifier(),
+ dataSetInformation.getSampleIdentifier())));
+ handler.handle(isFinishedData1);
+ final File dataSetPath =
+ new File(baseDir.getParentFile(), IdentifiedDataStrategy.DATASET_PREFIX
+ + DATA_SET_CODE);
+ assertEquals(true, dataSetPath.isDirectory());
+ appender.verifyLogHasHappened();
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public final void testMoveIdentifiedDataSetFileToBDSContainerWithOriginalData()
+ {
+ final ExperimentPE baseExperiment = createBaseExperiment(dataSetInformation);
+ baseExperiment.setProcessingInstructions(new ProcessingInstructionDTO[]
+ { create() });
+ final File baseDir = new File(workingDirectory, relativeTargetFolder);
+ prepareForStrategyIDENTIFIED(data1, targetData1, baseExperiment);
+ prepareForRegistration(data1);
+ context.checking(new Expectations()
+ {
+ {
+ one(limsService).registerDataSet(with(equal(SESSION_TOKEN)),
+ with(equal(dataSetInformation.getSampleIdentifier())),
+ with(equal(PROCEDURE_TYPE.getCode())),
+ with(new ExternalDataMatcher(targetData1)));
+
+ checkSuccessEmailNotification(this, dataSetInformation, DATA_SET_CODE,
+ baseExperiment.getRegistrator().getEmail());
+
+ allowing(storageProcessor).getStorageFormat();
+ will(returnValue(StorageFormat.BDS_DIRECTORY));
+ one(storageProcessor).storeData(baseExperiment, dataSetInformation,
+ typeExtractor, mailClient, data1, baseDir);
+ final File finalDataSetPath = new File(baseDir, DATA1_NAME);
+ will(returnValue(finalDataSetPath));
+
+ one(processorFactory).createProcessor();
+ will(returnValue(processor));
+ allowing(processor).getRequiredInputDataFormat();
+ will(returnValue(StorageFormat.PROPRIETARY));
+ one(storageProcessor).tryGetProprietaryData(finalDataSetPath);
+ final File finalOriginalDataSetPath = new File(finalDataSetPath, "original");
+ will(returnValue(finalOriginalDataSetPath));
+ one(processor).initiateProcessing(
+ baseExperiment.getProcessingInstructions()[0], dataSetInformation,
+ finalOriginalDataSetPath);
+ }
+ });
+ final LogMonitoringAppender appender =
+ LogMonitoringAppender.addAppender(LogCategory.OPERATION, createLogMsgOfSuccess(
+ dataSetInformation.getExperimentIdentifier(), dataSetInformation
+ .getSampleIdentifier()));
+ handler.handle(isFinishedData1);
+ final File dataSetPath =
+ new File(baseDir.getParentFile(), IdentifiedDataStrategy.DATASET_PREFIX
+ + DATA_SET_CODE);
+ assertEquals(true, dataSetPath.isDirectory());
+ appender.verifyLogHasHappened();
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public final void testMoveIdentifiedDataSetFileButMismatchOfDataFormat()
+ {
+ final ExperimentPE baseExperiment = createBaseExperiment(dataSetInformation);
+ baseExperiment.setProcessingInstructions(new ProcessingInstructionDTO[]
+ { create() });
+ final File baseDir = new File(workingDirectory, relativeTargetFolder);
+ targetData1.setStorageFormat(StorageFormat.PROPRIETARY);
+ prepareForStrategyIDENTIFIED(data1, targetData1, baseExperiment);
+ prepareForRegistration(data1);
+ context.checking(new Expectations()
+ {
+ {
+ one(limsService).registerDataSet(with(equal(SESSION_TOKEN)),
+ with(equal(dataSetInformation.getSampleIdentifier())),
+ with(equal(PROCEDURE_TYPE.getCode())),
+ with(new ExternalDataMatcher(targetData1)));
+
+ checkSuccessEmailNotification(this, dataSetInformation, DATA_SET_CODE,
+ baseExperiment.getRegistrator().getEmail());
+
+ allowing(storageProcessor).getStorageFormat();
+ will(returnValue(StorageFormat.PROPRIETARY));
+ one(storageProcessor).storeData(baseExperiment, dataSetInformation,
+ typeExtractor, mailClient, data1, baseDir);
+ final File finalDataSetPath = new File(baseDir, DATA1_NAME);
+ will(returnValue(finalDataSetPath));
+
+ one(processorFactory).createProcessor();
+ will(returnValue(processor));
+ allowing(processor).getRequiredInputDataFormat();
+ will(returnValue(StorageFormat.BDS_DIRECTORY));
+
+ }
+ });
+ final LogMonitoringAppender appender =
+ LogMonitoringAppender.addAppender(LogCategory.NOTIFY,
+ LOG_MSG_OF_DATA_FORMAT_MISMATCH);
+ handler.handle(isFinishedData1);
+ final File dataSetPath =
+ new File(baseDir.getParentFile(), IdentifiedDataStrategy.DATASET_PREFIX
+ + DATA_SET_CODE);
+ assertEquals(true, dataSetPath.isDirectory());
+ appender.verifyLogHasHappened();
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public final void testMoveIdentifiedDataSetFileButMismatchOfDataFormat2()
+ {
+ final ExperimentPE baseExperiment = createBaseExperiment(dataSetInformation);
+ baseExperiment.setProcessingInstructions(new ProcessingInstructionDTO[]
+ { create() });
+ final File baseDir = new File(workingDirectory, relativeTargetFolder);
+ prepareForStrategyIDENTIFIED(data1, targetData1, baseExperiment);
+ prepareForRegistration(data1);
+ context.checking(new Expectations()
+ {
+ {
+ one(limsService).registerDataSet(with(equal(SESSION_TOKEN)),
+ with(equal(dataSetInformation.getSampleIdentifier())),
+ with(equal(PROCEDURE_TYPE.getCode())),
+ with(new ExternalDataMatcher(targetData1)));
+
+ checkSuccessEmailNotification(this, dataSetInformation, DATA_SET_CODE,
+ baseExperiment.getRegistrator().getEmail());
+
+ allowing(storageProcessor).getStorageFormat();
+ will(returnValue(StorageFormat.BDS_DIRECTORY));
+ one(storageProcessor).storeData(baseExperiment, dataSetInformation,
+ typeExtractor, mailClient, data1, baseDir);
+ final File finalDataSetPath = new File(baseDir, DATA1_NAME);
+ will(returnValue(finalDataSetPath));
+
+ one(storageProcessor).tryGetProprietaryData(finalDataSetPath);
+ will(returnValue(null));
+
+ one(processorFactory).createProcessor();
+ will(returnValue(processor));
+
+ allowing(processor).getRequiredInputDataFormat();
+ will(returnValue(StorageFormat.PROPRIETARY));
+
+ }
+ });
+ final LogMonitoringAppender appender =
+ LogMonitoringAppender.addAppender(LogCategory.NOTIFY,
+ LOG_MSG_OF_DATA_FORMAT_MISMATCH);
+ handler.handle(isFinishedData1);
+ final File dataSetPath =
+ new File(baseDir.getParentFile(), IdentifiedDataStrategy.DATASET_PREFIX
+ + DATA_SET_CODE);
+ assertEquals(true, dataSetPath.isDirectory());
+ appender.verifyLogHasHappened();
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public final void testMoveInvalidDataSetFile()
+ {
+ assertEquals(new File(workingDirectory, DATA1_NAME), data1);
+ assertEquals(new File(new File(workingDirectory, FOLDER_NAME), DATA2_NAME), data2);
+ assert data1.exists() && data2.exists();
+ final ExperimentPE baseExperiment = createBaseExperiment(dataSetInformation);
+ prepareForStrategy(data1, baseExperiment);
+ context.checking(new Expectations()
+ {
+ {
+ one(limsService).tryToGetPropertiesOfTopSampleRegisteredFor(SESSION_TOKEN,
+ dataSetInformation.getSampleIdentifier());
+ will(returnValue(null));
+
+ final ExperimentIdentifier experimentIdentifier =
+ dataSetInformation.getExperimentIdentifier();
+ final String subject =
+ String.format(DataStrategyStore.SUBJECT_FORMAT, experimentIdentifier);
+ final String body =
+ DataStrategyStore.createInvalidSampleCodeMessage(dataSetInformation);
+ final String email = baseExperiment.getRegistrator().getEmail();
+ one(mailClient).sendMessage(subject, body, null, email);
+ }
+ });
+ handler.handle(isFinishedData1);
+
+ assertEquals(false, isFinishedData1.exists());
+ assertLog("ERROR OPERATION.DataStrategyStore - "
+ + "Incoming data set '<wd>/data1' claims to belong to experiment "
+ + "'/GROUP1/PROJECT1/EXPERIMENT1' and sample identifier 'MY-INSTANCE:/"
+ + SAMPLE_CODE
+ + "', "
+ + "but according to the openBIS server there is no such sample for this experiment "
+ + "(it has maybe been invalidated?). We thus consider it invalid."
+ + OSUtilities.LINE_SEPARATOR + "INFO OPERATION.FileRenamer - "
+ + "Moving file 'data1' from '<wd>' to '<wd>/invalid/DataSetType_O1'.");
+
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public final void testMoveUnidentifiedDataSetFile()
+ {
+ assertEquals(new File(workingDirectory, DATA1_NAME), data1);
+ assertEquals(new File(new File(workingDirectory, FOLDER_NAME), DATA2_NAME), data2);
+ assert data1.exists() && data2.exists();
+ prepareForStrategy(data1, null);
+ final File toDir =
+ new File(new File(workingDirectory, NamedDataStrategy
+ .getDirectoryName(DataStoreStrategyKey.UNIDENTIFIED)),
+ IdentifiedDataStrategy.createDataSetTypeDirectory(DATA_SET_TYPE));
+
+ final LogMonitoringAppender appender =
+ LogMonitoringAppender.addAppender(LogCategory.OPERATION, "to '" + toDir + "'");
+ handler.handle(isFinishedData1);
+ assertEquals(false, isFinishedData1.exists());
+ appender.verifyLogHasHappened();
+
+ checkNamedStrategyDirectoryPresent(DataStoreStrategyKey.UNIDENTIFIED, data1);
+
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public final void testMoveIdentifiedDataSetFolderButStoreDataFailed()
+ {
+ final ExperimentPE baseExperiment = createBaseExperiment(dataSetInformation);
+ final File baseDir = new File(workingDirectory, relativeTargetFolder);
+ prepareForStrategyIDENTIFIED(folder, targetData1, baseExperiment);
+ context.checking(new Expectations()
+ {
+ {
+ one(processorFactory).createProcessor();
+ will(returnValue(processor));
+
+ one(typeExtractor).getProcedureType(folder);
+ will(returnValue(PROCEDURE_TYPE));
+
+ one(storageProcessor).storeData(baseExperiment, dataSetInformation,
+ typeExtractor, mailClient, folder, baseDir);
+ will(throwException(new Exception("Could store data by storage processor")));
+
+ one(storageProcessor).unstoreData(with(equal(folder)), with(equal(baseDir)));
+ }
+ });
+ final LogMonitoringAppender appender =
+ LogMonitoringAppender.addAppender(LogCategory.OPERATION, createLogMsgOfSuccess(
+ dataSetInformation.getExperimentIdentifier(), dataSetInformation
+ .getSampleIdentifier()));
+ final LogMonitoringAppender appender2 =
+ LogMonitoringAppender.addAppender(LogCategory.NOTIFY, String.format(
+ TransferredDataSetHandler.DATA_SET_STORAGE_FAILURE_TEMPLATE,
+ dataSetInformation));
+ handler.handle(isFinishedFolder);
+
+ appender.verifyLogHasNotHappened();
+ appender2.verifyLogHasHappened();
+
+ checkNamedStrategyDirectoryPresent(DataStoreStrategyKey.ERROR, folder);
+
+ context.assertIsSatisfied();
+ }
+
+ private final void checkNamedStrategyDirectoryPresent(final DataStoreStrategyKey key,
+ final File dataSet)
+ {
+ final File strategyDirectory =
+ new File(workingDirectory, NamedDataStrategy.getDirectoryName(key));
+ assertEquals(true, strategyDirectory.exists());
+ final File dataSetTypeDir =
+ new File(strategyDirectory, IdentifiedDataStrategy
+ .createDataSetTypeDirectory(DATA_SET_TYPE));
+ assertEquals(true, dataSetTypeDir.exists());
+ final File targetFile = new File(dataSetTypeDir, dataSet.getName());
+ assertEquals(true, targetFile.exists());
+ assertEquals(false, dataSet.exists());
+ }
+
+ @Test
+ public void testMoveIdentifiedDataSetFolderButWebServiceRegistrationFailed()
+ {
+ final ExperimentPE baseExperiment = createBaseExperiment(dataSetInformation);
+ final File baseDir = new File(workingDirectory, relativeTargetFolder);
+ targetData1.setStorageFormat(null);
+ prepareForStrategyIDENTIFIED(folder, targetData1, baseExperiment);
+ prepareForRegistration(folder);
+ context.checking(new Expectations()
+ {
+ {
+ one(processorFactory).createProcessor();
+ will(returnValue(processor));
+ one(storageProcessor).storeData(baseExperiment, dataSetInformation,
+ typeExtractor, mailClient, folder, baseDir);
+ will(returnValue(new File(baseDir, DATA1_NAME)));
+
+ one(limsService).registerDataSet(with(equal(SESSION_TOKEN)),
+ with(equal(dataSetInformation.getSampleIdentifier())),
+ with(equal(PROCEDURE_TYPE.getCode())),
+ with(new ExternalDataMatcher(targetData1)));
+ will(throwException(new EnvironmentFailureException(
+ "Could not register data set folder")));
+
+ one(storageProcessor).unstoreData(with(equal(folder)), with(equal(baseDir)));
+ one(storageProcessor).getStorageFormat();
+ }
+ });
+ final LogMonitoringAppender appender =
+ LogMonitoringAppender.addAppender(LogCategory.OPERATION, createLogMsgOfSuccess(
+ dataSetInformation.getExperimentIdentifier(), dataSetInformation
+ .getSampleIdentifier()));
+ final LogMonitoringAppender appender2 =
+ LogMonitoringAppender.addAppender(LogCategory.NOTIFY, String.format(
+ TransferredDataSetHandler.DATA_SET_REGISTRATION_FAILURE_TEMPLATE,
+ dataSetInformation));
+
+ handler.handle(isFinishedFolder);
+
+ appender.verifyLogHasNotHappened();
+ appender2.verifyLogHasHappened();
+
+ context.assertIsSatisfied();
+ }
+}
\ No newline at end of file
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/imsb/HCSImageFileExtractorTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/imsb/HCSImageFileExtractorTest.java
new file mode 100644
index 00000000000..1c017bbdf2c
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/imsb/HCSImageFileExtractorTest.java
@@ -0,0 +1,247 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.imsb;
+
+import static org.testng.AssertJUnit.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.log4j.Level;
+import org.jmock.Expectations;
+import org.jmock.Mockery;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import ch.rinn.restrictions.Friend;
+import ch.systemsx.cisd.bds.hcs.Location;
+import ch.systemsx.cisd.bds.hcs.WellGeometry;
+import ch.systemsx.cisd.bds.storage.IDirectory;
+import ch.systemsx.cisd.bds.storage.IFile;
+import ch.systemsx.cisd.bds.storage.filesystem.NodeFactory;
+import ch.systemsx.cisd.common.filesystem.AbstractFileSystemTestCase;
+import ch.systemsx.cisd.common.logging.BufferedAppender;
+import ch.systemsx.cisd.etlserver.DataSetInformation;
+import ch.systemsx.cisd.etlserver.IHCSImageFileAccepter;
+
+/**
+ * Test cases for the {@link HCSImageFileExtractor}.
+ *
+ * @author Christian Ribeaud
+ */
+@Friend(toClasses = HCSImageFileExtractor.class)
+public final class HCSImageFileExtractorTest extends AbstractFileSystemTestCase
+{
+
+ private final IDirectory workingDirectoryNode;
+
+ public HCSImageFileExtractorTest()
+ {
+ super();
+ this.workingDirectoryNode = NodeFactory.createDirectoryNode(workingDirectory);
+ }
+
+ static class TestHCSImageFileExtractor extends HCSImageFileExtractor
+ {
+
+ private List<IFile> files;
+
+ @Override
+ List<IFile> listTiffFiles(final IDirectory directory)
+ {
+ return files;
+ }
+
+ public TestHCSImageFileExtractor(final Properties properties)
+ {
+ super(properties);
+ }
+
+ void setFiles(final List<IFile> files)
+ {
+ this.files = files;
+ }
+
+ }
+
+ private static final String WELL_GEOMETRY = "3x3";
+
+ private static final String SAMPLE_CODE = "CP042-1ab";
+
+ private final DataSetInformation dataSetInformation = createDataSetInformation();
+
+ private TestHCSImageFileExtractor fileExtractor;
+
+ private Mockery context;
+
+ private IHCSImageFileAccepter fileAccepter;
+
+ private BufferedAppender logRecorder;
+
+ private final void prepareFileExtractor()
+ {
+ context = new Mockery();
+ fileAccepter = context.mock(IHCSImageFileAccepter.class);
+ logRecorder = new BufferedAppender("%m", Level.WARN);
+ fileExtractor = new TestHCSImageFileExtractor(createProperties());
+ }
+
+ private final static DataSetInformation createDataSetInformation()
+ {
+ final DataSetInformation dataSetInformation = new DataSetInformation();
+ dataSetInformation.setSampleCode(SAMPLE_CODE);
+ return dataSetInformation;
+ }
+
+ private final static Properties createProperties()
+ {
+ final Properties props = new Properties();
+ props.setProperty(WellGeometry.WELL_GEOMETRY, WELL_GEOMETRY);
+ return props;
+ }
+
+ private final IFile createFile(final String fileName) throws IOException
+ {
+ final File file = new File(workingDirectory, fileName);
+ FileUtils.touch(file);
+ assertTrue(file.exists());
+ return NodeFactory.createFileNode(file);
+ }
+
+ //
+ // AbstractFileSystemTestCase
+ //
+
+ @Override
+ @BeforeMethod
+ public final void setUp() throws IOException
+ {
+ super.setUp();
+ prepareFileExtractor();
+ logRecorder.resetLogContent();
+ }
+
+ @AfterMethod
+ public final void tearDown()
+ {
+ // To following line of code should also be called at the end of each test method.
+ // Otherwise one do not known which test failed.
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public final void processWithNull()
+ {
+ try
+ {
+ fileExtractor.process(null, null, null);
+ fail("Null values not allowed here.");
+ } catch (final AssertionError ex)
+ {
+ // Nothing to do here.
+ }
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public final void testProcessWithIncorrectSample() throws IOException
+ {
+ final String imagePath = "CP002-2bc_H24_6_w460.tif";
+ final List<IFile> files = new ArrayList<IFile>();
+ fileExtractor.setFiles(files);
+ files.add(createFile(imagePath));
+ assertEquals("", logRecorder.getLogContent());
+ assertEquals(1, fileExtractor.process(NodeFactory.createDirectoryNode(workingDirectory),
+ dataSetInformation, fileAccepter).getInvalidFiles().size());
+ }
+
+ @Test
+ public final void testProcessHappyCase() throws IOException
+ {
+ final String imagePath1 = "Some_trash_" + SAMPLE_CODE + "_H24_6_w530.tif";
+ final String imagePath2 = "Some_trash_" + SAMPLE_CODE + "_H24_6_w460.tif";
+ final List<IFile> files = new ArrayList<IFile>();
+ fileExtractor.setFiles(files);
+ final IFile file1 = createFile(imagePath1);
+ files.add(file1);
+ final IFile file2 = createFile(imagePath2);
+ files.add(file2);
+ final int channel1 = 2;
+ final int channel2 = 1;
+ final Location plateLocation = new Location(24, 8);
+ final Location wellLocation = new Location(3, 2);
+ context.checking(new Expectations()
+ {
+ {
+ one(fileAccepter).accept(channel1, plateLocation, wellLocation, file1);
+ one(fileAccepter).accept(channel2, plateLocation, wellLocation, file2);
+ }
+ });
+ assertEquals("", logRecorder.getLogContent());
+ final List<IFile> invalidFiles =
+ fileExtractor.process(workingDirectoryNode, dataSetInformation, fileAccepter)
+ .getInvalidFiles();
+ assertEquals(0, invalidFiles.size());
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public final void testProcessWithNotEnoughTokens() throws IOException
+ {
+ final String imagePath = "H24_6_w460.tif";
+ final List<IFile> files = new ArrayList<IFile>();
+ fileExtractor.setFiles(files);
+ final IFile file1 = createFile(imagePath);
+ files.add(file1);
+ createFile(imagePath);
+ fileExtractor.process(workingDirectoryNode, dataSetInformation, fileAccepter);
+ assertEquals(1, fileExtractor.process(workingDirectoryNode, dataSetInformation, fileAccepter)
+ .getInvalidFiles().size());
+ }
+
+ @Test
+ public final void testProcessWithIncorrectPlateCoordinate() throws IOException
+ {
+ final String imagePath = "Screening_" + SAMPLE_CODE + "_XX_6_w530.tiff";
+ final List<IFile> files = new ArrayList<IFile>();
+ fileExtractor.setFiles(files);
+ final IFile file1 = createFile(imagePath);
+ files.add(file1);
+ createFile(imagePath);
+ fileExtractor.process(workingDirectoryNode, dataSetInformation, fileAccepter);
+ assertEquals(1, fileExtractor.process(workingDirectoryNode, dataSetInformation, fileAccepter)
+ .getInvalidFiles().size());
+ }
+
+ @Test
+ public final void testProcessWithIncorrectWellCoordinate() throws IOException
+ {
+ final String imagePath = "Doesnt_matter_" + SAMPLE_CODE + "_H24_s6_w530.tif";
+ final List<IFile> files = new ArrayList<IFile>();
+ fileExtractor.setFiles(files);
+ final IFile file1 = createFile(imagePath);
+ files.add(file1);
+ fileExtractor.process(workingDirectoryNode, dataSetInformation, fileAccepter);
+ assertEquals(1, fileExtractor.process(workingDirectoryNode, dataSetInformation, fileAccepter)
+ .getInvalidFiles().size());
+ }
+}
\ No newline at end of file
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/threev/DataSetInfoExtractorForDataAcquisitionTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/threev/DataSetInfoExtractorForDataAcquisitionTest.java
new file mode 100644
index 00000000000..bea0c12945e
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/threev/DataSetInfoExtractorForDataAcquisitionTest.java
@@ -0,0 +1,119 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.threev;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.fail;
+
+import java.io.File;
+import java.text.SimpleDateFormat;
+import java.util.Properties;
+
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.etlserver.CodeExtractortTestCase;
+import ch.systemsx.cisd.etlserver.DataSetInformation;
+import ch.systemsx.cisd.etlserver.IDataSetInfoExtractor;
+
+/**
+ * @author Franz-Josef Elmer
+ */
+public class DataSetInfoExtractorForDataAcquisitionTest extends CodeExtractortTestCase
+{
+ private static final String INDICES_OF_DATA_SET_CODE_ENTITIES =
+ PREFIX + DataSetInfoExtractorForDataAcquisition.INDICES_OF_DATA_SET_CODE_ENTITIES;
+
+ private static final String DATA_SET_CODE_ENTITIES_GLUE =
+ PREFIX + AbstractDataSetInfoExtractorFor3V.DATA_SET_CODE_ENTITIES_GLUE;
+
+ @Test
+ public void testHappyCaseWithOnlyMandatoryPorperties()
+ {
+ final Properties properties = new Properties();
+ properties.setProperty(INDICES_OF_DATA_SET_CODE_ENTITIES, "1, 0");
+ final IDataSetInfoExtractor extractor =
+ new DataSetInfoExtractorForDataAcquisition(properties);
+
+ final DataSetInformation dataSetInfo =
+ extractor.getDataSetInformation(new File("alpha.42.beta"));
+ assertEquals("42.alpha", dataSetInfo.getDataSetCode());
+ assertEquals("beta", dataSetInfo.getSampleIdentifier().getSampleCode());
+ assertEquals(null, dataSetInfo.getParentDataSetCode());
+ assertEquals(null, dataSetInfo.getProducerCode());
+ assertEquals(null, dataSetInfo.getProductionDate());
+ }
+
+ @Test
+ public void testHappyCaseWithAllProperties()
+ {
+ final Properties properties = new Properties();
+ properties.setProperty(INDICES_OF_DATA_SET_CODE_ENTITIES, "-1 -2");
+ properties.setProperty(DATA_SET_CODE_ENTITIES_GLUE, "-");
+ properties.setProperty(ENTITY_SEPARATOR, "_");
+ properties.setProperty(INDEX_OF_SAMPLE_CODE, "0");
+ properties.setProperty(INDEX_OF_PARENT_DATA_SET_CODE, "1");
+ properties.setProperty(INDEX_OF_DATA_PRODUCER_CODE, "2");
+ properties.setProperty(INDEX_OF_DATA_PRODUCTION_DATE, "3");
+ final String dateFormat = "yyyy-MM-dd";
+ properties.setProperty(DATA_PRODUCTION_DATE_FORMAT, dateFormat);
+ final IDataSetInfoExtractor extractor =
+ new DataSetInfoExtractorForDataAcquisition(properties);
+
+ final String date = "2007-12-24";
+ final DataSetInformation dataSetInfo =
+ extractor.getDataSetInformation(new File("a_b_c_" + date));
+ assertEquals("2007-12-24-c", dataSetInfo.getDataSetCode());
+ assertEquals("a", dataSetInfo.getSampleIdentifier().getSampleCode());
+ assertEquals("b", dataSetInfo.getParentDataSetCode());
+ assertEquals("c", dataSetInfo.getProducerCode());
+ assertEquals(date, new SimpleDateFormat(dateFormat).format(dataSetInfo.getProductionDate()));
+ }
+
+ @Test
+ public void testConstructorWithMissingMandatoryProperty()
+ {
+ try
+ {
+ new DataSetInfoExtractorForDataAcquisition(new Properties());
+ fail("ConfigurationFailureException expected");
+ } catch (final ConfigurationFailureException e)
+ {
+ final String message = e.getMessage();
+ assertEquals(
+ "Given key 'indices-of-data-set-code-entities' not found in properties '[]'",
+ message);
+ }
+ }
+
+ @Test
+ public void testConstructorWithInvalidValuesForPropertyIndicesOfDataSetCodeEntities()
+ {
+ try
+ {
+ final Properties properties = new Properties();
+ properties.setProperty(INDICES_OF_DATA_SET_CODE_ENTITIES, "2,u");
+ new DataSetInfoExtractorForDataAcquisition(properties);
+ fail("ConfigurationFailureException expected");
+ } catch (final ConfigurationFailureException e)
+ {
+ assertEquals("2. index in property 'indices-of-data-set-code-entities' "
+ + "isn't a number: 2,u", e.getMessage());
+ }
+
+ }
+}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/threev/DataSetInfoExtractorForImageAnalysisTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/threev/DataSetInfoExtractorForImageAnalysisTest.java
new file mode 100644
index 00000000000..46ae5ddb907
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/threev/DataSetInfoExtractorForImageAnalysisTest.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.threev;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.fail;
+
+import java.io.File;
+import java.util.Properties;
+
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.etlserver.CodeExtractortTestCase;
+import ch.systemsx.cisd.etlserver.DataSetInformation;
+import ch.systemsx.cisd.etlserver.IDataSetInfoExtractor;
+
+/**
+ * @author Franz-Josef Elmer
+ */
+public class DataSetInfoExtractorForImageAnalysisTest extends CodeExtractortTestCase
+{
+ private static final String INDICES_OF_PARENT_DATA_SET_CODE_ENTITIES =
+ PREFIX + DataSetInfoExtractorForImageAnalysis.INDICES_OF_PARENT_DATA_SET_CODE_ENTITIES;
+
+ @Test
+ public void testHappyCaseWithOnlyMandatoryPorperties()
+ {
+ Properties properties = new Properties();
+ properties.setProperty(INDICES_OF_PARENT_DATA_SET_CODE_ENTITIES, "1, 0");
+ IDataSetInfoExtractor extractor = new DataSetInfoExtractorForImageAnalysis(properties);
+
+ DataSetInformation dataSetInfo = extractor.getDataSetInformation(new File("alpha.42.beta"));
+ assertEquals("42.alpha", dataSetInfo.getParentDataSetCode());
+ assertEquals("beta", dataSetInfo.getSampleIdentifier().getSampleCode());
+ assertEquals(null, dataSetInfo.getDataSetCode());
+ assertEquals(null, dataSetInfo.getProducerCode());
+ assertEquals(null, dataSetInfo.getProductionDate());
+ }
+
+ @Test
+ public void testConstructorWithMissingMandatoryProperty()
+ {
+ try
+ {
+ new DataSetInfoExtractorForImageAnalysis(new Properties());
+ fail("ConfigurationFailureException expected");
+ } catch (ConfigurationFailureException e)
+ {
+ String message = e.getMessage();
+ assertEquals(
+ "Given key 'indices-of-parent-data-set-code-entities' not found in properties '[]'",
+ message);
+ }
+ }
+}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/threev/HCSImageFileExtractorTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/threev/HCSImageFileExtractorTest.java
new file mode 100644
index 00000000000..b74684e2a0b
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/threev/HCSImageFileExtractorTest.java
@@ -0,0 +1,186 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.threev;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Properties;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.log4j.Level;
+import org.jmock.Expectations;
+import org.jmock.Mockery;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.bds.hcs.Location;
+import ch.systemsx.cisd.bds.hcs.WellGeometry;
+import ch.systemsx.cisd.bds.storage.IDirectory;
+import ch.systemsx.cisd.bds.storage.IFile;
+import ch.systemsx.cisd.bds.storage.filesystem.NodeFactory;
+import ch.systemsx.cisd.common.filesystem.AbstractFileSystemTestCase;
+import ch.systemsx.cisd.common.logging.BufferedAppender;
+import ch.systemsx.cisd.etlserver.IHCSImageFileAccepter;
+
+/**
+ * Test cases for the {@link HCSImageFileExtractor}.
+ *
+ * @author Christian Ribeaud
+ */
+public final class HCSImageFileExtractorTest extends AbstractFileSystemTestCase
+{
+
+ private static final String WELL_GEOMETRY = "3x3";
+
+ private HCSImageFileExtractor fileExtractor;
+
+ private Mockery context;
+
+ private IHCSImageFileAccepter fileAccepter;
+
+ private BufferedAppender logRecorder;
+
+ private final IDirectory workingDirectoryNode;
+
+ public HCSImageFileExtractorTest()
+ {
+ super();
+ this.workingDirectoryNode = NodeFactory.createDirectoryNode(workingDirectory);
+ }
+
+ private final void prepareFileExtractor()
+ {
+ context = new Mockery();
+ fileAccepter = context.mock(IHCSImageFileAccepter.class);
+ logRecorder = new BufferedAppender("%m", Level.WARN);
+ fileExtractor = new HCSImageFileExtractor(createProperties());
+ }
+
+ private final static Properties createProperties()
+ {
+ final Properties props = new Properties();
+ props.setProperty(WellGeometry.WELL_GEOMETRY, WELL_GEOMETRY);
+ return props;
+ }
+
+ private final IFile createFile(final String fileName) throws IOException
+ {
+ final File file = new File(workingDirectory, fileName);
+ FileUtils.touch(file);
+ assertTrue(file.exists());
+ return NodeFactory.createFileNode(file);
+ }
+
+ //
+ // AbstractFileSystemTestCase
+ //
+
+ @Override
+ @BeforeMethod
+ public final void setUp() throws IOException
+ {
+ super.setUp();
+ prepareFileExtractor();
+ logRecorder.resetLogContent();
+ }
+
+ @AfterMethod
+ public final void tearDown()
+ {
+ // To following line of code should also be called at the end of each test method.
+ // Otherwise one do not known which test failed.
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public final void processWithNull()
+ {
+ boolean exceptionThrown = false;
+ try
+ {
+ fileExtractor.process(null, null, null);
+ } catch (AssertionError ex)
+ {
+ exceptionThrown = true;
+ }
+ assertTrue("Null values not allowed here.", exceptionThrown);
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public final void testProcessWithNoCorrectPrefix() throws IOException
+ {
+ final String imagePath = "H24_s6_w1_[UUID].tif";
+ createFile(imagePath);
+ assertEquals(0, fileExtractor.process(workingDirectoryNode, null, fileAccepter)
+ .getInvalidFiles().size());
+ }
+
+ @Test
+ public final void testProcessHappyCase() throws IOException
+ {
+ final String imagePath = "Screening_H24_s6_w1_[UUID].tif";
+ final IFile file = createFile(imagePath);
+ final int channel = 1;
+ final Location plateLocation = new Location(24, 8);
+ final Location wellLocation = new Location(3, 2);
+ context.checking(new Expectations()
+ {
+ {
+ one(fileAccepter).accept(channel, plateLocation, wellLocation, file);
+ }
+ });
+ assertEquals("", logRecorder.getLogContent());
+ assertTrue(fileExtractor.process(workingDirectoryNode, null, fileAccepter).getInvalidFiles()
+ .isEmpty());
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public final void testProcessWithNotEnoughTokens() throws IOException
+ {
+ final String imagePath = "Screening_H24_s6_w1.tif";
+ createFile(imagePath);
+ fileExtractor.process(workingDirectoryNode, null, fileAccepter);
+ assertEquals(0, fileExtractor.process(workingDirectoryNode, null, fileAccepter)
+ .getInvalidFiles().size());
+ }
+
+ @Test
+ public final void testProcessWithNoRightPlateCoordinate() throws IOException
+ {
+ final String imagePath = "Screening_XX_s6_w1_UUID.tif";
+ createFile(imagePath);
+ fileExtractor.process(workingDirectoryNode, null, fileAccepter);
+ assertEquals(1, fileExtractor.process(workingDirectoryNode, null, fileAccepter)
+ .getInvalidFiles().size());
+ }
+
+ @Test
+ public final void testProcessWithNoRightWellCoordinate() throws IOException
+ {
+ final String imagePath = "Screening_H24_6_w1_UUID.tif";
+ createFile(imagePath);
+ fileExtractor.process(workingDirectoryNode, null, fileAccepter);
+ assertEquals(1, fileExtractor.process(workingDirectoryNode, null, fileAccepter)
+ .getInvalidFiles().size());
+ }
+}
\ No newline at end of file
diff --git a/datastore_server/sourceTest/java/tests.xml b/datastore_server/sourceTest/java/tests.xml
new file mode 100644
index 00000000000..e8c6f023e2b
--- /dev/null
+++ b/datastore_server/sourceTest/java/tests.xml
@@ -0,0 +1,14 @@
+<!DOCTYPE suite SYSTEM "http://beust.com/testng/testng-1.0.dtd" >
+
+<suite name="All" verbose="1">
+ <test name="All">
+ <groups>
+ <run>
+ <exclude name="broken" />
+ </run>
+ </groups>
+ <packages>
+ <package name="ch.systemsx.cisd.etlserver.*" />
+ </packages>
+ </test>
+</suite>
diff --git a/datastore_server/sourceTest/java/tests_fast.xml b/datastore_server/sourceTest/java/tests_fast.xml
new file mode 100644
index 00000000000..3aee85772b9
--- /dev/null
+++ b/datastore_server/sourceTest/java/tests_fast.xml
@@ -0,0 +1,15 @@
+<!DOCTYPE suite SYSTEM "http://beust.com/testng/testng-1.0.dtd" >
+
+<suite name="Fast" verbose="1">
+ <test name="Fast">
+ <groups>
+ <run>
+ <exclude name="slow" />
+ <exclude name="broken" />
+ </run>
+ </groups>
+ <packages>
+ <package name="ch.systemsx.cisd.etlserver.*" />
+ </packages>
+ </test>
+</suite>
--
GitLab