LMS-1218 initial project check in

SVN: 12848

deep_sequencing_unit/.classpath

+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry kind="src" path="source/java"/>
+	<classpathentry kind="src" path="sourceTest/java"/>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/J2SE-1.5"/>
+	<classpathentry combineaccessrules="false" kind="src" path="/datastore_server"/>
+	<classpathentry combineaccessrules="false" kind="src" path="/common"/>
+	<classpathentry kind="output" path="targets/classes"/>
+</classpath>

deep_sequencing_unit/.project

+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>deep_sequencing_unit</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.jdt.core.javabuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.jdt.core.javanature</nature>
+	</natures>
+</projectDescription>

deep_sequencing_unit/build/build.xml

+<project name="deep_sequencing_unit" default="ci" basedir="..">
+	<import file="../../datastore_server/build/build.xml" />
+	<project-classpath name="ecp" classes="${classes}" />
+
+	<property name="original.dist" value="dist" />
+	<property name="mainfolder" value="deep_sequencing_unit" />
+	<property name="variant" value="-dsu"/>
+
+	<target name="compile" depends="build-common.compile, clean" />
+
+	<target name="run-tests">
+		<antcall target="build-common.run-tests">
+			<param name="test.suite" value="tests.xml" />
+		</antcall>
+	</target>
+
+        <!--
+        // Task for creating distributions
+        -->
+        <target name="dist" depends="datastore_server.make-dist"/>
+	
+        <!--
+        // Task for continuous integration server.
+        -->
+        <target name="ci" depends="build-common.ci, dist, check-dependencies"/>
+	
+
+</project>
\ No newline at end of file

deep_sequencing_unit/dist/etc/log.xml

+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+<log4j:configuration xmlns:log4j='http://jakarta.apache.org/log4j/'>
+
+  <appender name="DEFAULT" class="org.apache.log4j.DailyRollingFileAppender">
+
+    <param name="File" value="log/datastore_server_log.txt"/>
+    <param name="DatePattern" value="'.'yyyy-MM-dd"/>
+
+    <layout class="org.apache.log4j.PatternLayout">
+      <param name="ConversionPattern" value="%d %-5p [%t] %c - %m%n"/>
+    </layout>
+  
+  </appender>
+
+  <appender name="STDOUT" class="org.apache.log4j.ConsoleAppender">
+     <layout class="org.apache.log4j.PatternLayout">
+       <param name="ConversionPattern" value="%d %-5p [%t] %c - %m%n"/>
+     </layout>
+  </appender>
+
+  <appender name="NULL" class="org.apache.log4j.varia.NullAppender" />
+
+  <appender name="EMAIL" class="org.apache.log4j.net.SMTPAppender">
+
+    <param name="BufferSize" value="512" />
+    <param name="SMTPHost" value="localhost" />
+    <param name="From" value="dsu@localhost" />
+    <param name="To" value="root@localhost" />
+    <param name="Subject" value="ATTENTION: DSU Data Set Server" />
+    <param name="EvaluatorClass" value="ch.systemsx.cisd.common.logging.AlwaysTrueTriggeringEventEvaluator" />
+
+    <layout class="org.apache.log4j.PatternLayout">
+      <param name="ConversionPattern" value="%d %-5p [%t] %c - %m%n"/>
+    </layout>
+  
+    <!--filter class="org.apache.log4j.varia.LevelRangeFilter">
+      <param name="LevelMin" value="ERROR"/>
+      <param name="LevelMax" value="FATAL"/>
+    </filter-->
+  
+  </appender>
+
+  <category name="NOTIFY">
+    <priority value="info" />
+    <appender-ref ref="EMAIL" />
+  </category>    
+  
+  <root>
+    <priority value="info" />
+    <appender-ref ref="DEFAULT" />
+  </root>
+  
+</log4j:configuration>

deep_sequencing_unit/dist/etc/passwd

+a:a@admins.com:A:The Admin:J3fII6Pex7jnCBwF+uXz2mFuB1QVhPUi
+u:u@users.com:U:The User:dmbGKaGRmbX8YKfslMxUHObmYfjywkuT
+o:o@observers.com:O:The Observer:t53ADCnFnEFhBvHB7FPoHhbHeW2O1KJc
+etlserver::::SHGHSPawL/B3NKXD5nsu4fSrj5LwR2MX

deep_sequencing_unit/dist/etc/service.properties

+# Unique code of this Data Store Server. Not more than 40 characters.
+data-store-server-code = DSS1
+
+# The root directory of the data store
+storeroot-dir = data/store
+
+# The directory where the command queue file is located; defaults to storeroot-dir 
+commandqueue-dir =
+
+# Port
+port = 8444
+
+# Session timeout in minutes
+session-timeout = 720
+
+# Path to the keystore
+keystore.path = etc/openBIS.keystore
+
+# Password of the keystore
+keystore.password = changeit
+
+# Key password of the keystore
+keystore.key-password = changeit
+
+# The check interval (in seconds)
+check-interval = 60
+
+# The time-out for clean up work in the shutdown sequence (in seconds).
+# Note that that the maximal time for the shutdown sequence to complete can be as large 
+# as twice this time.
+# Remark: On a network file system, it is not recommended to turn this value to something 
+# lower than 180.
+shutdown-timeout = 180
+
+# If free disk space goes below value defined here, a notification email will be sent.
+# Value must be specified in kilobytes (1048576 = 1024 * 1024 = 1GB). If no high water mark is
+# specified or if value is negative, the system will not be watching.
+highwater-mark = -1
+
+# If a data set is successfully registered it sends out an email to the registrator. 
+# If this property is not specified, no email is sent to the registrator. This property
+# does not affect the mails which are sent, when the data set could not be registered.
+notify-successful-registration = false
+
+# The URL of the openBIS server
+server-url = https://localhost:8443/openbis/openbis
+
+# The username to use when contacting the openBIS server
+username = etlserver
+
+# The password to use when contacting the openBIS server
+password = etlserver
+
+# The base URL for Web client access.
+download-url = https://localhost:8444
+
+# SMTP properties (must start with 'mail' to be considered). 
+# mail.smtp.host = localhost
+# mail.from = datastore_server@localhost
+
+# ---------------- Timing parameters for file system operations on remote shares.
+
+# Time (in seconds) to wait for any file system operation to finish. Operations exceeding this 
+# timeout will be terminated. 
+timeout = 60
+# Number of times that a timed out operation will be tried again (0 means: every file system 
+# operation will only ever be performed once).
+max-retries = 11
+# Time (in seconds) to wait after an operation has been timed out before re-trying.  
+failure-interval = 10 
+
+# The period of no write access that needs to pass before an incoming data item is considered 
+# complete and ready to be processed (in seconds) [default: 300]. 
+# Valid only when auto-detection method is used to determine if an incoming data are ready to be processed.
+quiet-period = 10
+
+# Globally used separator character which separates entities in a data set file name 
+data-set-file-name-entity-separator = _
+
+# Specifies what should happen if an error occurs during dataset processing. 
+# By default this flag is set to false and user has to modify the 'faulty paths file' 
+# each time the faulty dataset should be processed again.
+# Set this flag to true if the processing should be repeated after some time without manual intervention.
+# Note that this can increase the server load.
+# reprocess-faulty-datasets = false
+
+# Comma separated names of processing threads. Each thread should have configuration properties prefixed with its name.
+# E.g. 'code-extractor' property for the thread 'my-etl' should be specified as 'my-etl.code-extractor'
+inputs = flow-cell
+
+# ---------------------------------------------------------------------------
+# 'flow-cell' thread configuration
+# ---------------------------------------------------------------------------
+# The directory to watch for incoming data.
+flow-cell.incoming-dir = data/incoming
+
+# Determines when the incoming data should be considered complete and ready to be processed.
+# Allowed values: 
+#  - auto-detection - when no write access will be detected for a specified 'quite-period'
+#  - marker-file		- when an appropriate marker file for the data exists. 
+# The default value is 'marker-file'.
+flow-cell.incoming-data-completeness-condition = auto-detection
+
+# The group the samples extracted by this thread belong to. If commented out or empty, then samples
+# are considered associated to a database instance (not group private). 
+# flow-cell.group-code = <change this>
+
+# ---------------- Plugin properties
+# The extractor class to use for code extraction
+flow-cell.data-set-info-extractor = ch.systemsx.cisd.etlserver.DefaultDataSetInfoExtractor
+flow-cell.data-set-info-extractor.strip-file-extension = true
+flow-cell.data-set-info-extractor.entity-separator = ${data-set-file-name-entity-separator}
+flow-cell.data-set-info-extractor.group-code = TEST
+# Location of file containing data set properties 
+#flow-cell.data-set-info-extractor.data-set-properties-file-name = data-set.properties
+
+# The extractor class to use for type extraction
+flow-cell.type-extractor = ch.systemsx.cisd.etlserver.SimpleTypeExtractor
+flow-cell.type-extractor.file-format-type = PROPRIETARY
+flow-cell.type-extractor.locator-type = RELATIVE_LOCATION
+flow-cell.type-extractor.data-set-type = PLG_RESULT
+flow-cell.type-extractor.is-measured = true
+
+# The storage processor (IStorageProcessor implementation)
+flow-cell.storage-processor = ch.ethz.bsse.cisd.dsu.dss.StorageProcessor
+flow-cell.storage-processor.processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor

deep_sequencing_unit/etc/log.xml

+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+<log4j:configuration xmlns:log4j='http://jakarta.apache.org/log4j/'>
+
+  <appender name="STDOUT" class="org.apache.log4j.ConsoleAppender">
+     <layout class="org.apache.log4j.PatternLayout">
+       <param name="ConversionPattern" value="%d %-5p [%t] %c - %m%n"/>
+     </layout>
+  </appender>
+
+  <appender name="NULL" class="org.apache.log4j.varia.NullAppender" />
+
+  <root>
+     <priority value ="info" />
+     <appender-ref ref="STDOUT" />
+  </root>
+
+</log4j:configuration>

deep_sequencing_unit/etc/service.properties

+# Unique code of this Data Store Server. Not more than 40 characters.
+data-store-server-code = DSS1
+
+# The root directory of the data store
+storeroot-dir = targets/playground/store
+
+# The directory where the command queue file is located; defaults to storeroot-dir 
+commandqueue-dir =
+
+# Port
+port = 8889
+
+# Session timeout in minutes
+session-timeout = 720
+
+# Path to the keystore
+keystore.path = ../datastore_server/dist/etc/openBIS.keystore
+
+# Password of the keystore
+keystore.password = changeit
+
+# Key password of the keystore
+keystore.key-password = changeit
+
+# The check interval (in seconds)
+check-interval = 5
+
+# The time-out for clean up work in the shutdown sequence (in seconds).
+# Note that that the maximal time for the shutdown sequence to complete can be as large 
+# as twice this time.
+# Remark: On a network file system, it is not recommended to turn this value to something 
+# lower than 180.
+shutdown-timeout = 2
+
+# If free disk space goes below value defined here, a notification email will be sent.
+# Value must be specified in kilobytes (1048576 = 1024 * 1024 = 1GB). If no high water mark is
+# specified or if value is negative, the system will not be watching.
+highwater-mark = -1
+
+# If a data set is successfully registered it sends out an email to the registrator. 
+# If this property is not specified, no email is sent to the registrator. This property
+# does not affect the mails which are sent, when the data set could not be registered.
+notify-successful-registration = false
+
+# The URL of the openBIS server
+server-url = http://localhost:8888/openbis
+
+# The username to use when contacting the openBIS server
+username = etlserver
+
+# The password to use when contacting the openBIS server
+password = etlserver
+
+# The base URL for Web client access.
+download-url = https://localhost:${port}
+
+# SMTP properties (must start with 'mail' to be considered). 
+# mail.smtp.host = localhost
+# mail.from = datastore_server@localhost
+
+# ---------------- Timing parameters for file system operations on remote shares.
+
+# Time (in seconds) to wait for any file system operation to finish. Operations exceeding this 
+# timeout will be terminated. 
+timeout = 60
+# Number of times that a timed out operation will be tried again (0 means: every file system 
+# operation will only ever be performed once).
+max-retries = 11
+# Time (in seconds) to wait after an operation has been timed out before re-trying.  
+failure-interval = 10 
+
+# The period of no write access that needs to pass before an incoming data item is considered 
+# complete and ready to be processed (in seconds) [default: 300]. 
+# Valid only when auto-detection method is used to determine if an incoming data are ready to be processed.
+quiet-period = 10
+
+# Globally used separator character which separates entities in a data set file name 
+data-set-file-name-entity-separator = _
+
+# Specifies what should happen if an error occurs during dataset processing. 
+# By default this flag is set to false and user has to modify the 'faulty paths file' 
+# each time the faulty dataset should be processed again.
+# Set this flag to true if the processing should be repeated after some time without manual intervention.
+# Note that this can increase the server load.
+# reprocess-faulty-datasets = false
+
+# Comma separated names of processing threads. Each thread should have configuration properties prefixed with its name.
+# E.g. 'code-extractor' property for the thread 'my-etl' should be specified as 'my-etl.code-extractor'
+inputs = flow-cell
+
+# ---------------------------------------------------------------------------
+# 'flow-cell' thread configuration
+# ---------------------------------------------------------------------------
+# The directory to watch for incoming data.
+flow-cell.incoming-dir = targets/playground/incoming-flow-cell
+
+# Determines when the incoming data should be considered complete and ready to be processed.
+# Allowed values: 
+#  - auto-detection - when no write access will be detected for a specified 'quite-period'
+#  - marker-file		- when an appropriate marker file for the data exists. 
+# The default value is 'marker-file'.
+flow-cell.incoming-data-completeness-condition = auto-detection
+
+# The group the samples extracted by this thread belong to. If commented out or empty, then samples
+# are considered associated to a database instance (not group private). 
+# flow-cell.group-code = <change this>
+
+# ---------------- Plugin properties
+# The extractor class to use for code extraction
+flow-cell.data-set-info-extractor = ch.systemsx.cisd.etlserver.DefaultDataSetInfoExtractor
+flow-cell.data-set-info-extractor.strip-file-extension = true
+flow-cell.data-set-info-extractor.entity-separator = ${data-set-file-name-entity-separator}
+flow-cell.data-set-info-extractor.group-code = TEST
+# Location of file containing data set properties 
+#flow-cell.data-set-info-extractor.data-set-properties-file-name = data-set.properties
+
+# The extractor class to use for type extraction
+flow-cell.type-extractor = ch.systemsx.cisd.etlserver.SimpleTypeExtractor
+flow-cell.type-extractor.file-format-type = PROPRIETARY
+flow-cell.type-extractor.locator-type = RELATIVE_LOCATION
+flow-cell.type-extractor.data-set-type = FLOW_LINE
+flow-cell.type-extractor.is-measured = true
+
+# The storage processor (IStorageProcessor implementation)
+flow-cell.storage-processor = ch.ethz.bsse.cisd.dsu.dss.StorageProcessor
+flow-cell.storage-processor.processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor

deep_sequencing_unit/source/java/ch/ethz/bsse/cisd/dsu/dss/FlowLineFeeder.java

+/*
+ * Copyright 2009 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.bsse.cisd.dsu.dss;
+
+import java.io.File;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+import javax.naming.ConfigurationException;
+
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.common.utilities.PropertyUtils;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IPostRegistrationDatasetHandler;
+import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
+
+/**
+ * 
+ *
+ * @author Franz-Josef Elmer
+ */
+class FlowLineFeeder implements IPostRegistrationDatasetHandler
+{
+    static final String FLOW_LINE_DROP_BOX_TEMPLATE = "flow-line-drop-box-template";
+    static final String FILE_TYPE = ".srf";
+    
+    private final MessageFormat flowLineDropBoxTemplate;
+    
+    FlowLineFeeder(Properties properties)
+    {
+        flowLineDropBoxTemplate =
+                new MessageFormat(PropertyUtils.getMandatoryProperty(properties,
+                        FLOW_LINE_DROP_BOX_TEMPLATE));
+    }
+
+    public void handle(File originalData, DataSetInformation dataSetInformation)
+    {
+        System.out.println("FlowLineFeeder.handle(): "+originalData+" "+dataSetInformation);
+        String flowcellID = originalData.getName();
+        List<File> files = new ArrayList<File>();
+        findFiles(originalData, files);
+        for (File file : files)
+        {
+            String flowLine = extractFlowLine(file);
+            File dropBox = createDropBoxFile(flowLine);
+            File flowLineDataSet = new File(dropBox, flowcellID + "_" + flowLine);
+            boolean success = flowLineDataSet.mkdir();
+            if (success == false)
+            {
+                throw new EnvironmentFailureException("Couldn't create folder '"
+                        + flowLineDataSet.getAbsolutePath() + "'.");
+            }
+            
+        }
+
+    }
+
+    private File createDropBoxFile(String flowLine)
+    {
+        File dropBox = new File(flowLineDropBoxTemplate.format(flowLine));
+        if (dropBox.exists() == false)
+        {
+            throw new ConfigurationFailureException("Drop box '" + dropBox + "' does not exist.");
+        }
+        if (dropBox.isDirectory() == false)
+        {
+            throw new ConfigurationFailureException("Drop box '" + dropBox + "' is not a directory.");
+        }
+        return dropBox;
+    }
+
+    private String extractFlowLine(File file)
+    {
+        String name = file.getName();
+        int lastIndexOfDot = name.lastIndexOf('.');
+        String nameWithoutType = name;
+        if (lastIndexOfDot >= 0)
+        {
+            nameWithoutType = name.substring(0, lastIndexOfDot);
+        }
+        int lastIndexOfUnderScore = nameWithoutType.lastIndexOf('_');
+        String flowLine = nameWithoutType;
+        if (lastIndexOfUnderScore >= 0)
+        {
+            flowLine = nameWithoutType.substring(lastIndexOfUnderScore + 1);
+        }
+        return flowLine;
+    }
+    
+    private void findFiles(File file, List<File> files)
+    {
+        if (file.isFile() && file.getName().endsWith(FILE_TYPE))
+        {
+            files.add(file);
+        }
+        if (file.isDirectory())
+        {
+            for (File child : file.listFiles())
+            {
+                findFiles(child, files);
+            }
+        }
+    }
+
+    public void undoLastOperation()
+    {
+        System.out.println("FlowLineFeeder.undoLastOperation()");
+    }
+
+}

deep_sequencing_unit/source/java/ch/ethz/bsse/cisd/dsu/dss/StorageProcessor.java

+/*
+ * Copyright 2009 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.bsse.cisd.dsu.dss;
+
+import java.util.Properties;
+
+import ch.systemsx.cisd.etlserver.DelegatingStorageProcessorWithDropbox;
+
+/**
+ * 
+ *
+ * @author Franz-Josef Elmer
+ */
+public class StorageProcessor extends DelegatingStorageProcessorWithDropbox
+{
+    public StorageProcessor(Properties properties)
+    {
+        super(properties, new FlowLineFeeder(properties));
+    }
+
+}