initial check in for dsu files

SVN: 12736

openbis_all/source/bash-dsu/RTA_check_on_missing_images.sh

+#!/bin/bash
+# Checks on missing pictures in the 'Images' folder of the new Illumina Pipeline 1.4
+# July 2009
+# Author: Manuel Kohler
+# Needs one parameter to specify the number of cycles which may vary
+
+export NUMBER_OF_CYCLES=$1
+export NUMBER_OF_LANES=8
+export NUMBER_OF_TILES_PER_LANE=100
+export NUMBER_OF_IMAGES_PER_TILE=2
+export IMAGES_PER_CYCLE=$[ ${NUMBER_OF_TILES_PER_LANE}*${NUMBER_OF_IMAGES_PER_TILE} ]
+#----------------------------------------------
+export RUN_BASE=/array0/Incoming/Runs
+export LATEST_FOLDER=`ls -1tr $RUN_BASE | tail -1`  
+#export IMAGE_PATH=$RUN_BASE/$LATEST_FOLDER/Images
+export IMAGE_PATH=$2
+#----------------------------------------------
+export MAILX="/bin/mail"
+export MAIL_LIST="manuel.kohler@bsse.ethz.ch"
+#----------------------------------------------
+export BOX=`uname -n`
+export PRG=`basename $0`
+export USAGE="Usage: ${PRG} <Number_of_cycles> <Image_Path>"
+export DAY=`date |cut -c1-3`
+export BUILDSTAMP=`date '+%Y.%m.%d_%H_%M'`
+export MISSING_IMAGES=$IMAGE_PATH/missing_images_${BUILDSTAMP}.txt
+#----------------------------------------------
+if [ -z "${NUMBER_OF_CYCLES}"  -o -z "${IMAGE_PATH}" ]
+then
+   echo "${USAGE}"
+   exit 1
+fi
+#----------------------------------------------
+echo -e "Checking $IMAGE_PATH as Image Folders"
+echo -e "Looking for $IMAGES_PER_CYCLE Images in $NUMBER_OF_CYCLES cycle folders..."
+
+# Create a reference file
+for j in cnf cif; do
+ for (( k = 1; k <= $NUMBER_OF_TILES_PER_LANE; k++ )); do
+  echo \_$k.$j >> $IMAGE_PATH/image_reference.txt;
+ done
+done
+#----------------------------------------------
+for i in $IMAGE_PATH/L00*; do 
+ for c in $i/*; do
+  NUMBER=`ls -1 $c | wc -l`;
+  if [ $NUMBER -ne ${IMAGES_PER_CYCLE} ]; 
+   then
+    echo $c 
+    ls -1 $c > $IMAGE_PATH/incomplete_files.txt
+#    cat $IMAGE_PATH/incomplete_files.txt | cut -d "_" -f3- | sort -n > $IMAGE_PATH/incomplete_files-s.txt
+    diff $IMAGE_PATH/incomplete_files.txt  $IMAGE_PATH/image_reference.txt; 
+ fi
+ done;
+done > $IMAGE_PATH/missing.txt 
+#----------------------------------------------
+sed '/^[0-9]/d' $IMAGE_PATH/missing.txt > $IMAGE_PATH/missing_tmp.txt
+sed 's/^> //' $IMAGE_PATH/missing_tmp.txt > $MISSING_IMAGES
+#----------------------------------------------
+#rm  $IMAGE_PATH/image_reference.txt $IMAGE_PATH/incomplete_files.txt
+#----------------------------------------------
+$MAILX -s "Missing Images in last GA run on $BOX discovered! ( ${MISSING_IMAGES} )" $MAIL_LIST < ${MISSING_IMAGES}
+exit 0;

openbis_all/source/bash-dsu/add_p.sh

+#!/bin/bash
+# Simply adds a p to the int and nse files  
+
+for i in `ls *int*`; do 
+  #j=`echo $i |  cut -c 1-16` 
+  mv $i $i.p
+  #echo -e "Moving $i to $j";
+done

openbis_all/source/bash-dsu/check_on_missing_images.sh

+#!/bin/bash
+# Checks on missing pictures in the 'Images' folder of the new Illumina Pipeline 1.4
+# July 2009
+# Author: Manuel Kohler
+# Needs one parameter to specify the number of cycles which may vary
+
+export NUMBER_OF_CYCLES=$1
+export NUMBER_OF_LANES=8
+export NUMBER_OF_TILES_PER_LANE=100
+export NUMBER_OF_IMAGES_PER_TILE=4
+export IMAGES_PER_CYCLE=$[ ${NUMBER_OF_TILES_PER_LANE}*${NUMBER_OF_IMAGES_PER_TILE} ]
+#----------------------------------------------
+export RUN_BASE=/array0/Runs
+export LATEST_FOLDER=`ls -1tr $RUN_BASE | tail -1`  
+export IMAGE_PATH=$RUN_BASE/$LATEST_FOLDER/Images
+#----------------------------------------------
+export MAILX="/bin/mailx"
+export MAIL_LIST="manuel.kohler@bsse.ethz.ch"
+#----------------------------------------------
+export BOX=`uname -n`
+export PRG=`basename $0`
+export USAGE="Usage: ${PRG} <Number_of_cycles>"
+export DAY=`date |cut -c1-3`
+export BUILDSTAMP=`date '+%Y.%m.%d_%H_%M'`
+export MISSING_IMAGES=$IMAGE_PATH/missing_images_${BUILDSTAMP}.txt
+#----------------------------------------------
+if [ -z "${NUMBER_OF_CYCLES}" ]
+then
+   echo "${USAGE}"
+   exit 1
+fi
+#----------------------------------------------
+echo -e "Checking $IMAGE_PATH as Image Folders"
+echo -e "Looking for $IMAGES_PER_CYCLE Images in $NUMBER_OF_CYCLES cycle folders..."
+
+# Create a reference file
+for (( k = 1; k <= $NUMBER_OF_TILES_PER_LANE; k++ )); do
+ for j in a c g t; do
+  echo $k\_$j.tif >> $IMAGE_PATH/image_reference.txt;
+ done
+done
+#----------------------------------------------
+for i in $IMAGE_PATH/L00*; do 
+ for c in $i/*; do
+  NUMBER=`ls -1 $c | wc -l`;
+  if [ $NUMBER -ne ${IMAGES_PER_CYCLE} ]; 
+   then
+    echo $c 
+    ls -1 $c > $IMAGE_PATH/incomplete_files.txt
+    cat $IMAGE_PATH/incomplete_files.txt | cut -d "_" -f3- | sort -n > $IMAGE_PATH/incomplete_files-s.txt
+    diff $IMAGE_PATH/incomplete_files-s.txt  $IMAGE_PATH/image_reference.txt; 
+ fi
+ done;
+done > $IMAGE_PATH/missing.txt 
+#----------------------------------------------
+sed '/^[0-9]/d' $IMAGE_PATH/missing.txt > $IMAGE_PATH/missing_tmp.txt
+sed 's/^> //' $IMAGE_PATH/missing_tmp.txt > $MISSING_IMAGES
+#----------------------------------------------
+rm  $IMAGE_PATH/image_reference.txt $IMAGE_PATH/incomplete_files.txt $IMAGE_PATH/incomplete_files-s.txt $IMAGE_PATH/missing.txt $IMAGE_PATH/missing_tmp.txt
+#----------------------------------------------
+$MAILX -s "Missing Images in last GA run on $BOX discovered! ( ${MISSING_IMAGES} )" $MAIL_LIST < ${MISSING_IMAGES}
+exit 0;

openbis_all/source/bash-dsu/cif2txt.sh

+# Manuel Kohler 2009, CISD, ETH Zürich
+# wrapper for the cifToTxt binary provided by Illumina
+
+#Usage: cifToTxt [options]
+
+#Command line options:
+#  -h [ --help ]                 produce help message and exit
+#  -I [ --IPAR ]                 generate IPAR data
+#  -N [ --noise ]                convert noise instead of intensities
+#  -l [ --lane ] arg             identifier of the lane (1, 2, 3, ..., 8)
+#  -t [ --tile ] arg             identifier of the tile (1, 2, 3, ..., 110)
+#  -r [ --repeat ] arg (=1)      identifier of the repeat (1, 2, ...)
+#  -f [ --first-cycle ] arg (=1) first cycle to use (1-based)
+#  -n [ --number-of-cycles ] arg number of cycles to convert
+#  -i [ --input-dir ] arg (=.)   directory where the CIF directories are located
+#  -o [ --output-dir ] arg (=.)  directory where the output file should be 
+#                                written
+#  -c [ --compression ] arg      where arg=bzip2|gzip|none. The data compression
+#                                format used for the output file.
+#!/bin/bash
+
+INTENSITY_FOLDER=$1
+CYCLES=$2
+NUMBER_OF_LANES=8
+NUMBER_OF_TILES=100
+#INT_NSE_DIR=$1/int_nse
+PRG=`basename $0`
+USAGE="Usage: ${PRG} <Path_to_Intensity_Folder> <Number_of_Cycles> \n\nEXAMPLE: ${PRG} /array0/Runs/090720_42HUDAAXX/Data/Intensities/ 38" 
+
+if [ -z "${INTENSITY_FOLDER}" -o -z "${CYCLES}" ]
+then
+   echo -e "${USAGE}"
+   exit 1
+fi
+
+
+# INT_NSE_DIR  there?
+[ -d $INT_NSE_DIRR ] || mkdir -p $INT_NSE_DIR
+
+for (( l = 1; l <= $NUMBER_OF_LANES; l++ )); do
+ for (( t = 1; t <= $NUMBER_OF_TILES; t++ )); do
+  echo Lane $l Tile $t;
+  # Convert Signal cif to int
+  /dsf/GAPipeline/bin/cifToTxt -l $l -t $t -n $CYCLES -i $INTENSITY_FOLDER -o $INTENSITY_FOLDER -c none 
+  # Convert Noise cnf to nse (additional -N)
+  /dsf/GAPipeline/bin/cifToTxt -N -l $l -t $t -n $CYCLES -i $INTENSITY_FOLDER -o $INTENSITY_FOLDER -c none 
+ done
+done

openbis_all/source/bash-dsu/convert_template.sh

+#!/bin/bash
+Eland2Wig 1 drosmel_r5 Kc0.5 36
+Eland2Wig 2 musmus_ncbi37 TNK27me3 145
+Eland2Wig 3 musmus_ncbi37 TNK27me3 145
+Eland2Wig 4 musmus_ncbi37 ESCTCF 380
+Eland2Wig 8 homsap_ncbi36.50 HEK7b 36

openbis_all/source/bash-dsu/createmd5sums

+#!/bin/bash
+cd Data
+echo Checksumming Lane 1
+find . -type f | grep -Ev '/s_[2345678][^/]*$' | xargs md5sum >s_1.md5 
+echo Checksumming Lane 2
+find . -type f | grep -Ev '/s_[1345678][^/]*$' | xargs md5sum >s_2.md5 
+echo Checksumming Lane 3
+find . -type f | grep -Ev '/s_[1245678][^/]*$' | xargs md5sum >s_3.md5 
+echo Checksumming Lane 4
+find . -type f | grep -Ev '/s_[1235678][^/]*$' | xargs md5sum >s_4.md5 
+echo Checksumming Lane 5
+find . -type f | grep -Ev '/s_[1234678][^/]*$' | xargs md5sum >s_5.md5 
+echo Checksumming Lane 6
+find . -type f | grep -Ev '/s_[1234578][^/]*$' | xargs md5sum >s_6.md5 
+echo Checksumming Lane 7
+find . -type f | grep -Ev '/s_[1234568][^/]*$' | xargs md5sum >s_7.md5 
+echo Checksumming Lane 8
+find . -type f | grep -Ev '/s_[1234567][^/]*$' | xargs md5sum >s_8.md5 
\ No newline at end of file

openbis_all/source/bash-dsu/do-sync.sh

+#!/bin/bash
+
+ADD_ARG=$1
+
+STOR=bs-ssvr01
+RUN=$(basename $(pwd))
+cd /array0/Runs/${RUN} || exit 1
+
+DEST_BSSE="dsf/Samples"
+DEST_FMI="dsf-fmi/Samples"
+DEST_BZ="dsf-biozentrum"
+DEST_UB="dsf-unibs"
+DEST_C1="dsf-customer1"
+DEST_C2="dsf-customer2"
+DEST_C3="dsf-customer3"
+
+DEST1="${STOR}::${DEST_BZ}"
+DEST2="${STOR}::${DEST_BZ}"
+DEST3="${STOR}::${DEST_BZ}"
+DEST4="${STOR}::${DEST_BZ}"
+DEST5="${STOR}::${DEST_BSSE}"
+DEST6="${STOR}::${DEST_BZ}"
+DEST7="${STOR}::${DEST_BZ}"
+DEST8="${STOR}::${DEST_FMI}"
+
+# Lane
+LANE1="UB-DE-257"  
+LANE2="UB-DE-258"  
+LANE3="UB-DE-259"  
+LANE4="UB-DE-260"  
+LANE6="UB-DE-261"  
+LANE7="UB-DE-262"  
+LANE8="FMI-HG-263"  
+# Lane 5 BSSE Control Lane
+LANE5="BSSE-IN-134"
+
+rsync -xav --delete --delete-excluded --exclude "*-tmp-*" --exclude "Images" --exclude "*~" --exclude "s_[2345678]*" Data/ ${DEST1}/${LANE1}-${RUN}-1 --stats $ADD_ARG
+rsync -xav --delete --delete-excluded --exclude "*-tmp-*" --exclude "Images" --exclude "*~" --exclude "s_[1345678]*" Data/ ${DEST2}/${LANE2}-${RUN}-2 --stats $ADD_ARG
+rsync -xav --delete --delete-excluded --exclude "*-tmp-*" --exclude "Images" --exclude "*~" --exclude "s_[1245678]*" Data/ ${DEST3}/${LANE3}-${RUN}-3 --stats $ADD_ARG
+rsync -xav --delete --delete-excluded --exclude "*-tmp-*" --exclude "Images" --exclude "*~" --exclude "s_[1235678]*" Data/ ${DEST4}/${LANE4}-${RUN}-4 --stats $ADD_ARG
+rsync -xav --delete --delete-excluded --exclude "*-tmp-*" --exclude "Images" --exclude "*~" --exclude "s_[1234678]*" Data/ ${DEST5}/${LANE5}-${RUN}-5 --stats $ADD_ARG
+rsync -xav --delete --delete-excluded --exclude "*-tmp-*" --exclude "Images" --exclude "*~" --exclude "s_[1234578]*" Data/ ${DEST6}/${LANE6}-${RUN}-6 --stats $ADD_ARG
+rsync -xav --delete --delete-excluded --exclude "*-tmp-*" --exclude "Images" --exclude "*~" --exclude "s_[1234568]*" Data/ ${DEST7}/${LANE7}-${RUN}-7 --stats $ADD_ARG
+rsync -xav --delete --delete-excluded --exclude "*-tmp-*" --exclude "Images" --exclude "*~" --exclude "s_[1234567]*" Data/ ${DEST8}/${LANE8}-${RUN}-8 --stats $ADD_ARG
+# dirvish-timestemp

openbis_all/source/bash-dsu/enter

+if [ -f /etc/bashrc ]; then
+	. /etc/bashrc
+fi
+
+export PATH=/dsf/:/dsf/Commands/Original/Goat/:/usr/lib64/qt-3.3/bin:/usr/kerberos/bin:/usr/local/bin:/bin:/usr/bin:/dsf/eland2wig
+export HOME=/dsf/Analysis/Runs/
+export COLORS=/etc/DIR_COLORS
+export PROMPT_COMMAND='echo -ne "\033]0;${USER}@${HOSTNAME%%.*}:${PWD/#$HOME/~}"; echo -ne "\007"'
+export PS1='\w \$ '
+alias pstree='pstree -A'
+alias jed='emacs '
+alias emasc='emacs '

openbis_all/source/bash-dsu/getimages

+#!/bin/bash
+mkdir Data/DemoImages
+cp -xav Images/L*/C2.1/s_*_1_*.tif Data/DemoImages
+cp -xav Images/L*/C2.1/s_*_20_*.tif Data/DemoImages
+cp -xav Images/L*/C2.1/s_*_40_*.tif Data/DemoImages
+cp -xav Images/L*/C2.1/s_*_60_*.tif Data/DemoImages
+cp -xav Images/L*/C2.1/s_*_80_*.tif Data/DemoImages
+cp -xav Images/L*/C2.1/s_*_100_*.tif Data/DemoImages

openbis_all/source/bash-dsu/getmetainfo

+#!/bin/bash
+a=`pwd`
+b=`basename $a`
+for c in 1 2 3 4 5 6 7 8; do 
+mysql -E -u bsse --password=theycconnection -h yellowcouch.org -D BsseSampleTracking >Data/s_$c.txt <<EOF
+SELECT * FROM PreparedView
+WHERE flow_id='$b' AND lane_id=$c
+EOF
+mysql -X -u bsse --password=theycconnection -h yellowcouch.org -D BsseSampleTracking >Data/s_$c.xml <<EOF
+SELECT * FROM PreparedView
+WHERE flow_id='$b' AND lane_id=$c
+EOF
+done

openbis_all/source/bash-dsu/goat_with_missing_tiles.sh

+#!/bin/bash
+
+
+RUN_DIR=/dsf/Analysis/Runs/090701_426LKAAXX/ 
+MISSING_TILES=--tiles=s_1_000[1-9],s_1_00[1-5][0-9],s_1_006[0-1],s_1_006[3-9],s_1_00[7-9][0-9],s_1_0100,s_2,s_3_000[1-9],s_3_00[1-4][0-9],s_3_0050,s_3_005[3-9],s_3_00[6-7][0-9],s_3_008[0-8],s_3_009[0-9],s_3_0100,s_4,s_5_000[1-9],s_5_00[1-3][0-9],s_5_0040,s_5_004[2-9],s_5_005[0-9],s_5_006[0-2],s_5_0064,s_5_006[6-7],s_5_0069,s_5_00[7-9][0-9],s_5_0100,s_6_000[1-9],s_6_00[1-2][0-9],s_6_003[0-8],s_6_00[4-9][0-9],s_6_0100,s_7_000[1-9],s_7_00[1-4][0-9],s_7_005[0-2],s_7_005[4-5],s_7_005[7-9],s_7_00[6-9][0-9],s_7_0100,s_8_000[1-9],s_8_00[1-6][0-9],s_8_007[0-8],s_8_0081,s_8_008[4-9],s_8_009[0-9],s_8_0100
+GOAT=/dsf/GAPipeline-1.4.0/bin/goat_pipeline.py
+#CONTROL_LANE=--control-lane=5
+
+cd $RUN_DIR
+echo Started: `date`
+$GOAT --control-lane=5 $MISSING_TILES ./ --make
+
+echo Finished: `date`

openbis_all/source/bash-dsu/lanes2srf

+#!/bin/bash
+SRF_DIR=Srf
+[ -d $SRF_DIR ] || mkdir -p $SRF_DIR ]
+for a in 1 2 3 4 5 6 7 8; do
+  echo Converting Lane $a to SRF
+  time /dsf/illumina2srf -I -b -o $SRF_DIR/s_${a}.srf s_${a}_*_qseq.txt ;
+done

openbis_all/source/bash-dsu/lanes2srf-pre1.3

+#!/bin/bash
+mkdir Srf
+for a in 1 2 3 4 5 6 7 8; do
+  echo Converting Lane $a to SRF
+  time illumina2srf -r -p -C 0.6 -o Srf/s_${a}.srf -N %r:%l:%t: -n %x:%y s_${a}_*_seq.txt ;
+done

openbis_all/source/bash-dsu/lanes2srf_RTA

+#!/bin/bash
+
+BUSTARD=$1
+SRF_DIR=Srf_RTA
+
+[ -d $SRF_DIR ] || mkdir $SRF_DIR
+for a in 1 2 3 4 5 6 7 8; do
+  echo Converting Lane $a to SRF
+  time /dsf/illumina2srf -b -o $SRF_DIR/s_${a}_RTA.srf $BUSTARD/s_${a}_*_qseq.txt ;
+done

openbis_all/source/bash-dsu/rm_p.sh

+#!/bin/bash
+# Simply removes the p from the int and nse files  
+
+for i in `ls *.p`; do 
+  j=`echo $i |  cut -c 1-16` 
+  mv $i $j
+  echo -e "Moving $i to $j";
+done

openbis_all/source/bash-dsu/sendtostorage

+#!/bin/bash
+getmetainfo
+a=`pwd`
+b=`basename $a`
+
+echo $b
+
+mysql -N -r -u bsse --password=theycconnection -h yellowcouch.org -D BsseSampleTracking >send-tmp-cmd <<EOF
+SELECT CONCAT(
+'chmod -R ugo-rwx,u=rwX,',permission,' Data\n',
+'chgrp -R ',\`group\`,' Data\n',
+'rsync -xav --delete --delete-excluded --exclude "*-tmp-*" --exclude "Images" --exclude "*~" --exclude "s_',mask,'*" Data/ ',hostdirectory,E.institute,'-',E.pi,'-',sample_id,'-',flow_id,'-',lane_id
+) as command
+FROM FlowCells F
+JOIN Id2External I ON (F.sample_id=I.id)
+JOIN External E ON (E.external_id=I.external_id)
+JOIN Cust2Target C
+JOIN ExcludeMask USING (lane_id)
+WHERE C.institute=E.institute and C.pi=E.pi and flow_id='$b' and lane_id='$1'
+
+UNION
+
+SELECT CONCAT(
+'chmod -R ugo-rwx,u=rwX,',permission,' Data\n',
+'chgrp -R ',\`group\`,' Data\n',
+'rsync -xav --delete --delete-excluded --exclude "*-tmp-*" --exclude "Images" --exclude "*~" --exclude "s_',mask,'*" Data/ ',hostdirectory,E.institute,'-',E.pi,'-',sample_id,'-',flow_id,'-',lane_id
+) as command
+FROM FlowCells F
+JOIN Id2Others O ON (F.sample_id=O.id)
+JOIN Cust2Target C
+JOIN ExcludeMask USING (lane_id)
+JOIN Id2External I2 ON (F.sample_id=I2.id)
+JOIN External E ON (E.external_id=I2.external_id)
+WHERE C.institute=O.institute and C.pi=O.pi and flow_id='$b' and lane_id='$1'
+
+EOF
+
+chmod 700 send-tmp-cmd
+cat send-tmp-cmd
+./send-tmp-cmd
+rm send-tmp-cmd

openbis_all/source/bash-dsu/uploadqr

+#!/bin/bash
+rsync -xavz --delete *.png *.htm wernersa@yellowcouch.org:/home/analysis/public_html/dsu/qr/$1