From fdb0115784ed9715852089f7f9ce6635ca753183 Mon Sep 17 00:00:00 2001
From: felmer <felmer>
Date: Tue, 10 Nov 2009 16:03:18 +0000
Subject: [PATCH] LMS-1242 implemented and tested
SVN: 13291
---
.../bsse/cisd/dsu/dss/FlowLaneFeeder.java | 54 ++++++++-
.../bsse/cisd/dsu/dss/FlowLaneFeederTest.java | 114 ++++++++++++++++--
2 files changed, 157 insertions(+), 11 deletions(-)
diff --git a/deep_sequencing_unit/source/java/ch/ethz/bsse/cisd/dsu/dss/FlowLaneFeeder.java b/deep_sequencing_unit/source/java/ch/ethz/bsse/cisd/dsu/dss/FlowLaneFeeder.java
index 8556523cad8..728b2ff92b2 100644
--- a/deep_sequencing_unit/source/java/ch/ethz/bsse/cisd/dsu/dss/FlowLaneFeeder.java
+++ b/deep_sequencing_unit/source/java/ch/ethz/bsse/cisd/dsu/dss/FlowLaneFeeder.java
@@ -19,6 +19,8 @@ package ch.ethz.bsse.cisd.dsu.dss;
import java.io.File;
import java.text.MessageFormat;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
@@ -31,6 +33,7 @@ import org.apache.log4j.Logger;
import ch.systemsx.cisd.common.Constants;
import ch.systemsx.cisd.common.TimingParameters;
+import ch.systemsx.cisd.common.concurrent.ConcurrencyUtilities;
import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
import ch.systemsx.cisd.common.exceptions.UserFailureException;
@@ -41,6 +44,8 @@ import ch.systemsx.cisd.common.filesystem.IFileOperations;
import ch.systemsx.cisd.common.filesystem.IImmutableCopier;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.process.ProcessExecutionHelper;
+import ch.systemsx.cisd.common.process.ProcessResult;
import ch.systemsx.cisd.common.utilities.ExtendedProperties;
import ch.systemsx.cisd.common.utilities.PropertyUtils;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
@@ -71,6 +76,8 @@ class FlowLaneFeeder implements IPostRegistrationDatasetHandler
static final String EXTERNAL_SAMPLE_NAME_KEY = "EXTERNAL_SAMPLE_NAME";
static final String FLOW_LANE_DROP_BOX_TEMPLATE = "flow-lane-drop-box-template";
+
+ static final String SRF_INFO_PATH = "srf-info-path";
static final String ENTITY_SEPARATOR_KEY = "entity-separator";
@@ -95,6 +102,8 @@ class FlowLaneFeeder implements IPostRegistrationDatasetHandler
private final Map<String, File> transferDropBoxes = new HashMap<String, File>();
+ private final String srfInfoPathOrNull;
+
FlowLaneFeeder(Properties properties, IEncapsulatedOpenBISService service)
{
this.service = service;
@@ -102,6 +111,16 @@ class FlowLaneFeeder implements IPostRegistrationDatasetHandler
new MessageFormat(PropertyUtils.getMandatoryProperty(properties,
FLOW_LANE_DROP_BOX_TEMPLATE));
entitySepaparator = properties.getProperty(ENTITY_SEPARATOR_KEY, DEFAULT_ENTITY_SEPARATOR);
+ srfInfoPathOrNull = properties.getProperty(SRF_INFO_PATH);
+ if (srfInfoPathOrNull != null)
+ {
+ File srfInfo = new File(srfInfoPathOrNull);
+ if (srfInfo.isFile() == false)
+ {
+ throw new ConfigurationFailureException("File '" + srfInfo.getAbsolutePath()
+ + "' does not exists or is a folder.");
+ }
+ }
copier = FastRecursiveHardLinkMaker.tryCreate(TimingParameters.getDefaultParameters());
fileOperations = FileOperations.getInstance();
Properties transferDropBoxMapping =
@@ -134,6 +153,7 @@ class FlowLaneFeeder implements IPostRegistrationDatasetHandler
}
for (File file : files)
{
+ List<String> srfInfo = getSRFInfo(file);
String flowLane = extractFlowLane(file);
Sample flowLaneSample = flowLaneSampleMap.get(flowLane);
if (flowLaneSample == null)
@@ -160,7 +180,7 @@ class FlowLaneFeeder implements IPostRegistrationDatasetHandler
}
createHartLink(file, flowLaneDataSet);
createMetaDataFileAndHartLinkInTransferDropBox(flowLaneDataSet, flowLaneSample,
- flowLane);
+ flowLane, srfInfo);
File markerFile = new File(dropBox, Constants.IS_FINISHED_PREFIX + fileName);
createdFiles.add(markerFile);
FileUtilities.writeToFile(markerFile, "");
@@ -173,6 +193,28 @@ class FlowLaneFeeder implements IPostRegistrationDatasetHandler
}
}
+
+ private List<String> getSRFInfo(File file)
+ {
+ if (srfInfoPathOrNull == null)
+ {
+ return Collections.emptyList();
+ }
+ List<String> command = Arrays.asList(srfInfoPathOrNull, "-l1", file.getAbsolutePath());
+ ProcessResult result = ProcessExecutionHelper.run(command, operationLog, operationLog, ConcurrencyUtilities.NO_TIMEOUT, ProcessExecutionHelper.OutputReadingStrategy.ALWAYS, true);
+ List<String> output = result.getOutput();
+ if (result.isOK() == false)
+ {
+ StringBuilder builder = new StringBuilder();
+ for (String outputLine : output)
+ {
+ builder.append("\n").append(outputLine);
+ }
+ throw new UserFailureException("Invalid SRF file '" + file.getAbsolutePath() + "':"
+ + builder);
+ }
+ return output;
+ }
private Map<String, Sample> createFlowLaneSampleMap(DataSetInformation dataSetInformation)
{
@@ -194,7 +236,7 @@ class FlowLaneFeeder implements IPostRegistrationDatasetHandler
}
private void createMetaDataFileAndHartLinkInTransferDropBox(File flowLaneDataSet,
- Sample flowLaneSample, String flowLane)
+ Sample flowLaneSample, String flowLane, List<String> srfInfo)
{
if (flowLaneSample == null)
{
@@ -224,6 +266,14 @@ class FlowLaneFeeder implements IPostRegistrationDatasetHandler
externalSampleName = value;
}
}
+ if (srfInfo.isEmpty() == false)
+ {
+ builder.append("\n==== SRF Info ====\n");
+ for (String line : srfInfo)
+ {
+ builder.append(line).append('\n');
+ }
+ }
String metaFileName =
flowLaneSample.getCode()
+ (externalSampleName == null ? "" : "_" + externalSampleName)
diff --git a/deep_sequencing_unit/sourceTest/java/ch/ethz/bsse/cisd/dsu/dss/FlowLaneFeederTest.java b/deep_sequencing_unit/sourceTest/java/ch/ethz/bsse/cisd/dsu/dss/FlowLaneFeederTest.java
index a3e3baec389..1c52b1278a5 100644
--- a/deep_sequencing_unit/sourceTest/java/ch/ethz/bsse/cisd/dsu/dss/FlowLaneFeederTest.java
+++ b/deep_sequencing_unit/sourceTest/java/ch/ethz/bsse/cisd/dsu/dss/FlowLaneFeederTest.java
@@ -18,7 +18,6 @@ package ch.ethz.bsse.cisd.dsu.dss;
import java.io.File;
import java.io.FilenameFilter;
-import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
@@ -31,7 +30,9 @@ import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
+import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
import ch.systemsx.cisd.base.tests.AbstractFileSystemTestCase;
+import ch.systemsx.cisd.base.utilities.OSUtilities;
import ch.systemsx.cisd.common.Constants;
import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
@@ -86,10 +87,10 @@ public class FlowLaneFeederTest extends AbstractFileSystemTestCase
private File dropBox1;
private File dropBox2;
private File transferDropBox;
+ private File srfInfo;
- @Override
@BeforeMethod
- public void setUp() throws IOException
+ public void beforeMethod() throws Exception
{
super.setUp();
LogInitializer.init();
@@ -102,12 +103,8 @@ public class FlowLaneFeederTest extends AbstractFileSystemTestCase
assertEquals(true, dropBox2.mkdirs());
transferDropBox = new File(workingDirectory, TRANSFER_DROP_BOX);
assertEquals(true, transferDropBox.mkdirs());
- Properties properties = new Properties();
- properties.setProperty(FlowLaneFeeder.FLOW_LANE_DROP_BOX_TEMPLATE, new File(
- workingDirectory, DROP_BOX_PREFIX).getAbsolutePath()
- + "{0}");
- properties.setProperty(FlowLaneFeeder.TRANSFER_PREFIX + AFFILIATION, transferDropBox.getAbsolutePath());
- flowLaneFeeder = new FlowLaneFeeder(properties, service);
+ srfInfo = new File(workingDirectory, "srfInfo");
+ flowLaneFeeder = createFeeder(null);
}
@AfterMethod
@@ -265,6 +262,80 @@ public class FlowLaneFeederTest extends AbstractFileSystemTestCase
context.assertIsSatisfied();
}
+ @Test
+ public void testHappyCaseWithSRFInfo()
+ {
+ flowLaneFeeder = createFeeder("echo option: $1\necho file: $2");
+
+ File flowCell = new File(workingDirectory, SAMPLE_CODE);
+ assertEquals(true, flowCell.mkdir());
+ File logs = new File(flowCell, "logs");
+ assertEquals(true, logs.mkdir());
+ FileUtilities.writeToFile(new File(logs, "basic.log"), "hello log");
+ File srfFolder = new File(flowCell, "SRF");
+ assertEquals(true, srfFolder.mkdir());
+ File originalFlowLane1 = new File(srfFolder, "s_1.srf");
+ FileUtilities.writeToFile(originalFlowLane1, "hello flow lane 1");
+ File originalFlowLane2 = new File(srfFolder, "2.srf");
+ FileUtilities.writeToFile(originalFlowLane2, "hello flow lane 2");
+ prepareLoadFlowCellSample(EXAMPLE_FLOW_CELL_SAMPLE);
+
+ Sample fl1 = createFlowLaneSample(1);
+ Sample fl2 = createFlowLaneSample(2);
+ prepareListFlowLanes(EXAMPLE_FLOW_CELL_SAMPLE, Arrays.asList(fl1, fl2));
+ prepareGetProperties(Arrays.asList(fl1, fl2));
+
+ flowLaneFeeder.handle(flowCell, EXAMPLE_DATA_SET_INFO);
+
+ checkFlowLaneDataSet(originalFlowLane1, "1");
+ checkFlowLaneDataSet(originalFlowLane2, "2");
+
+ File[] transferedFiles = transferDropBox.listFiles();
+ assertEquals(1, transferedFiles.length);
+ String sampleName =
+ SAMPLE_CODE + SampleIdentifier.CONTAINED_SAMPLE_CODE_SEPARARTOR_STRING + "2";
+ assertEquals("G2_" + sampleName, transferedFiles[0].getName());
+ File metaFile = getFile(transferedFiles[0], FlowLaneFeeder.META_DATA_FILE_TYPE);
+ assertEquals(sampleName + "_" + EXTERNAL_SAMPLE_NAME + FlowLaneFeeder.META_DATA_FILE_TYPE,
+ metaFile.getName());
+ List<String> metaData = FileUtilities.loadToStringList(metaFile);
+ String lastLine = metaData.remove(metaData.size() - 1);
+ assertEquals("[Parent\tnull, Code\tfc:2, Contact Person Email\tab@c.de, "
+ + "AFFILIATION\tfmi, EXTERNAL_SAMPLE_NAME\text23, , "
+ + "==== SRF Info ====, option: -l1]", metaData.toString());
+ AssertionUtil.assertContains("file: ", lastLine);
+ assertEquals(8, metaData.size());
+ assertHardLinkOnSameFile(originalFlowLane2, getFile(transferedFiles[0], "2.srf"));
+
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testInvalidSRFFile()
+ {
+ flowLaneFeeder = createFeeder("exit 1");
+ File flowCell = new File(workingDirectory, SAMPLE_CODE);
+ assertEquals(true, flowCell.mkdir());
+ File originalFlowLane1 = new File(flowCell, "s_1.srf");
+ FileUtilities.writeToFile(originalFlowLane1, "hello flow lane 1");
+ prepareLoadFlowCellSample(EXAMPLE_FLOW_CELL_SAMPLE);
+
+ Sample fl1 = createFlowLaneSample(1);
+ prepareListFlowLanes(EXAMPLE_FLOW_CELL_SAMPLE, Arrays.asList(fl1));
+
+ try
+ {
+ flowLaneFeeder.handle(flowCell, EXAMPLE_DATA_SET_INFO);
+ fail("UserFailureException expected");
+ } catch (UserFailureException ex)
+ {
+ AssertionUtil.assertContains("Invalid SRF file", ex.getMessage());
+ AssertionUtil.assertContains("s_1.srf", ex.getMessage());
+ }
+
+ context.assertIsSatisfied();
+ }
+
private File getFile(File folder, final String fileNameExtension)
{
File[] files = folder.listFiles(new FilenameFilter()
@@ -419,4 +490,29 @@ public class FlowLaneFeederTest extends AbstractFileSystemTestCase
return sample;
}
+ private FlowLaneFeeder createFeeder(String srfInfoScriptOrNull)
+ {
+ Properties properties = new Properties();
+ properties.setProperty(FlowLaneFeeder.FLOW_LANE_DROP_BOX_TEMPLATE, new File(
+ workingDirectory, DROP_BOX_PREFIX).getAbsolutePath()
+ + "{0}");
+ properties.setProperty(FlowLaneFeeder.TRANSFER_PREFIX + AFFILIATION, transferDropBox.getAbsolutePath());
+ if (srfInfoScriptOrNull != null)
+ {
+ properties.setProperty(FlowLaneFeeder.SRF_INFO_PATH, srfInfo.getAbsolutePath());
+ FileUtilities.writeToFile(srfInfo, srfInfoScriptOrNull);
+ File chmod = OSUtilities.findExecutable("chmod");
+ assertNotNull(chmod);
+ try
+ {
+ String cmd = chmod.getAbsolutePath() + " +x " + srfInfo.getAbsolutePath();
+ Runtime.getRuntime().exec(cmd).waitFor();
+ } catch (Exception ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+ return new FlowLaneFeeder(properties, service);
+ }
+
}
--
GitLab