diff --git a/sanofi/dist/etc/sanofi-dropbox/dropbox-all-in-one-with-library.py b/sanofi/dist/etc/sanofi-dropbox/dropbox-all-in-one-with-library.py
index 7c9d0bb88377eb0627840cdc8317a3c0bc23b0ec..de8974b372d4b438b8b685271d86363eafa84edc 100644
--- a/sanofi/dist/etc/sanofi-dropbox/dropbox-all-in-one-with-library.py
+++ b/sanofi/dist/etc/sanofi-dropbox/dropbox-all-in-one-with-library.py
@@ -121,31 +121,62 @@ def findDir(incomingFile, dirNameMarker):
return File(incomingPath, file)
return None
+""" Removes trailing empty strings from a list """
+def removeTrailingEmptyElements(list):
+ pos = len(list)
+ while (pos > 0):
+ pos = pos - 1
+ if not list[pos].strip():
+ del list[pos]
+ else:
+ break
+ return list
+
# ======================================
# end generic utility functions
# ======================================
def rollback_service(service, ex):
- plateCode = plate.getCode()
- plateLink = createPlateLink(OPENBIS_URL, plateCode)
+ incomingFileName = incoming.getName()
errorMessage = ex.getMessage()
- sendEmail("openBIS: Data registration failed for %s" % (plateCode), """
+ if not errorMessage:
+ errorMessage = ex.toString()
+
+ if plateCode:
+ plateLink = createPlateLink(OPENBIS_URL, plateCode)
+ sendEmail("openBIS: Data registration failed for %s" % (plateCode), """
Dear openBIS user,
Registering new data for plate %(plateLink)s has failed with error '%(errorMessage)s'.
+ The name of the incoming folder '%(incomingFileName)s' was added to '.faulty_paths'. Please,
+ repair the problem and remove the entry from '.faulty_paths' to retry registration.
+
+ This email has been generated automatically.
+
+ Administrator
+ """ % vars(), False)
+ else:
+ sendEmail("openBIS: Data registration failed for folder '%s'" % (incomingFileName), """
+ Dear user,
+
+ openBIS was unable to understand the name of an incoming folder.
+ Detailed error message was '%(errorMessage)s'.
+
This email has been generated automatically.
Administrator
- """ % vars(), False)
+ """ % vars(), False)
+
def commit_transaction(service, transaction):
- plateCode = plate.getCode()
+ incomingFileName = incoming.getName()
plateLink = createPlateLink(OPENBIS_URL, plateCode)
sendEmail("openBIS: New data registered for %s" % (plateCode), """
Dear openBIS user,
- New data for the plate %(plateLink)s has been registered.
+ New data from folder '%(incomingFileName)s' has been successfully registered in plate %(plateLink)s.
+
This email has been generated automatically.
Have a nice day!
@@ -196,15 +227,15 @@ def findPlateByCode(code):
def parseIncomingDirname(dirName):
"""
Parses the name of an incoming dataset folder from the format
- 'AcquisitionBatch_BarCode_Timestamp' to a tuple (acquisitionBatch, barCode)
+ '<ACQUISITION_BATCH_NAME>_<BAR_CODE>_<TIMESTAMP>' to a tuple (acquisitionBatch, plateCode)
"""
tokens = dirName.split("_")
if len(tokens) < 2:
- raise RuntimeError("Data set directory name does not match the pattern 'AcquisitionBatch_BarCode_Timestamp': " + dirName)
+ raise RuntimeError("Data set directory name does not match the pattern '<ACQUISITION_BATCH_NAME>_<BAR_CODE>_<TIMESTAMP>': " + dirName)
acquisitionBatch = tokens[0]
- barCode = tokens[1].split('.')[0]
- return (acquisitionBatch, barCode)
+ plateCode = tokens[1].split('.')[0]
+ return (acquisitionBatch, plateCode)
def removeDuplicates(list):
dict = {}
@@ -266,32 +297,28 @@ class PlateInitializer:
def getWellCode(self, x, y):
return ConversionUtils.convertToSpreadsheetLocation(Point(x,y))
- def getPlateDimensions(self):
- """
- parses the plate geometry property from the form "384_WELLS_16X24"
- to a tuple of integers (plateHeight, plateWidth)
- """
+ def getPlateGeometryDimensions(self):
plateGeometryString = self.plate.getPropertyValue(ScreeningConstants.PLATE_GEOMETRY)
geometry = Geometry.createFromPlateGeometryString(plateGeometryString)
return (geometry.height, geometry.width)
def validateLibraryDimensions(self, tsvLines):
- (plateHeight, plateWidth) = self.getPlateDimensions()
+ (plateHeight, plateWidth) = self.getPlateGeometryDimensions()
numLines = len(tsvLines)
- if plateHeight != len(tsvLines) :
- raise RuntimeError("The geometry property of plate %s (height=%s)"
- " does not agree with the value of the %s"
- " property in experiment %s (height=%s)." % \
- (self.plateCode, plateHeight, self.LIBRARY_TEMPLATE_PROPNAME, self.experimentId, numLines))
+ if plateHeight < len(tsvLines) :
+ raise RuntimeError("The property %s of experiment '%s' contains %s rows, but the"
+ " geometry of plate '%s' allows a maximum of %s rows. You should either reduce"
+ " the number of rows in the library template or change the plate geometry." %
+ (self.LIBRARY_TEMPLATE_PROPNAME, self.experimentId, numLines, self.plateCode, plateHeight))
for i in range(0, len(tsvLines)):
lineWidth = len(tsvLines[i])
- if plateWidth != lineWidth:
- raise RuntimeError("The geometry property of plate %s (width=%s)"
- " does not agree with the value of the %s"
- " property in experiment %s (line=%s, width=%s)." % \
- (self.plateCode, plateWidth, self.LIBRARY_TEMPLATE_PROPNAME, self.experimentId, i, lineWidth))
+ if plateWidth < lineWidth:
+ raise RuntimeError("The property %s of experiment '%s' contains %s columns in row %s, but the"
+ " geometry of plate '%s' allows a maximum of %s columns. You should either reduce"
+ " the number of columns in the library template or change the plate geometry." %
+ (self.LIBRARY_TEMPLATE_PROPNAME, self.experimentId, lineWidth, (i + 1), self.plateCode, plateHeight))
def parseLibraryTemplate(self):
template = experiment.getPropertyValue(self.LIBRARY_TEMPLATE_PROPNAME)
@@ -299,7 +326,10 @@ class PlateInitializer:
raise RuntimeError("Experiment %s has no library template value in property %s" \
% (self.experimentId, self.LIBRARY_TEMPLATE_PROPNAME))
- tsvLists = [ line.split("\t") for line in template.splitlines() ]
+ lines = template.splitlines()
+ lines = removeTrailingEmptyElements(lines)
+ tsvLists = [ removeTrailingEmptyElements(line.split("\t")) for line in lines ]
+
self.validateLibraryDimensions(tsvLists)
library = {}
@@ -521,10 +551,10 @@ class MyImageDataSetConfig(SimpleImageDataConfig):
if incoming.isDirectory():
transaction = service.transaction(incoming, factory)
- (batchName, barCode) = parseIncomingDirname(incoming.getName())
- plate = findPlateByCode(barCode)
+ (batchName, plateCode) = parseIncomingDirname(incoming.getName())
+ plate = findPlateByCode(plateCode)
if not plate.getExperiment():
- raise RuntimeError("Plate with code '%(barCode)s' is not associated with experiment" % vars())
+ raise RuntimeError("Plate with code '%(plateCode)s' is not associated with experiment" % vars())
experimentId = plate.getExperiment().getExperimentIdentifier()
experiment = transaction.getExperiment(experimentId)
diff --git a/sanofi/sourceTest/java/ch/systemsx/cisd/sanofi/dss/test/SanofiDropboxJythonTest.java b/sanofi/sourceTest/java/ch/systemsx/cisd/sanofi/dss/test/SanofiDropboxJythonTest.java
index c324c81d458a3142d994f5455f5085de1c184bf5..939a1ad02c4d8f7fd158aaee40685bd1d4532826 100644
--- a/sanofi/sourceTest/java/ch/systemsx/cisd/sanofi/dss/test/SanofiDropboxJythonTest.java
+++ b/sanofi/sourceTest/java/ch/systemsx/cisd/sanofi/dss/test/SanofiDropboxJythonTest.java
@@ -17,6 +17,7 @@
package ch.systemsx.cisd.sanofi.dss.test;
import static ch.systemsx.cisd.common.Constants.IS_FINISHED_PREFIX;
+import static ch.systemsx.cisd.common.test.AssertionUtil.assertContains;
import java.io.File;
import java.io.IOException;
@@ -126,37 +127,115 @@ public class SanofiDropboxJythonTest extends AbstractJythonDataSetHandlerTest
private static final String EXPERIMENT_IDENTIFIER = "/SANOFI/PROJECT/EXP";
private static final String PLATE_IDENTIFIER = "/SANOFI/TEST-PLATE";
- @BeforeMethod
+ private RecordingMatcher<ch.systemsx.cisd.openbis.generic.shared.dto.AtomicEntityOperationDetails> atomicatOperationDetails;
+
+ private RecordingMatcher<ListMaterialCriteria> materialCriteria;
+
+ private RecordingMatcher<String> email;
+
@Override
+ @BeforeMethod
public void setUp() throws IOException
{
super.setUp();
+ atomicatOperationDetails =
+ new RecordingMatcher<ch.systemsx.cisd.openbis.generic.shared.dto.AtomicEntityOperationDetails>();
+ materialCriteria = new RecordingMatcher<ListMaterialCriteria>();
+ email = new RecordingMatcher<String>();
}
@Test
- public void testHappyCaseWithLibraryCreation() throws IOException
+ public void testLibraryWider() throws IOException
{
- setUpHomeDataBaseExpectations();
- Properties properties =
- createThreadPropertiesRelativeToScriptsFolder("dropbox-all-in-one-with-library.py");
- createHandler(properties, false, true);
- createData();
+ createDataSetHandler(false, false);
+ final Sample plate =
+ plateWithLibTemplateAndGeometry("1.45\t\tH\n0.12\t0.002\tL", "10_WELLS_1X10");
+ context.checking(new Expectations()
+ {
+ {
- final String libraryTemplate = "1.45\t\tH\n0.12\t0.002\tL";
- final Sample plate = createPlate(libraryTemplate, "6_WELLS_2X3");
- setUpPlateSearchExpectations(plate);
- setUpLibraryTemplateExpectations(plate);
+ SampleIdentifier sampleIdentifier =
+ SampleIdentifierFactory.parse(plate.getIdentifier());
+ one(openBisService).tryGetSampleWithExperiment(sampleIdentifier);
+ will(returnValue(plate));
+
+ one(mailClient).sendMessage(with(any(String.class)), with(email),
+ with(aNull(String.class)), with(any(From.class)),
+ with(equal(EXPERIMENT_RECIPIENTS)));
+ }
+ });
+
+ try
+ {
+ handler.handle(markerFile);
+ fail("Registration should fail with library validation error");
+ } catch (RuntimeException rex)
+ {
+ final String error =
+ "The property LIBRARY_TEMPLATE of experiment '/SANOFI/PROJECT/EXP' contains 2 rows, "
+ + "but the geometry of plate 'TEST-PLATE' allows a maximum of 1 rows. You should either reduce the "
+ + "number of rows in the library template or change the plate geometry.";
+ assertContains(error, rex.getMessage());
+ assertContains(error, email.recordedObject());
+ assertContains(IMAGE_DATA_SET_DIR_NAME, email.recordedObject());
+ }
+
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testLibraryHigher() throws IOException
+ {
+ createDataSetHandler(false, false);
+ final Sample plate =
+ plateWithLibTemplateAndGeometry("1.45\t\tH\n0.12\t0.002\tL", "5_WELLS_5X1");
+ context.checking(new Expectations()
+ {
+ {
+
+ SampleIdentifier sampleIdentifier =
+ SampleIdentifierFactory.parse(plate.getIdentifier());
+ one(openBisService).tryGetSampleWithExperiment(sampleIdentifier);
+ will(returnValue(plate));
+
+ one(mailClient).sendMessage(with(any(String.class)), with(email),
+ with(aNull(String.class)), with(any(From.class)),
+ with(equal(EXPERIMENT_RECIPIENTS)));
+ }
+ });
+
+ try
+ {
+ handler.handle(markerFile);
+ fail("Registration should fail with library validation error");
+ } catch (RuntimeException rex)
+ {
+ final String error =
+ "The property LIBRARY_TEMPLATE of experiment '/SANOFI/PROJECT/EXP' contains 3 "
+ + "columns in row 1, but the geometry of plate 'TEST-PLATE' allows a maximum of "
+ + "5 columns. You should either reduce the number of columns in the library "
+ + "template or change the plate geometry.";
+ assertContains(error, rex.getMessage());
+ assertContains(error, email.recordedObject());
+ assertContains(IMAGE_DATA_SET_DIR_NAME, email.recordedObject());
+ }
+
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testHappyCaseWithLibraryCreation() throws IOException
+ {
+ createDataSetHandler(false, true);
+ final Sample plate =
+ plateWithLibTemplateAndGeometry("1.45\t\tH\n0.12\t0.002\tL", "6_WELLS_10X10");
final MockDataSet<Map<String, Object>> queryResult = new MockDataSet<Map<String, Object>>();
queryResult.add(createQueryResult("A1"));
queryResult.add(createQueryResult("B1"));
queryResult.add(createQueryResult("B2"));
- final RecordingMatcher<ch.systemsx.cisd.openbis.generic.shared.dto.AtomicEntityOperationDetails> atomicatOperationDetails =
- new RecordingMatcher<ch.systemsx.cisd.openbis.generic.shared.dto.AtomicEntityOperationDetails>();
- final RecordingMatcher<ListMaterialCriteria> materialCriteria =
- new RecordingMatcher<ListMaterialCriteria>();
- final RecordingMatcher<String> email = new RecordingMatcher<String>();
+ setDataSetExpectations();
context.checking(new Expectations()
{
{
@@ -171,30 +250,6 @@ public class SanofiDropboxJythonTest extends AbstractJythonDataSetHandlerTest
exactly(5).of(openBisService).createPermId();
will(returnValue("well-permId"));
- one(openBisService).createDataSetCode();
- will(returnValue(IMAGE_DATA_SET_CODE));
-
- one(openBisService).createDataSetCode();
- will(returnValue(OVERLAY_DATA_SET_CODE));
-
- one(openBisService).createDataSetCode();
- will(returnValue(ANALYSIS_DATA_SET_CODE));
-
- one(dataSetValidator).assertValidDataSet(
- IMAGE_DATA_SET_TYPE,
- new File(new File(stagingDirectory, IMAGE_DATA_SET_CODE),
- IMAGE_DATA_SET_DIR_NAME));
-
- one(dataSetValidator).assertValidDataSet(
- OVERLAY_DATA_SET_TYPE,
- new File(new File(stagingDirectory, OVERLAY_DATA_SET_CODE),
- OVERLAYS_DATA_SET_DIR_NAME));
-
- one(dataSetValidator).assertValidDataSet(
- ANALYSIS_DATA_SET_TYPE,
- new File(new File(stagingDirectory, ANALYSIS_DATA_SET_CODE),
- ANALYSIS_DATA_SET_FILE_NAME));
-
SampleIdentifier sampleIdentifier =
SampleIdentifierFactory.parse(plate.getIdentifier());
exactly(4).of(openBisService).tryGetSampleWithExperiment(sampleIdentifier);
@@ -247,10 +302,11 @@ public class SanofiDropboxJythonTest extends AbstractJythonDataSetHandlerTest
assertEquals(ANALYSIS_DATA_SET_CODE, analysisDataSet.getCode());
assertEquals(ANALYSIS_DATA_SET_TYPE, analysisDataSet.getDataSetType());
+
AssertionUtil
.assertContains(
- "New data for the plate <a href='https://bwl27.sanofi-aventis.com:8443/openbis#entity=SAMPLE"
- + "&sample_type=PLATE&action=SEARCH&code=TEST-PLATE'>TEST-PLATE</a> has been registered.",
+ "New data from folder 'batchNr_plateCode.variant_2011.07.05' has been successfully registered in plate "
+ + "<a href='https://bwl27.sanofi-aventis.com:8443/openbis#entity=SAMPLE&sample_type=PLATE&action=SEARCH&code=plateCode'>plateCode</a>",
email.recordedObject());
context.assertIsSatisfied();
}
@@ -331,6 +387,57 @@ public class SanofiDropboxJythonTest extends AbstractJythonDataSetHandlerTest
}
+ public Sample plateWithLibTemplateAndGeometry(String libraryTemplate, String plateGeometry)
+ throws IOException
+ {
+ Sample plate = createPlate(libraryTemplate, plateGeometry);
+ setUpPlateSearchExpectations(plate);
+ setUpLibraryTemplateExpectations(plate);
+ return plate;
+ }
+
+ private void createDataSetHandler(boolean shouldRegistrationFail, boolean rethrowExceptions)
+ throws IOException
+ {
+ setUpHomeDataBaseExpectations();
+ Properties properties =
+ createThreadPropertiesRelativeToScriptsFolder("dropbox-all-in-one-with-library.py");
+ createHandler(properties, shouldRegistrationFail, rethrowExceptions);
+ createData();
+ }
+
+ private void setDataSetExpectations()
+ {
+ context.checking(new Expectations()
+ {
+ {
+ one(openBisService).createDataSetCode();
+ will(returnValue(IMAGE_DATA_SET_CODE));
+
+ one(openBisService).createDataSetCode();
+ will(returnValue(OVERLAY_DATA_SET_CODE));
+
+ one(openBisService).createDataSetCode();
+ will(returnValue(ANALYSIS_DATA_SET_CODE));
+
+ one(dataSetValidator).assertValidDataSet(
+ IMAGE_DATA_SET_TYPE,
+ new File(new File(stagingDirectory, IMAGE_DATA_SET_CODE),
+ IMAGE_DATA_SET_DIR_NAME));
+
+ one(dataSetValidator).assertValidDataSet(
+ OVERLAY_DATA_SET_TYPE,
+ new File(new File(stagingDirectory, OVERLAY_DATA_SET_CODE),
+ OVERLAYS_DATA_SET_DIR_NAME));
+
+ one(dataSetValidator).assertValidDataSet(
+ ANALYSIS_DATA_SET_TYPE,
+ new File(new File(stagingDirectory, ANALYSIS_DATA_SET_CODE),
+ ANALYSIS_DATA_SET_FILE_NAME));
+ }
+ });
+ }
+
private void setUpPlateSearchExpectations(final Sample plate)
{
context.checking(new Expectations()