SE-302 make colors of each channel deterministic, allow to skip validation of...

SE-302 make colors of each channel deterministic, allow to skip validation of the plate name in the image file name (hidden option for demo purposes)

SVN: 17845

screening/source/java/ch/systemsx/cisd/openbis/dss/etl/AbstractHCSImageFileExtractor.java

@@ -19,11 +19,9 @@ package ch.systemsx.cisd.openbis.dss.etl;
 import java.io.File;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Properties;
-import java.util.Set;
 
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.lang.StringUtils;
@@ -70,7 +68,7 @@ abstract public class AbstractHCSImageFileExtractor implements IHCSImageFileExtr
             Location plateLocation, Location wellLocation, Float timepointOrNull,
             String imageRelativePath);
 
-    abstract protected Set<Channel> getAllChannels();
+    abstract protected List<Channel> getAllChannels();
 
     /**
      * Extracts the plate location from argument. Returns <code>null</code> if the operation fails.
@@ -146,8 +144,8 @@ abstract public class AbstractHCSImageFileExtractor implements IHCSImageFileExtr
         }
     }
 
-    protected static final Logger operationLog =
-            LogFactory.getLogger(LogCategory.OPERATION, AbstractHCSImageFileExtractor.class);
+    protected static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
+            AbstractHCSImageFileExtractor.class);
 
     protected static final String IMAGE_FILE_NOT_STANDARDIZABLE =
             "Image file '%s' could not be standardized given following tokens: %s.";
@@ -179,25 +177,25 @@ abstract public class AbstractHCSImageFileExtractor implements IHCSImageFileExtr
      * @return <code>null</code> if the argument could not be splitted into tokens.
      */
     protected final static ImageFileInfo tryExtractDefaultImageInfo(File imageFile,
-            SampleIdentifier datasetSample)
+            SampleIdentifier datasetSample, boolean shouldValidatePlateName)
     {
         final String baseName = FilenameUtils.getBaseName(imageFile.getPath());
         final String[] tokens = StringUtils.split(baseName, TOKEN_SEPARATOR);
         if (tokens == null || tokens.length < 4)
         {
-            if (operationLog.isDebugEnabled())
+            if (operationLog.isInfoEnabled())
             {
-                operationLog.debug(String.format(IMAGE_FILE_NOT_ENOUGH_ENTITIES, imageFile));
+                operationLog.info(String.format(IMAGE_FILE_NOT_ENOUGH_ENTITIES, imageFile));
             }
             return null;
         }
         final String sampleCode = tokens[tokens.length - 4];
-        if (sampleCode != null
+        if (shouldValidatePlateName && sampleCode != null
                 && sampleCode.equalsIgnoreCase(datasetSample.getSampleCode()) == false)
         {
-            if (operationLog.isDebugEnabled())
+            if (operationLog.isInfoEnabled())
             {
-                operationLog.debug(String.format(IMAGE_FILE_BELONGS_TO_WRONG_SAMPLE, imageFile,
+                operationLog.info(String.format(IMAGE_FILE_BELONGS_TO_WRONG_SAMPLE, imageFile,
                         datasetSample, sampleCode));
             }
             return null;
@@ -250,24 +248,24 @@ abstract public class AbstractHCSImageFileExtractor implements IHCSImageFileExtr
                 }
             } else
             {
-                if (operationLog.isDebugEnabled())
+                if (operationLog.isInfoEnabled())
                 {
-                    operationLog.debug(String.format(IMAGE_FILE_NOT_STANDARDIZABLE, imageFile,
+                    operationLog.info(String.format(IMAGE_FILE_NOT_STANDARDIZABLE, imageFile,
                             imageInfo));
                 }
                 invalidFiles.add(imageFile);
             }
         }
-        return new HCSImageFileExtractionResult(acquiredImages, Collections
-                .unmodifiableList(invalidFiles), getAllChannels());
+        return new HCSImageFileExtractionResult(acquiredImages,
+                Collections.unmodifiableList(invalidFiles), getAllChannels());
 
     }
 
     private String getRelativeImagePath(File incomingDataSetDirectory, final File imageFile)
     {
         String imageRelativePath =
-                FileUtilities.getRelativeFile(incomingDataSetDirectory, new File(imageFile
-                        .getPath()));
+                FileUtilities.getRelativeFile(incomingDataSetDirectory,
+                        new File(imageFile.getPath()));
         assert imageRelativePath != null : "Image relative path should not be null.";
         return imageRelativePath;
     }
@@ -290,9 +288,9 @@ abstract public class AbstractHCSImageFileExtractor implements IHCSImageFileExtr
         return components;
     }
 
-    protected final static Set<Channel> createChannels(List<ChannelDescription> channelDescriptions)
+    protected final static List<Channel> createChannels(List<ChannelDescription> channelDescriptions)
     {
-        Set<Channel> channels = new HashSet<Channel>();
+        List<Channel> channels = new ArrayList<Channel>();
         for (ChannelDescription channelDescription : channelDescriptions)
         {
             channels.add(new Channel(channelDescription.getCode(), null, null, channelDescription

screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSDatasetUploader.java

@@ -20,8 +20,8 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.Map.Entry;
+import java.util.Set;
 
 import ch.systemsx.cisd.openbis.dss.etl.ScreeningContainerDatasetInfoHelper.ExperimentWithChannelsAndContainer;
 import ch.systemsx.cisd.openbis.dss.etl.dataaccess.IImagingQueryDAO;
@@ -36,7 +36,7 @@ import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgSp
 public class HCSDatasetUploader
 {
     public static void upload(IImagingQueryDAO dao, ImageDatasetInfo info,
-            List<AcquiredPlateImage> images, Set<HCSImageFileExtractionResult.Channel> channels)
+            List<AcquiredPlateImage> images, List<HCSImageFileExtractionResult.Channel> channels)
     {
         new HCSDatasetUploader(dao).upload(info, images, channels);
     }
@@ -49,7 +49,7 @@ public class HCSDatasetUploader
     }
 
     private void upload(ImageDatasetInfo info, List<AcquiredPlateImage> images,
-            Set<HCSImageFileExtractionResult.Channel> channels)
+            List<HCSImageFileExtractionResult.Channel> channels)
     {
         ExperimentWithChannelsAndContainer basicStruct =
                 ScreeningContainerDatasetInfoHelper.getOrCreateExperimentWithChannelsAndContainer(
@@ -134,8 +134,8 @@ public class HCSDatasetUploader
 
     private static AcquiredImageInStack makeAcquiredImageInStack(AcquiredPlateImage image)
     {
-        return new AcquiredImageInStack(image.getChannelCode(), image.getImageReference(), image
-                .getThumbnailFilePathOrNull());
+        return new AcquiredImageInStack(image.getChannelCode(), image.getImageReference(),
+                image.getThumbnailFilePathOrNull());
     }
 
     private ImgChannelStackDTO makeStackDtoWithouId(AcquiredPlateImage image, Long[][] spotIds,
@@ -249,8 +249,8 @@ public class HCSDatasetUploader
     {
         ImgImageDTO dto =
                 new ImgImageDTO(dao.createImageId(), imageReferenceOrNull.getRelativeImagePath(),
-                        imageReferenceOrNull.tryGetPage(), imageReferenceOrNull
-                                .tryGetColorComponent());
+                        imageReferenceOrNull.tryGetPage(),
+                        imageReferenceOrNull.tryGetColorComponent());
         return dto;
     }
 
@@ -262,8 +262,8 @@ public class HCSDatasetUploader
     {
         List<ImgSpotDTO> oldSpots = dao.listSpots(contId);
         List<ImgSpotDTO> newSpots =
-                createNewSpots(contId, images, oldSpots, info.getContainerRows(), info
-                        .getContainerColumns(), info.getContainerPermId());
+                createNewSpots(contId, images, oldSpots, info.getContainerRows(),
+                        info.getContainerColumns(), info.getContainerPermId());
         newSpots.addAll(oldSpots);
         return makeTechIdMatrix(newSpots, info.getContainerRows(), info.getContainerColumns());
     }

screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSImageFileExtractionResult.java

@@ -17,7 +17,6 @@ package ch.systemsx.cisd.openbis.dss.etl;
 
 import java.io.File;
 import java.util.List;
-import java.util.Set;
 
 /**
  * Class which contains the image extraction process results.
@@ -32,10 +31,10 @@ public final class HCSImageFileExtractionResult
     /** The invalid files found. */
     private final List<File> invalidFiles;
 
-    private final Set<Channel> channels;
+    private final List<Channel> channels;
 
     public HCSImageFileExtractionResult(List<AcquiredPlateImage> images, List<File> invalidFiles,
-            Set<Channel> channels)
+            List<Channel> channels)
     {
         this.images = images;
         this.invalidFiles = invalidFiles;
@@ -52,7 +51,7 @@ public final class HCSImageFileExtractionResult
         return invalidFiles;
     }
 
-    public Set<Channel> getChannels()
+    public List<Channel> getChannels()
     {
         return channels;
     }

screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSImageFileExtractor.java

@@ -20,11 +20,11 @@ import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Properties;
-import java.util.Set;
 
 import ch.systemsx.cisd.bds.hcs.Geometry;
 import ch.systemsx.cisd.bds.hcs.Location;
 import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.utilities.PropertyUtils;
 import ch.systemsx.cisd.openbis.dss.etl.HCSImageFileExtractionResult.Channel;
 import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifier;
 import ch.systemsx.cisd.openbis.plugin.screening.shared.basic.dto.ChannelDescription;
@@ -49,6 +49,13 @@ public class HCSImageFileExtractor extends AbstractHCSImageFileExtractor
 {
     private static final String TILE_MAPPING = "tile_mapping";
 
+    // boolean property, if true the names of the plate in file name and directory name have to
+    // match.
+    // True by default.
+    private static final String CHECK_PLATE_NAME_FLAG_PROPERTY_NAME = "validate-plate-name";
+
+    private final boolean shouldValidatePlateName;
+
     private final TileMapper tileMapperOrNull;
 
     private final List<ChannelDescription> channelDescriptions;
@@ -66,6 +73,8 @@ public class HCSImageFileExtractor extends AbstractHCSImageFileExtractor
         this.wellGeometry = getWellGeometry(properties);
         this.tileMapperOrNull =
                 TileMapper.tryCreate(properties.getProperty(TILE_MAPPING), wellGeometry);
+        this.shouldValidatePlateName =
+                PropertyUtils.getBoolean(properties, CHECK_PLATE_NAME_FLAG_PROPERTY_NAME, true);
     }
 
     private void checkChannelsAndColorComponents()
@@ -145,7 +154,7 @@ public class HCSImageFileExtractor extends AbstractHCSImageFileExtractor
     }
 
     @Override
-    protected Set<Channel> getAllChannels()
+    protected List<Channel> getAllChannels()
     {
         return createChannels(channelDescriptions);
     }
@@ -153,6 +162,6 @@ public class HCSImageFileExtractor extends AbstractHCSImageFileExtractor
     @Override
     protected ImageFileInfo tryExtractImageInfo(File imageFile, SampleIdentifier datasetSample)
     {
-        return tryExtractDefaultImageInfo(imageFile, datasetSample);
+        return tryExtractDefaultImageInfo(imageFile, datasetSample, shouldValidatePlateName);
     }
 }

screening/source/java/ch/systemsx/cisd/openbis/dss/etl/PlateStorageProcessor.java

@@ -22,7 +22,6 @@ import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Properties;
 import java.util.Set;
@@ -102,11 +101,11 @@ public final class PlateStorageProcessor extends AbstractStorageProcessor
     /** The directory where <i>original</i> data could be found. */
     private static final String DIR_ORIGINAL = ScreeningConstants.ORIGINAL_DATA_DIR;
 
-    private static final Logger operationLog =
-            LogFactory.getLogger(LogCategory.OPERATION, PlateStorageProcessor.class);
+    private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
+            PlateStorageProcessor.class);
 
-    private static final Logger notificationLog =
-            LogFactory.getLogger(LogCategory.NOTIFY, PlateStorageProcessor.class);
+    private static final Logger notificationLog = LogFactory.getLogger(LogCategory.NOTIFY,
+            PlateStorageProcessor.class);
 
     // tiles geometry, e.g. 3x4 if the well is divided into 12 tiles (3 rows, 4 columns)
     private static final String SPOT_GEOMETRY_PROPERTY = "well_geometry";
@@ -460,8 +459,8 @@ public final class PlateStorageProcessor extends AbstractStorageProcessor
         {
             throw UserFailureException.fromTemplate(
                     "No extractable files were found inside a dataset '%s'."
-                            + " Have you changed your naming convention?", incomingDataSetDirectory
-                            .getAbsolutePath());
+                            + " Have you changed your naming convention?",
+                    incomingDataSetDirectory.getAbsolutePath());
         }
         checkCompleteness(imageCheckList, dataSetInformation, incomingDataSetDirectory.getName(),
                 mailClient);
@@ -756,19 +755,19 @@ public final class PlateStorageProcessor extends AbstractStorageProcessor
                             new HCSImageFileAccepter(imageFileRootDirectory,
                                     extractChannelCodes(descriptions));
                     ch.systemsx.cisd.etlserver.HCSImageFileExtractionResult originalResult =
-                            extractor.process(NodeFactory
-                                    .createDirectoryNode(incomingDataSetDirectory),
+                            extractor.process(
+                                    NodeFactory.createDirectoryNode(incomingDataSetDirectory),
                                     dataSetInformation, accepter);
-                    Set<HCSImageFileExtractionResult.Channel> channels =
+                    List<HCSImageFileExtractionResult.Channel> channels =
                             convert(originalResult.getChannels());
                     return new HCSImageFileExtractionResult(accepter.getImages(),
                             asRelativePaths(originalResult.getInvalidFiles()), channels);
                 }
 
-                private Set<HCSImageFileExtractionResult.Channel> convert(Set<Channel> channels)
+                private List<HCSImageFileExtractionResult.Channel> convert(Set<Channel> channels)
                 {
-                    Set<HCSImageFileExtractionResult.Channel> result =
-                            new HashSet<HCSImageFileExtractionResult.Channel>();
+                    List<HCSImageFileExtractionResult.Channel> result =
+                            new ArrayList<HCSImageFileExtractionResult.Channel>();
                     for (Channel channel : channels)
                     {
                         result.add(new HCSImageFileExtractionResult.Channel(getChannelCodeOrLabel(
@@ -820,8 +819,8 @@ public final class PlateStorageProcessor extends AbstractStorageProcessor
                 final Location tileLocation, final IFile imageFile)
         {
             final String imageRelativePath =
-                    FileUtilities.getRelativeFile(imageFileRootDirectory, new File(imageFile
-                            .getPath()));
+                    FileUtilities.getRelativeFile(imageFileRootDirectory,
+                            new File(imageFile.getPath()));
             assert imageRelativePath != null : "Image relative path should not be null.";
             String channelCode = getChannelCodeOrLabel(channelCodes, channel);
             AcquiredPlateImage imageDesc =

screening/source/java/ch/systemsx/cisd/openbis/dss/etl/ScreeningContainerDatasetInfoHelper.java

@@ -19,7 +19,6 @@ package ch.systemsx.cisd.openbis.dss.etl;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 import ch.systemsx.cisd.common.exceptions.UserFailureException;
 import ch.systemsx.cisd.openbis.dss.etl.HCSImageFileExtractionResult.Channel;
@@ -62,8 +61,8 @@ public class ScreeningContainerDatasetInfoHelper
     public static long createImageDataset(IImagingQueryDAO dao, ImageDatasetInfo info, long contId)
     {
         ImgDatasetDTO dataset =
-                new ImgDatasetDTO(info.getDatasetPermId(), info.getTileRows(), info
-                        .getTileColumns(), contId, info.hasImageSeries());
+                new ImgDatasetDTO(info.getDatasetPermId(), info.getTileRows(),
+                        info.getTileColumns(), contId, info.hasImageSeries());
         return dao.addDataset(dataset);
     }
 
@@ -100,7 +99,7 @@ public class ScreeningContainerDatasetInfoHelper
      */
     public static ExperimentWithChannelsAndContainer getOrCreateExperimentWithChannelsAndContainer(
             IImagingQueryDAO dao, ScreeningContainerDatasetInfo info,
-            Set<HCSImageFileExtractionResult.Channel> channels)
+            List<HCSImageFileExtractionResult.Channel> channels)
     {
         ScreeningContainerDatasetInfoHelper helper = new ScreeningContainerDatasetInfoHelper(dao);
         synchronized (IImagingQueryDAO.class)
@@ -131,8 +130,8 @@ public class ScreeningContainerDatasetInfoHelper
         } else
         {
             ImgContainerDTO container =
-                    new ImgContainerDTO(containerPermId, info.getContainerRows(), info
-                            .getContainerColumns(), expId);
+                    new ImgContainerDTO(containerPermId, info.getContainerRows(),
+                            info.getContainerColumns(), expId);
             containerId = dao.addContainer(container);
             return new CreatedOrFetchedEntity(false, containerId);
         }
@@ -219,7 +218,7 @@ public class ScreeningContainerDatasetInfoHelper
     // ------ channels creation ------------------------------
 
     private Map<String, Long> getOrCreateChannels(long expId,
-            Set<HCSImageFileExtractionResult.Channel> channels)
+            List<HCSImageFileExtractionResult.Channel> channels)
     {
         List<ImgChannelDTO> allChannels = dao.getChannelsByExperimentId(expId);
         if (allChannels.size() == 0)
@@ -231,7 +230,7 @@ public class ScreeningContainerDatasetInfoHelper
         }
     }
 
-    private Map<String, Long> updateChannels(long expId, Set<Channel> channels,
+    private Map<String, Long> updateChannels(long expId, List<Channel> channels,
             List<ImgChannelDTO> allChannels)
     {
         Map<String/* name */, ImgChannelDTO> existingChannels = asNameMap(allChannels);
@@ -244,7 +243,7 @@ public class ScreeningContainerDatasetInfoHelper
         return map;
     }
 
-    private Map<String, Long> createChannels(long expId, Set<Channel> channels)
+    private Map<String, Long> createChannels(long expId, List<Channel> channels)
     {
         Map<String, Long> map = new HashMap<String, Long>();
         for (HCSImageFileExtractionResult.Channel channel : channels)

screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dynamix/HCSImageFileExtractor.java

@@ -26,7 +26,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
-import java.util.Set;
 
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.lang.StringUtils;
@@ -69,7 +68,7 @@ public class HCSImageFileExtractor extends AbstractHCSImageFileExtractor
     }
 
     @Override
-    protected final Set<Channel> getAllChannels()
+    protected final List<Channel> getAllChannels()
     {
         return createChannels(channelDescriptions);
     }

screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/imaging/dataaccess/IImagingReadonlyQueryDAO.java

@@ -111,14 +111,11 @@ public interface IImagingReadonlyQueryDAO extends BaseQuery
     @Select("select count(*) from CHANNELS where DS_ID = ?{1} or EXP_ID = ?{2}")
     public int countChannelByDatasetIdOrExperimentId(long datasetId, long experimentId);
 
-    @Select("select * from CHANNELS where DS_ID = ?{1} or EXP_ID = ?{2} order by CODE")
+    @Select("select * from CHANNELS where DS_ID = ?{1} or EXP_ID = ?{2} order by ID")
     public List<ImgChannelDTO> getChannelsByDatasetIdOrExperimentId(long datasetId,
             long experimentId);
 
-    @Select(sql = "select id from CHANNELS where DS_ID = ?{1} or EXP_ID = ?{2} order by LABEL", fetchSize = FETCH_SIZE)
-    public long[] getChannelIdsByDatasetIdOrExperimentId(long datasetId, long experimentId);
-
-    @Select(sql = "select * from CHANNELS where EXP_ID = ?{1} order by LABEL", fetchSize = FETCH_SIZE)
+    @Select(sql = "select * from CHANNELS where EXP_ID = ?{1} order by ID", fetchSize = FETCH_SIZE)
     public List<ImgChannelDTO> getChannelsByExperimentId(long experimentId);
 
     @Select("select * from SPOTS where cont_id = ?{1}")

screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/ImagingQueryDAOTest.java

@@ -181,14 +181,18 @@ public class ImagingQueryDAOTest extends AbstractDBTest
         assertEquals(CHANNEL_LABEL, channelDTOS.get(1).getLabel());
 
         // test getChannelIdsByDatasetIdOrExperimentId
-        long[] channels = dao.getChannelIdsByDatasetIdOrExperimentId(datasetId, experimentId);
-        assertEquals(2, channels.length);
-        AssertJUnit.assertTrue(channels[0] == channelId1 && channels[1] == channelId2
-                || channels[1] == channelId1 && channels[0] == channelId2);
+        List<ImgChannelDTO> channels =
+                dao.getChannelsByDatasetIdOrExperimentId(datasetId, experimentId);
+        assertEquals(2, channels.size());
+        AssertJUnit.assertTrue(channels.get(0).getId() == channelId1
+                && channels.get(1).getId() == channelId2 || channels.get(1).getId() == channelId1
+                && channels.get(0).getId() == channelId2);
 
         // test get id of first channel
-        assertEquals(channels[0], dao.tryGetChannelIdByChannelCodeDatasetIdOrExperimentId(
-                datasetId, experimentId, "dsChannel").intValue());
+        assertEquals(
+                channels.get(0).getId(),
+                dao.tryGetChannelIdByChannelCodeDatasetIdOrExperimentId(datasetId, experimentId,
+                        "dsChannel").intValue());
 
         List<ImgChannelDTO> experimentChannels = dao.getChannelsByExperimentId(experimentId);
         assertEquals(1, experimentChannels.size());