[LMS-2085] extend raw image dropbox to use image metadata : added algorithm to...

[LMS-2085] extend raw image dropbox to use image metadata : added algorithm to calculate tile positions

SVN: 20554

screening/source/java/ch/systemsx/cisd/openbis/dss/etl/biozentrum/ImageMetadataExtractor.java

@@ -22,8 +22,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
-import java.util.Set;
-import java.util.TreeSet;
 
 import ch.systemsx.cisd.common.exceptions.UserFailureException;
 import ch.systemsx.cisd.imagereaders.IImageReader;
@@ -54,56 +52,84 @@ public class ImageMetadataExtractor
     }
 
     /**
+     * NOTE : the speed of this algorithm can be immensely improved by
+     * 
+     * <pre>
+     * 1) Avoiding auto-boxing (we have multiple collections storing primitive types) 
+     * 2) Avoid parsing all numbers twice 
+     * 3) Use a sorted collection (tree?) + binary search when looking if a value is already present.
+     * </pre>
+     * 
      * @param tileToMetadataMap mapping from tile number to image metadata
      */
     public static Map<Integer/* tile number */, Location> tryGetTileMapping(
-            Map<Integer/* tile number */, Map<String/* name */, Object/* value */>> tileToMetadataMap)
+            Map<Integer/* tile number */, Map<String/* name */, Object/* value */>> tileToMetadataMap,
+            double epsilon)
     {
-        Set<Number> xCoords = new TreeSet<Number>();
-        Set<Number> yCoords = new TreeSet<Number>();
+        List<Double> xCoords = new ArrayList<Double>();
+        List<Double> yCoords = new ArrayList<Double>();
+
         for (Map<String, Object> metadata : tileToMetadataMap.values())
         {
-            xCoords.add(extractXCoord(metadata));
-            yCoords.add(extractYCoord(metadata));
+            addIfNotPresent(xCoords, extractXCoord(metadata), epsilon);
+            addIfNotPresent(yCoords, extractYCoord(metadata), epsilon);
         }
 
-        List<Number> sortedXCoords = new ArrayList<Number>(xCoords);
-        List<Number> sortedYCoords = new ArrayList<Number>(yCoords);
-
         Map<Integer, Location> result = new HashMap<Integer, Location>();
         for (Entry<Integer, Map<String, Object>> entry : tileToMetadataMap.entrySet())
         {
             Integer tileNumber = entry.getKey();
-            Location location = extractLocation(entry.getValue(), sortedXCoords, sortedYCoords);
+            Location location = extractLocation(entry.getValue(), xCoords, yCoords, epsilon);
             result.put(tileNumber, location);
         }
         return result;
     }
 
-    private static Location extractLocation(Map<String, Object> metadata, List<Number> xCoords,
-            List<Number> yCoords)
+    private static void addIfNotPresent(List<Double> values, Double value, double epsilon)
+    {
+        int idx = findIdxByEpsilon(values, value, epsilon);
+        if (idx < 0)
+        {
+            values.add(value);
+        }
+    }
+
+    private static int findIdxByEpsilon(List<Double> values, double toFind, double epsilon)
+    {
+        for (int idx = 0; idx < values.size(); idx++)
+        {
+            if (Math.abs(values.get(idx) - toFind) < epsilon)
+            {
+                return idx;
+            }
+        }
+        return -1;
+    }
+
+    private static Location extractLocation(Map<String, Object> metadata, List<Double> xCoords,
+            List<Double> yCoords, double epsilon)
     {
-        Number x = extractXCoord(metadata);
-        Number y = extractYCoord(metadata);
+        double x = extractXCoord(metadata);
+        double y = extractYCoord(metadata);
 
-        int locationX = xCoords.indexOf(x);
-        int locationY = yCoords.indexOf(y);
+        int locationX = findIdxByEpsilon(xCoords, x, epsilon);
+        int locationY = findIdxByEpsilon(yCoords, y, epsilon);
 
         Location location = new Location(locationY, locationX);
         return location;
     }
 
-    private static Number extractXCoord(Map<String, Object> metadata)
+    private static Double extractXCoord(Map<String, Object> metadata)
     {
         return extractNumber(metadata, POSITION_X_PROP);
     }
 
-    private static Number extractYCoord(Map<String, Object> metadata)
+    private static Double extractYCoord(Map<String, Object> metadata)
     {
         return extractNumber(metadata, POSITION_Y_PROP);
     }
 
-    private static Number extractNumber(Map<String, Object> metadata, String propName)
+    private static Double extractNumber(Map<String, Object> metadata, String propName)
     {
         String numberAsString = (String) metadata.get(propName);
         try

screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/Location.java

@@ -25,4 +25,39 @@ public class Location
     {
         return column;
     }
+
+    @Override
+    public boolean equals(Object obj)
+    {
+        if (this == obj)
+        {
+            return true;
+        }
+        if (obj == null)
+        {
+            return false;
+        }
+        if (getClass() != obj.getClass())
+        {
+            return false;
+        }
+        Location other = (Location) obj;
+        return column == other.column && row == other.row;
+    }
+
+    @Override
+    public int hashCode()
+    {
+        final int prime = 31;
+        int result = 1;
+        result = prime * result + column;
+        result = prime * result + row;
+        return result;
+    }
+
+    @Override
+    public String toString()
+    {
+        return "[" + row + ":" + column + "]";
+    }
 }
\ No newline at end of file

screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/biozentrum/ImageMetadataExtractorTest.java

 /*
  * Copyright 2011 ETH Zuerich, CISD
+
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,12 +17,18 @@
 
 package ch.systemsx.cisd.openbis.dss.etl.biozentrum;
 
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.lang.builder.ToStringBuilder;
 import org.testng.AssertJUnit;
 import org.testng.annotations.Test;
 
+import com.csvreader.CsvReader;
+
 import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.Location;
 
 /**
@@ -29,40 +36,135 @@ import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.Location;
  */
 public class ImageMetadataExtractorTest extends AssertJUnit
 {
+    private final String TEST_FILE =
+            "./sourceTest/java/ch/systemsx/cisd/openbis/dss/etl/biozentrum/metadata.csv";
+
+    /**
+     * the test data works for epsilon values from 0.7 up to approx. 890
+     */
+    private final double EPSILON = 1.0;
+
+    private class ImageFile
+    {
+        Integer tileNumber;
+
+        String positionX;
+
+        String positionY;
+
+        /** A01, A02 .. etc */
+        String positionID;
+        
+        @Override
+        public String toString() {
+            ToStringBuilder builder = new ToStringBuilder(this);
+            builder.append("tileNumber", tileNumber);
+            builder.append("positionX", positionX);
+            builder.append("positionY", positionY);
+            builder.append("positionID", positionID);
+            return builder.toString();
+        }
+    }
+
     @Test
     public void testTileMapping() throws Exception
     {
-        Map<Integer, Map<String, Object>> tileMetadata =
-                new HashMap<Integer, Map<String, Object>>();
+        
+        List<ImageFile> imageFiles = readMetaDataFromFile(TEST_FILE);
+        
+        Map<Integer, Location> locations =
+                ImageMetadataExtractor.tryGetTileMapping(createMetaData(imageFiles), EPSILON);
+        
+        Map<String, List<ImageFile>> groupByPosition = groupByPositions(imageFiles);
+        for (String position : groupByPosition.keySet()) {
+            List<ImageFile> imagesForPosition = groupByPosition.get(position);
+            ImageFile firstImage = imagesForPosition.get(0);
+            for (ImageFile otherImage : imagesForPosition)
+            {
+                assertEquals(locations, firstImage, otherImage);
+            }
+        }
+
+        assertLocationDimensions(48, 48, locations.values());
+    }
+
+    private void assertEquals(Map<Integer, Location> locations, ImageFile expected, ImageFile actual)
+    {
+        Location expectedLocation = locations.get(expected.tileNumber);
+        Location actualLocation = locations.get(actual.tileNumber);
+
+        String error =
+                String.format("%s has been assigned a different location than %s", expected, actual);
+        assertEquals(error, expectedLocation, actualLocation);
+    }
+
+    private void assertLocationDimensions(int maxRow, int maxColumn, Collection<Location> locations)
+    {
+        for (Location location : locations)
+        {
+            assertTrue("Location should not have row > " + maxRow, maxRow > location.getRow());
+            assertTrue("Location should not have column > " + maxColumn,
+                    maxColumn > location.getColumn());
 
-        tileMetadata.put(1, createTileMetadata(1, 1));
-        tileMetadata.put(2, createTileMetadata(2, 2));
-        tileMetadata.put(3, createTileMetadata(3, 3));
-        tileMetadata.put(4, createTileMetadata(1, 2));
-        tileMetadata.put(5, createTileMetadata(3, 2));
+        }
 
-        Map<Integer, Location> locations = ImageMetadataExtractor.tryGetTileMapping(tileMetadata);
+    }
+
+    private List<ImageFile> readMetaDataFromFile(String fileName) throws Exception
+    {
+        List<ImageFile> result = new ArrayList<ImageFile>();
+        CsvReader reader = new CsvReader(fileName);
+        
+        reader.readHeaders();
 
-        assertLocation(0, 0, 1, locations);
-        assertLocation(1, 1, 2, locations);
-        assertLocation(2, 2, 3, locations);
-        assertLocation(0, 1, 4, locations);
-        assertLocation(2, 1, 5, locations);
+        int tileNumber = 1;
+        while (reader.readRecord())
+        {
+            ImageFile img = new ImageFile();
+            img.tileNumber = tileNumber++;
+            img.positionX = reader.get("PositionX");
+            img.positionY = reader.get("PositionY");
+            img.positionID = reader.get("stage-label");
+            
+            result.add(img);
+        }
 
+        reader.close();
+        
+        return result;
+    }
+
+    private Map<Integer, Map<String, Object>> createMetaData(List<ImageFile> imageFiles)
+    {
+        Map<Integer, Map<String, Object>> result = new HashMap<Integer, Map<String, Object>>();
+        for (ImageFile imgFile : imageFiles)
+        {
+            result.put(imgFile.tileNumber, createTileMetadata(imgFile));
+        }
+        return result;
     }
 
-    private void assertLocation(int column, int row, int tileNumber, Map<Integer, Location> locations)
+    private Map<String, List<ImageFile>> groupByPositions(List<ImageFile> imageFiles)
     {
-        Location location = locations.get(tileNumber);
-        assertEquals(column, location.getColumn());
-        assertEquals(row, location.getRow());
+        Map<String, List<ImageFile>> result = new HashMap<String, List<ImageFile>>();
+        for (ImageFile imgFile : imageFiles)
+        {
+            List<ImageFile> list = result.get(imgFile.positionID);
+            if (list == null)
+            {
+                list = new ArrayList<ImageFile>();
+                result.put(imgFile.positionID, list);
+            }
+            list.add(imgFile);
+        }
+        return result;
     }
 
-    private Map<String, Object> createTileMetadata(double x, double y)
+    private Map<String, Object> createTileMetadata(ImageFile imgFile)
     {
         Map<String, Object> metadata = new HashMap<String, Object>();
-        metadata.put(ImageMetadataExtractor.POSITION_X_PROP, String.valueOf(x));
-        metadata.put(ImageMetadataExtractor.POSITION_Y_PROP, String.valueOf(y));
+        metadata.put(ImageMetadataExtractor.POSITION_X_PROP, imgFile.positionX);
+        metadata.put(ImageMetadataExtractor.POSITION_Y_PROP, imgFile.positionY);
         return metadata;
     }