LMS-1511 use merged stream to deliver screening images

SVN: 15787

screening/build/build.xml

@@ -125,6 +125,7 @@
   		</fileset> 
       <fileset dir="${classes.common}">
             <include name="ch/systemsx/cisd/common/exceptions/**/*.class" />
+            <include name="ch/systemsx/cisd/common/io/ConcatFileOutputStreamWriter.class" />
       </fileset> 
     </jar>
     <recursive-jar destfile="${dist.screening-api.lib}/spring-ext.jar">

screening/source/java/ch/systemsx/cisd/openbis/dss/screening/server/DssServiceRpcScreening.java

@@ -18,13 +18,12 @@ package ch.systemsx.cisd.openbis.dss.screening.server;
 
 import java.awt.image.BufferedImage;
 import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -35,6 +34,8 @@ import ch.systemsx.cisd.bds.hcs.HCSDatasetLoader;
 import ch.systemsx.cisd.bds.hcs.Location;
 import ch.systemsx.cisd.bds.storage.INode;
 import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.common.exceptions.UserFailureException;
+import ch.systemsx.cisd.common.io.ConcatFileInputStream;
 import ch.systemsx.cisd.openbis.dss.generic.server.AbstractDssServiceRpc;
 import ch.systemsx.cisd.openbis.dss.generic.server.images.ImageChannelsUtils;
 import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.DatasetFileLines;
@@ -217,40 +218,74 @@ public class DssServiceRpcScreening extends AbstractDssServiceRpc implements
                 featureNames);
     }
 
-    public InputStream loadImage(String sessionToken, PlateImageReference imageReference)
+    public InputStream loadImages(String sessionToken, List<PlateImageReference> imageReferences)
     {
-        ExternalData imageDataset =
-                tryFindImageDataset(sessionToken, imageReference.getDatasetCode());
-        if (imageDataset != null)
+        Map<String, HCSDatasetLoader> imageLoadersMap =
+                getImageDatasetsMap(sessionToken, imageReferences);
+        List<File> imageFiles = new ArrayList<File>();
+        try
         {
-            return getImageStream(imageDataset, imageReference);
-        } else
+            for (PlateImageReference imageReference : imageReferences)
+            {
+                HCSDatasetLoader imageAccessor =
+                        imageLoadersMap.get(imageReference.getDatasetCode());
+                assert imageAccessor != null : "imageAccessor not found for: " + imageReference;
+                File imageFile = getImageFile(imageAccessor, imageReference);
+                imageFiles.add(imageFile);
+            }
+        } finally
         {
-            return null;
+            closeDatasetLoaders(imageLoadersMap.values());
+        }
+        return new ConcatFileInputStream(imageFiles);
+    }
+
+    private static void closeDatasetLoaders(Collection<HCSDatasetLoader> loaders)
+    {
+        for (HCSDatasetLoader loader : loaders)
+        {
+            loader.close();
+        }
+    }
+
+    // throws exception if some datasets cannot be found
+    private Map<String/* image or feature vector dataset code */, HCSDatasetLoader> getImageDatasetsMap(
+            String sessionToken, List<PlateImageReference> imageReferences)
+    {
+        Map<String/* dataset code */, HCSDatasetLoader> imageDatasetsMap =
+                new HashMap<String, HCSDatasetLoader>();
+        for (PlateImageReference imageReference : imageReferences)
+        {
+            if (imageDatasetsMap.containsKey(imageReference.getDatasetCode()) == false)
+            {
+                ExternalData imageDataset =
+                        tryFindImageDataset(sessionToken, imageReference.getDatasetCode());
+                if (imageDataset != null)
+                {
+                    HCSDatasetLoader imageAccessor = createImageLoader(imageDataset.getCode());
+                    imageDatasetsMap.put(imageReference.getDatasetCode(), imageAccessor);
+                } else
+                {
+                    throw UserFailureException.fromTemplate(
+                            "Cannot find an image dataset for the reference: %s", imageReference);
+                }
+            }
         }
+        return imageDatasetsMap;
     }
 
-    private InputStream getImageStream(ExternalData imageDataset, PlateImageReference imageRef)
+    private File getImageFile(HCSDatasetLoader imageAccessor, PlateImageReference imageRef)
     {
-        HCSDatasetLoader imageAccessor = createImageLoader(imageDataset.getCode());
         Location wellLocation = asLocation(imageRef.getWellPosition());
         Location tileLocation =
                 getTileLocation(imageRef.getTile(), imageAccessor.getWellGeometry());
         try
         {
-            File path =
-                    ImageChannelsUtils.getImagePath(imageAccessor, wellLocation, tileLocation,
-                            imageRef.getChannel());
-            return new FileInputStream(path);
+            return ImageChannelsUtils.getImagePath(imageAccessor, wellLocation, tileLocation,
+                    imageRef.getChannel());
         } catch (EnvironmentFailureException e)
         {
             throw createNoImageException(imageRef);
-        } catch (FileNotFoundException ex)
-        {
-            throw createNoImageException(imageRef);
-        } finally
-        {
-            imageAccessor.close();
         }
     }
 

screening/source/java/ch/systemsx/cisd/openbis/dss/screening/shared/api/v1/IDssServiceRpcScreening.java

@@ -49,10 +49,13 @@ public interface IDssServiceRpcScreening
             List<String> featureNames);
 
     /**
-     * Provide image for a given image reference (given by data set code, well position, channel and
-     * tile).
+     * Provide images for a given list of image references (given by data set code, well position,
+     * channel and tile). The result is encoded into one stream, which consist of multiple blocks in
+     * a format: (<block-size><block-of-bytes>)*, where block-size is the block size in bytes
+     * encoded as one long number. The number of blocks is equal to the number of specified
+     * references and the order of blocks corresponds to the order of image references.
      */
-    InputStream loadImage(String sessionToken, PlateImageReference imageReferences);
+    InputStream loadImages(String sessionToken, List<PlateImageReference> imageReferences);
 
     /**
      * For a given set of image data sets, provide all image channels that have been acquired and

screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/client/api/v1/ScreeningClientApiTest.java

@@ -18,16 +18,13 @@ package ch.systemsx.cisd.openbis.plugin.screening.client.api.v1;
 
 import java.io.File;
 import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Properties;
 
 import org.apache.log4j.PropertyConfigurator;
-import org.apache.log4j.lf5.util.StreamUtils;
 
 import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.FeatureVectorDataset;
 import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.FeatureVectorDatasetReference;
@@ -36,7 +33,6 @@ import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.ImageDatasetM
 import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.ImageDatasetReference;
 import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.Plate;
 import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.PlateImageReference;
-import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.PlateSingleImage;
 import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.WellPosition;
 
 /**
@@ -112,38 +108,33 @@ public class ScreeningClientApiTest
     private static void loadImages(ScreeningOpenbisServiceFacade facade,
             IDatasetIdentifier datasetIdentifier) throws FileNotFoundException, IOException
     {
-        for (int well = 1; well <= 5; well++)
+        File dir = new File(datasetIdentifier.getDatasetCode());
+        dir.mkdir();
+
+        List<PlateImageReference> imageRefs = new ArrayList<PlateImageReference>();
+        List<File> imageFiles = new ArrayList<File>();
+        for (int wellRow = 1; wellRow <= 5; wellRow++)
         {
-            for (int channel = 1; channel <= 2; channel++)
+            for (int wellCol = 1; wellCol <= 5; wellCol++)
             {
-                for (int tile = 1; tile <= 1; tile++)
+                for (int channel = 1; channel <= 2; channel++)
                 {
-                    List<PlateImageReference> imageRefs = new ArrayList<PlateImageReference>();
-                    imageRefs.add(new PlateImageReference(well, well, tile, channel,
-                            datasetIdentifier));
-                    List<PlateSingleImage> images = facade.loadImages(imageRefs);
-                    saveImages(images, datasetIdentifier.getDatasetCode());
+                    for (int tile = 1; tile <= 1; tile++)
+                    {
+
+                        PlateImageReference imageRef =
+                                new PlateImageReference(wellRow, wellCol, tile, channel,
+                                        datasetIdentifier);
+                        imageRefs.add(imageRef);
+                        imageFiles.add(new File(dir, createImageFileName(imageRef)));
+                    }
                 }
             }
         }
+        facade.loadImages(imageRefs, imageFiles);
     }
 
-    private static void saveImages(List<PlateSingleImage> images, String dirName)
-            throws FileNotFoundException, IOException
-    {
-        File dir = new File(dirName);
-        dir.mkdir();
-        for (PlateSingleImage image : images)
-        {
-            FileOutputStream out = new FileOutputStream(new File(dir, createImageFileName(image)));
-            InputStream in = image.getImage();
-            StreamUtils.copyThenClose(in, out);
-            out.close();
-            in.close();
-        }
-    }
-
-    private static String createImageFileName(PlateSingleImage image)
+    private static String createImageFileName(PlateImageReference image)
     {
         WellPosition well = image.getWellPosition();
         return "img_row" + well.getWellRow() + "_col" + well.getWellColumn() + "_channel"

screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/client/api/v1/ScreeningOpenbisServiceFacade.java

 package ch.systemsx.cisd.openbis.plugin.screening.client.api.v1;
 
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import ch.systemsx.cisd.common.io.ConcatFileOutputStreamWriter;
 import ch.systemsx.cisd.common.spring.HttpInvokerUtils;
 import ch.systemsx.cisd.openbis.dss.screening.shared.api.v1.IDssServiceRpcScreening;
 import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.IScreeningApiServer;
@@ -19,7 +27,6 @@ import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.ImageDatasetR
 import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.Plate;
 import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.PlateIdentifier;
 import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.PlateImageReference;
-import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.PlateSingleImage;
 
 /**
  * A client side facade of openBIS and Datastore Server API.
@@ -156,25 +163,97 @@ public class ScreeningOpenbisServiceFacade
     }
 
     /**
-     * Provide images for a given set of image references (given by data set code, well position,
-     * channel and tile).
+     * Saves images for a given list of image references (given by data set code, well position,
+     * channel and tile) in the specified files.<br>
+     * The number of image references has to be the same as the number of files.
+     * 
+     * @throws IOException when reading images from the server or writing them to the files fails
+     */
+    public void loadImages(List<PlateImageReference> imageReferences, List<File> imageOutputFiles)
+            throws IOException
+    {
+        final Map<PlateImageReference, OutputStream> imageRefToFileMap =
+                createImageToFileMap(imageReferences, imageOutputFiles);
+        loadImages(imageReferences, new IImageOutputStreamProvider()
+            {
+                public OutputStream getOutputStream(PlateImageReference imageReference)
+                        throws IOException
+                {
+                    return imageRefToFileMap.get(imageReference);
+                }
+            });
+        closeOutputStreams(imageRefToFileMap.values());
+    }
+
+    private static void closeOutputStreams(Collection<OutputStream> streams) throws IOException
+    {
+        for (OutputStream stream : streams)
+        {
+            stream.close();
+        }
+    }
+
+    private static Map<PlateImageReference, OutputStream> createImageToFileMap(
+            List<PlateImageReference> imageReferences, List<File> imageOutputFiles)
+            throws FileNotFoundException
+    {
+        assert imageReferences.size() == imageOutputFiles.size() : "there should be one file specified for each image reference";
+        Map<PlateImageReference, OutputStream> map =
+                new HashMap<PlateImageReference, OutputStream>();
+        for (int i = 0; i < imageReferences.size(); i++)
+        {
+            OutputStream out =
+                    new BufferedOutputStream(new FileOutputStream(imageOutputFiles.get(i)));
+            map.put(imageReferences.get(i), out);
+        }
+        return map;
+    }
+
+    /**
+     * An interface to provide mapping between image references and output streams where the images
+     * should be saved.
+     */
+    public static interface IImageOutputStreamProvider
+    {
+        /**
+         * @return output stream where the image for the specified reference should be saved.
+         * @throws IOException when creating the output stream fails
+         */
+        OutputStream getOutputStream(PlateImageReference imageReference) throws IOException;
+    }
+
+    /**
+     * Saves images for a given list of image references (given by data set code, well position,
+     * channel and tile) in the provided output streams. Output streams will not be closed
+     * automatically.<br>
+     * The number of image references has to be the same as the number of files.
+     * 
+     * @throws IOException when reading images from the server or writing them to the output streams
+     *             fails
      */
-    public List<PlateSingleImage> loadImages(List<PlateImageReference> imageReferences)
+    public void loadImages(List<PlateImageReference> imageReferences,
+            IImageOutputStreamProvider outputStreamProvider) throws IOException
     {
         if (imageReferences.size() == 0)
         {
-            return new ArrayList<PlateSingleImage>();
+            return;
         }
         String datastoreServerUrl = extractDatastoreServerUrl(imageReferences);
         IDssServiceRpcScreening dssServer = getScreeningDssServer(datastoreServerUrl);
 
-        List<PlateSingleImage> images = new ArrayList<PlateSingleImage>();
-        for (PlateImageReference imageRef : imageReferences)
+        InputStream stream = dssServer.loadImages(sessionToken, imageReferences);
+        try
+        {
+            ConcatFileOutputStreamWriter imagesWriter = new ConcatFileOutputStreamWriter(stream);
+            for (PlateImageReference imageRef : imageReferences)
+            {
+                OutputStream output = outputStreamProvider.getOutputStream(imageRef);
+                imagesWriter.writeNextBlock(output);
+            }
+        } finally
         {
-            InputStream stream = dssServer.loadImage(sessionToken, imageRef);
-            images.add(new PlateSingleImage(imageRef, stream));
+            stream.close();
         }
-        return images;
     }
 
     /**

screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/api/v1/dto/ImageDatasetMetadata.java

@@ -40,7 +40,7 @@ public class ImageDatasetMetadata implements Serializable
     /**
      * number of channels in which images have been acquired for the described dataset
      */
-    public int getChannelsNumber()
+    public int getNumberOfChannels()
     {
         return channelsNumber;
     }
@@ -48,7 +48,7 @@ public class ImageDatasetMetadata implements Serializable
     /**
      * number of image tiles (aka fields) into which each well is splited
      */
-    public int getTilesNumber()
+    public int getNumberOfTiles()
     {
         return tilesNumber;
     }

screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/api/v1/dto/PlateImageReference.java

@@ -55,4 +55,39 @@ public class PlateImageReference extends DatasetIdentifier implements Serializab
         return "Image for [dataset " + getDatasetCode() + ", well " + wellPosition + ", channel "
                 + channel + ", tile " + tile + "]";
     }
+
+    @Override
+    public int hashCode()
+    {
+        final int prime = 31;
+        int result = super.hashCode();
+        result = prime * result + super.hashCode();
+        result = prime * result + channel;
+        result = prime * result + tile;
+        result = prime * result + wellPosition.hashCode();
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object obj)
+    {
+        if (this == obj)
+            return true;
+        if (!super.equals(obj))
+            return false;
+        if (getClass() != obj.getClass())
+            return false;
+        if (super.equals(obj) == false)
+            return false;
+
+        PlateImageReference other = (PlateImageReference) obj;
+        if (channel != other.channel)
+            return false;
+        if (tile != other.tile)
+            return false;
+        if (!wellPosition.equals(other.wellPosition))
+            return false;
+        return true;
+    }
+
 }
\ No newline at end of file

screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/shared/api/v1/dto/PlateSingleImage.java

-package ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto;
-
-import java.io.InputStream;
-import java.io.Serializable;
-
-/**
- * Contains a stream with a single image from a plate and information from which well, channel and
- * tile it comes.
- * 
- * @author Tomasz Pylak
- */
-public class PlateSingleImage implements Serializable
-{
-    private static final long serialVersionUID = 1L;
-
-    private final PlateImageReference imageReference;
-
-    private final InputStream image;
-
-    public PlateSingleImage(PlateImageReference imageReference, InputStream image)
-    {
-        this.imageReference = imageReference;
-        this.image = image;
-    }
-
-    /** position of the well to which the image belongs */
-    public WellPosition getWellPosition()
-    {
-        return imageReference.getWellPosition();
-    }
-
-    /**
-     * tile (aka field) number, each well can be separated into many tiles, for each tile one image
-     * is acquired.
-     */
-    public int getTile()
-    {
-        return imageReference.getTile();
-    }
-
-    /** Index of the channel. Starts from 1. */
-    public int getChannel()
-    {
-        return imageReference.getChannel();
-    }
-
-    /** stream with a png image */
-    public InputStream getImage()
-    {
-        return image;
-    }
-
-    /** The dataset which has been specified to fetched the image. */
-    public IDatasetIdentifier getDataset()
-    {
-        return imageReference;
-    }
-
-}
\ No newline at end of file