LMS-2389 sanofi dropbox: allow to influence how thumbnails are generated by...

LMS-2389 sanofi dropbox: allow to influence how thumbnails are generated by convert, make 'convert' the default way of generating thumbnails (the other way is not thread-safe), do not register a dataset if thumbnail generation was failing

SVN: 22096

screening/source/java/ch/systemsx/cisd/openbis/dss/etl/Hdf5ThumbnailGenerator.java

@@ -20,7 +20,9 @@ import java.awt.image.BufferedImage;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.List;
 
 import javax.imageio.ImageIO;
@@ -29,6 +31,7 @@ import org.apache.log4j.Logger;
 
 import ch.systemsx.cisd.base.utilities.OSUtilities;
 import ch.systemsx.cisd.common.concurrent.ConcurrencyUtilities;
+import ch.systemsx.cisd.common.concurrent.FailureRecord;
 import ch.systemsx.cisd.common.concurrent.ITaskExecutor;
 import ch.systemsx.cisd.common.concurrent.ParallelizedExecutor;
 import ch.systemsx.cisd.common.exceptions.Status;
@@ -141,8 +144,8 @@ class Hdf5ThumbnailGenerator implements IHdf5WriterClient
         return newImagePath;
     }
 
-    private byte[] generateThumbnail(ByteArrayOutputStream bufferOutputStream, File img, String imageIdOrNull)
-            throws IOException
+    private byte[] generateThumbnail(ByteArrayOutputStream bufferOutputStream, File img,
+            String imageIdOrNull) throws IOException
     {
         byte[] byteArray;
         if (thumbnailsStorageFormat.isGenerateWithImageMagic())
@@ -161,9 +164,16 @@ class Hdf5ThumbnailGenerator implements IHdf5WriterClient
                 thumbnailsStorageFormat.getMaxWidth() + "x"
                         + thumbnailsStorageFormat.getMaxHeight();
         String imageFilePath = imageFile.getPath();
+        List<String> params = new ArrayList<String>();
+        params.addAll(Arrays.asList(convertUtilityOrNull.getPath(), imageFilePath, "-scale", size));
+        List<String> additionalParams = thumbnailsStorageFormat.getImageMagicParams();
+        if (additionalParams != null)
+        {
+            params.addAll(additionalParams);
+        }
+        params.add("png:-");
         final ProcessResult result =
-                ProcessExecutionHelper.run(Arrays.asList(convertUtilityOrNull.getPath(),
-                        imageFilePath, "-scale", size, "png:-"), operationLog, machineLog,
+                ProcessExecutionHelper.run(params, operationLog, machineLog,
                         ConcurrencyUtilities.NO_TIMEOUT,
                         ProcessIOStrategy.BINARY_DISCARD_STDERR_IO_STRATEGY, false);
         if (result.isOK() == false)
@@ -177,8 +187,8 @@ class Hdf5ThumbnailGenerator implements IHdf5WriterClient
         }
     }
 
-    private byte[] generateThumbnailInternally(File imageFile,
-            String imageIdOrNull, ByteArrayOutputStream bufferOutputStream) throws IOException
+    private byte[] generateThumbnailInternally(File imageFile, String imageIdOrNull,
+            ByteArrayOutputStream bufferOutputStream) throws IOException
     {
         BufferedImage image = loadImage(imageFile, imageIdOrNull);
         BufferedImage thumbnail =
@@ -231,8 +241,16 @@ class Hdf5ThumbnailGenerator implements IHdf5WriterClient
 
     public void runWithSimpleWriter(IHDF5SimpleWriter writer)
     {
-        ParallelizedExecutor.process(plateImages, createThumbnailGenerator(writer),
-                thumbnailsStorageFormat.getAllowedMachineLoadDuringGeneration(), 100,
-                "Thumbnails generation", MAX_RETRY_OF_FAILED_GENERATION);
+        Collection<FailureRecord<AcquiredSingleImage>> errors =
+                ParallelizedExecutor.process(plateImages, createThumbnailGenerator(writer),
+                        thumbnailsStorageFormat.getAllowedMachineLoadDuringGeneration(), 100,
+                        "Thumbnails generation", MAX_RETRY_OF_FAILED_GENERATION);
+        if (errors.size() > 0)
+        {
+            throw new IllegalStateException(
+                    String.format(
+                            "There were errors when generating %d thumbnails, the whole thumbnails generation process fails.",
+                            errors.size()));
+        }
     }
 }

screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/SimpleImageDataConfig.java

@@ -16,6 +16,8 @@
 
 package ch.systemsx.cisd.openbis.dss.etl.dto.api.v1;
 
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 
 import ch.systemsx.cisd.base.image.IImageTransformerFactory;
@@ -52,7 +54,7 @@ abstract public class SimpleImageDataConfig
      * just the {@link ImageMetadata} object returned by {@link #extractImageMetadata(String)}.
      * <p>
      * In case of a image container file format (like multi-page TIFF) this method should
-     * overridden. 
+     * overridden.
      * 
      * @param imageIdentifiers Identifiers of all images contained in the image file.
      */
@@ -142,7 +144,9 @@ abstract public class SimpleImageDataConfig
 
     private int maxThumbnailWidthAndHeight = 256;
 
-    private boolean generateThumbnailsWithImageMagic = false;
+    private boolean generateThumbnailsWithImageMagic = true;
+
+    private List<String> thumbnailsGenerationImageMagicParams = Collections.emptyList();
 
     private boolean generateThumbnailsInHighQuality = false;
 
@@ -176,6 +180,7 @@ abstract public class SimpleImageDataConfig
             thumbnailsStorageFormat.setMaxWidth(getMaxThumbnailWidthAndHeight());
             thumbnailsStorageFormat.setMaxHeight(getMaxThumbnailWidthAndHeight());
             thumbnailsStorageFormat.setGenerateWithImageMagic(generateThumbnailsWithImageMagic);
+            thumbnailsStorageFormat.setImageMagicParams(thumbnailsGenerationImageMagicParams);
             thumbnailsStorageFormat.setHighQuality(generateThumbnailsInHighQuality);
             imageStorageConfiguraton.setThumbnailsStorageFormat(thumbnailsStorageFormat);
         }
@@ -284,14 +289,30 @@ abstract public class SimpleImageDataConfig
     }
 
     /**
-     * if true ImageMagic 'convert' utility will be used to generate thumbnails. It should be
-     * installed and accessible.
+     * Decides if ImageMagic 'convert' utility will be used to generate thumbnails. True by default.
+     * <p>
+     * The tool should be installed and accessible, otherwise set this option to false and set
+     * {@link #setAllowedMachineLoadDuringThumbnailsGeneration(double)} to
+     * 1/numberOfYourProcessorCores. Internal library will be used to generate thumbnails, but it is
+     * not able to generate thumbnails in parallel.
      */
     public void setUseImageMagicToGenerateThumbnails(boolean generateWithImageMagic)
     {
         this.generateThumbnailsWithImageMagic = generateWithImageMagic;
     }
 
+    /**
+     * Sets additional parameters which should be passed to ImageMagic 'convert' utility when it is
+     * used to generate thumbnails.
+     * <p>
+     * Example: pass "-contrast-stretch 2%" to discard 2% of brightest and darkest pixels in the
+     * thumbnails.
+     */
+    public void setThumbnailsGenerationImageMagicParams(String[] imageMagicParams)
+    {
+        this.thumbnailsGenerationImageMagicParams = Arrays.asList(imageMagicParams);
+    }
+
     /**
      * if true and thumbnails generation is switched on, thumbnails will be generated with high
      * quality.
@@ -356,7 +377,7 @@ abstract public class SimpleImageDataConfig
     {
         this.imageLibraryInfoOrNull = new ImageLibraryInfo(imageLibraryName, readerName);
     }
-    
+
     /**
      * Sets the image library to be used for reading images. Available libraries are: IJ, ImageIO,
      * JAI, and BioFormats. The first image file is used to determine the actual reader. Note, that
@@ -366,7 +387,7 @@ abstract public class SimpleImageDataConfig
     {
         this.imageLibraryInfoOrNull = new ImageLibraryInfo(imageLibraryName);
     }
-    
+
     // --- predefined image dataset types
 
     /**
@@ -445,7 +466,7 @@ abstract public class SimpleImageDataConfig
     {
         return isMeasured;
     }
-    
+
     public boolean isMicroscopyData()
     {
         return isMicroscopy;

screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/ThumbnailsStorageFormat.java

@@ -16,6 +16,9 @@
 
 package ch.systemsx.cisd.openbis.dss.etl.dto.api.v1;
 
+import java.util.Collections;
+import java.util.List;
+
 import ch.systemsx.cisd.common.utilities.AbstractHashable;
 
 /**
@@ -45,6 +48,8 @@ public class ThumbnailsStorageFormat extends AbstractHashable
 
     private boolean generateWithImageMagic = false;
 
+    private List<String> imageMagicParams = Collections.emptyList();
+
     /**
      * Creates empty object which instructs that the thumbnails should be generated with default
      * settings. Use setters to change default behaviour (you will probably not have to).
@@ -83,6 +88,11 @@ public class ThumbnailsStorageFormat extends AbstractHashable
         return generateWithImageMagic;
     }
 
+    public List<String> getImageMagicParams()
+    {
+        return imageMagicParams;
+    }
+
     // --- setters ---
 
     /** Sets the maximum width of a thumbnail. */
@@ -124,12 +134,23 @@ public class ThumbnailsStorageFormat extends AbstractHashable
     /**
      * if true ImageMagic 'convert' utility should be installed and will be used to generate
      * thumbnails. <br>
-     * Note: if image library has been specified to handle the images, it will be ignored for
-     * thumbnails generation if convert is supposed to be used.
+     * Note: if images should be handled with a specific image library, it will be ignored for
+     * thumbnails generation if 'convert' is supposed to be used. Make sure that 'convert' can deal
+     * with your images in such a case.
      */
     public void setGenerateWithImageMagic(boolean generateWithImageMagic)
     {
         this.generateWithImageMagic = generateWithImageMagic;
     }
 
+    /**
+     * Sets additional parameters which should be passed to ImageMagic 'convert' utility when it is
+     * used to generate thumbnails. Example: pass "-contrast-stretch 2%" to discard 2% of brightest
+     * and darkest pixels in the thumbnails.
+     */
+    public void setImageMagicParams(List<String> imageMagicParams)
+    {
+        this.imageMagicParams = imageMagicParams;
+    }
+
 }