SSDM-9648: first usable version

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/jython/JythonBasedProcessingPlugin.java

@@ -52,7 +52,7 @@ public class JythonBasedProcessingPlugin implements IProcessingPluginTask
 
     transient IHierarchicalContentProvider hierarchicalContentProvider;
 
-    protected static String getScriptPathProperty(Properties properties)
+    public static String getScriptPathProperty(Properties properties)
     {
         return PropertyUtils.getMandatoryProperty(properties, SCRIPT_PATH);
     }

screening/source/java/ch/systemsx/cisd/openbis/dss/etl/MicroscopyThumbnailsCreationTask.java

@@ -16,29 +16,43 @@
 
 package ch.systemsx.cisd.openbis.dss.etl;
 
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.util.ArrayList;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Properties;
-import java.util.Stack;
 
 import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi;
 import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet;
 import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.fetchoptions.DataSetFetchOptions;
 import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.search.DataSetSearchCriteria;
+import ch.systemsx.cisd.common.jython.evaluator.IJythonEvaluator;
 import ch.systemsx.cisd.common.utilities.ICredentials;
+import ch.systemsx.cisd.etlserver.DefaultStorageProcessor;
 import ch.systemsx.cisd.etlserver.plugins.AbstractMaintenanceTaskWithStateFile;
+import ch.systemsx.cisd.etlserver.registrator.api.v2.IDataSet;
+import ch.systemsx.cisd.etlserver.registrator.api.v2.IDataSetRegistrationTransactionV2;
+import ch.systemsx.cisd.etlserver.registrator.api.v2.IDataSetUpdatable;
 import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContent;
-import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContentNode;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.IImageGenerationAlgorithm;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.SimpleImageContainerDataConfig;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.SimpleImageDataConfig;
 import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.ImageDataSetInformation;
 import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.ImageDataSetStructure;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.jython.JythonBasedProcessingPlugin;
+import ch.systemsx.cisd.openbis.dss.generic.shared.DataSetProcessingContext;
 import ch.systemsx.cisd.openbis.dss.generic.shared.IDataSetDirectoryProvider;
 import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
 import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.api.internal.v2.ISearchService;
+import ch.systemsx.cisd.openbis.dss.screening.server.plugins.jython.ScreeningJythonIngestionService;
+import ch.systemsx.cisd.openbis.dss.screening.server.plugins.jython.ScreeningPluginScriptRunnerFactory;
 import ch.systemsx.cisd.openbis.dss.shared.DssScreeningUtils;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.TableModel;
 import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.IImagingReadonlyQueryDAO;
-import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgAcquiredImageDTO;
-import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgContainerDTO;
-import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgImageDatasetDTO;
 
 /**
  * @author Franz-Josef Elmer
@@ -50,9 +64,12 @@ public class MicroscopyThumbnailsCreationTask extends AbstractMaintenanceTaskWit
 
     private String dataSetThumbnailType;
 
+    private Properties properties;
+
     @Override
     public void setUp(String pluginName, Properties properties)
     {
+        this.properties = properties;
         defineStateFile(properties, getDirectoryProvider().getStoreRoot());
         dataSetContainerType = properties.getProperty("data-set-container-type", "MICROSCOPY_IMG_CONTAINER");
         dataSetThumbnailType = properties.getProperty("data-set-thumbnail-type", "MICROSCOPY_IMG_THUMBNAIL");
@@ -67,42 +84,105 @@ public class MicroscopyThumbnailsCreationTask extends AbstractMaintenanceTaskWit
         searchCriteria.withRegistrationDate().thatIsLaterThanOrEqualTo(getLastRegistrationDate(new Date(0)));
         DataSetFetchOptions fetchOptions = new DataSetFetchOptions();
         fetchOptions.withComponents().withType();
+        fetchOptions.withComponents().withExperiment();
+        fetchOptions.withComponents().withSample();
         fetchOptions.sortBy().registrationDate();
         List<DataSet> containerDataSets = getService().searchDataSets(sessionToken, searchCriteria, fetchOptions).getObjects();
         for (DataSet containerDataSet : containerDataSets)
         {
-            if (hasNoThumbnails(containerDataSet))
+            if (hasNoThumbnails(containerDataSet) && containerDataSet.getComponents().isEmpty() == false)
             {
-                String containerCode = containerDataSet.getCode();
-                System.err.println("has no thumbnails: " + containerCode);
-                IImagingReadonlyQueryDAO imageDb = getImageDb();
-                ImageDataSetStructure imageDataSetStructure = Utils.getImageDataSetStructure(imageDb, containerCode);
-                ImageDataSetInformation imageDataSetInformation = new ImageDataSetInformation();
-                imageDataSetInformation.setImageDataSetStructure(imageDataSetStructure);
-                
-                System.err.println(imageDataSetStructure);
-                IHierarchicalContent content = getHierarchicalContentProvider().asContent(containerCode);
-                IHierarchicalContentNode rootNode = content.getRootNode();
-                printFiles(rootNode, "");
+                cerateThumbnailDataSet(sessionToken, containerDataSet);
             }
-//            updateTimeStampFile(renderTimeStampAndCode(containerDataSet.getRegistrationDate(), containerDataSet.getCode()));
+            updateTimeStampFile(renderTimeStampAndCode(containerDataSet.getRegistrationDate(), containerDataSet.getCode()));
         }
-        
     }
-    
-    private void printFiles(IHierarchicalContentNode node, String indentation)
+
+    private void cerateThumbnailDataSet(String sessionToken, DataSet containerDataSet)
     {
-        System.out.println(indentation+node.getName());
-        if (node.isDirectory())
+        String containerCode = containerDataSet.getCode();
+        IImagingReadonlyQueryDAO imageDb = getImageDb();
+        ImageDataSetStructure imageDataSetStructure = Utils.getImageDataSetStructure(imageDb, containerCode);
+        SimpleImageContainerDataConfig config = new SimpleImageContainerDataConfig();
+
+        ScreeningPluginScriptRunnerFactory scriptRunnerFactory = createScriptRunner(imageDataSetStructure, config);
+        Properties ingestionServiceProperties = createIngestionServiceProperties();
+        File storeRoot = getDirectoryProvider().getStoreRoot();
+        ScreeningJythonIngestionService ingestionService =
+                new ScreeningJythonIngestionService(ingestionServiceProperties, storeRoot, scriptRunnerFactory)
+                    {
+                        private static final long serialVersionUID = 1L;
+
+                        @Override
+                        public TableModel process(IDataSetRegistrationTransactionV2 transaction,
+                                Map<String, Object> parameters, DataSetProcessingContext context)
+                        {
+                            // Populate object 'config' by the jython script based on 'imageDataSetStructure'
+                            super.process(transaction, parameters, context);
+                            composeThumbnailDataSet(transaction, containerDataSet, imageDataSetStructure, config, context);
+                            return null;
+                        }
+                    };
+        IHierarchicalContentProvider contentProvider = getHierarchicalContentProvider();
+        DataSetProcessingContext context =
+                new DataSetProcessingContext(contentProvider, null, null, null, null, null, sessionToken);
+        ingestionService.createAggregationReport(new HashMap<String, Object>(), context);
+    }
+
+    private void composeThumbnailDataSet(IDataSetRegistrationTransactionV2 transaction, DataSet containerDataSet,
+            ImageDataSetStructure imageDataSetStructure, SimpleImageDataConfig config, DataSetProcessingContext context)
+    {
+        String containerCode = containerDataSet.getCode();
+        IDataSetUpdatable container = transaction.getDataSetForUpdate(containerCode);
+        DataSet mainDataSet = containerDataSet.getComponents().get(0);
+        IImageGenerationAlgorithm imageGenerationAlgorithm = config.getImageGenerationAlgorithm();
+        if (imageGenerationAlgorithm != null)
         {
-            List<IHierarchicalContentNode> childNodes = node.getChildNodes();
-            for (IHierarchicalContentNode childNode : childNodes)
-            {
-                printFiles(childNode, indentation + "  ");
-            }
+            List<IDataSet> thumbnailDatasets = new ArrayList<>();
+            IImageProvider imageProvider = new ImageCache();
+            IHierarchicalContent content = context.getHierarchicalContentProviderUnfiltered().asContent(containerCode);
+            imageGenerationAlgorithm.setContent(content);
+            ImageDataSetInformation imageDataSetInformation = new ImageDataSetInformation();
+            imageDataSetInformation.setImageDataSetStructure(imageDataSetStructure);
+            List<BufferedImage> images = imageGenerationAlgorithm.generateImages(imageDataSetInformation,
+                    thumbnailDatasets, imageProvider);
+            IDataSet dataSet = Utils.createDataSetAndImageFiles(transaction, imageGenerationAlgorithm, images);
+            List<String> components = new ArrayList<>(container.getContainedDataSetCodes());
+            components.add(dataSet.getDataSetCode());
+            container.setContainedDataSetCodes(components);
+            ISearchService searchService = transaction.getSearchService();
+            dataSet.setExperiment(searchService.getExperimentByPermId(mainDataSet.getExperiment().getPermId().getPermId()));
+            dataSet.setSample(searchService.getSampleByPermId(mainDataSet.getSample().getPermId().getPermId()));
         }
     }
-    
+
+    private Properties createIngestionServiceProperties()
+    {
+        Properties ingestionServiceProperties = new Properties(properties);
+        ingestionServiceProperties.setProperty(DefaultStorageProcessor.DO_NOT_CREATE_ORIGINAL_DIR_KEY, "true");
+        return ingestionServiceProperties;
+    }
+
+    private ScreeningPluginScriptRunnerFactory createScriptRunner(ImageDataSetStructure imageDataSetStructure, SimpleImageContainerDataConfig config)
+    {
+        ScreeningPluginScriptRunnerFactory scriptRunnerFactory = new ScreeningPluginScriptRunnerFactory(
+                JythonBasedProcessingPlugin.getScriptPathProperty(properties))
+            {
+
+                private static final long serialVersionUID = 1L;
+
+                @Override
+                protected IJythonEvaluator createEvaluator(String scriptString, String[] jythonPath, DataSetProcessingContext context)
+                {
+                    IJythonEvaluator evaluator = super.createEvaluator(scriptString, jythonPath, context);
+                    evaluator.set("image_data_set_structure", imageDataSetStructure);
+                    evaluator.set("image_config", config);
+                    return evaluator;
+                }
+            };
+        return scriptRunnerFactory;
+    }
+
     private boolean hasNoThumbnails(DataSet containerDataSet)
     {
         List<DataSet> components = containerDataSet.getComponents();
@@ -131,12 +211,12 @@ public class MicroscopyThumbnailsCreationTask extends AbstractMaintenanceTaskWit
     {
         return ServiceProvider.getDataStoreService().getDataSetDirectoryProvider();
     }
-    
+
     protected IHierarchicalContentProvider getHierarchicalContentProvider()
     {
         return ServiceProvider.getHierarchicalContentProvider();
     }
-    
+
     protected IImagingReadonlyQueryDAO getImageDb()
     {
         return DssScreeningUtils.getQuery();

screening/source/java/ch/systemsx/cisd/openbis/dss/etl/Utils.java

@@ -18,23 +18,32 @@ package ch.systemsx.cisd.openbis.dss.etl;
 
 import java.awt.Dimension;
 import java.awt.image.BufferedImage;
+import java.io.File;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
+import javax.imageio.ImageIO;
+
 import org.apache.log4j.Logger;
 
+import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
 import ch.systemsx.cisd.common.logging.LogCategory;
 import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.etlserver.registrator.api.v2.IDataSet;
+import ch.systemsx.cisd.etlserver.registrator.api.v2.IDataSetRegistrationTransactionV2;
 import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContentNode;
 import ch.systemsx.cisd.openbis.dss.etl.dto.ImageLibraryInfo;
 import ch.systemsx.cisd.openbis.dss.etl.dto.api.Channel;
 import ch.systemsx.cisd.openbis.dss.etl.dto.api.ChannelColorRGB;
+import ch.systemsx.cisd.openbis.dss.etl.dto.api.IImageGenerationAlgorithm;
 import ch.systemsx.cisd.openbis.dss.etl.dto.api.ImageFileInfo;
 import ch.systemsx.cisd.openbis.dss.etl.dto.api.ImageStorageConfiguraton;
 import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.ImageDataSetStructure;
 import ch.systemsx.cisd.openbis.dss.etl.dto.api.transformations.ImageTransformation;
 import ch.systemsx.cisd.openbis.dss.generic.shared.dto.Size;
 import ch.systemsx.cisd.openbis.dss.generic.shared.utils.ImageUtil;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetKind;
 import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.Geometry;
 import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.IImagingReadonlyQueryDAO;
 import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgAcquiredImageEnrichedDTO;
@@ -210,4 +219,24 @@ public class Utils
         String imageLibraryName = dataset.getImageLibraryName();
         return imageLibraryName == null ? null : new ImageLibraryInfo(imageLibraryName, dataset.getImageReaderName());
     }
+
+    public static IDataSet createDataSetAndImageFiles(IDataSetRegistrationTransactionV2 transaction, IImageGenerationAlgorithm algorithm, List<BufferedImage> images)
+    {
+        IDataSet representative = transaction.createNewDataSet(algorithm.getDataSetTypeCode(), DataSetKind.PHYSICAL);
+        for (int i = 0; i < images.size(); i++)
+        {
+            BufferedImage imageData = images.get(i);
+            String imageFile = transaction.createNewFile(representative, algorithm.getImageFileName(i));
+            File f = new File(imageFile);
+            try
+            {
+                ImageIO.write(imageData, "png", f);
+            } catch (IOException e)
+            {
+                throw new EnvironmentFailureException("Can not save representative thumbnail to file '"
+                        + f + "': " + e, e);
+            }
+        }
+        return representative;
+    }
 }

screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/IImageGenerationAlgorithm.java

@@ -4,6 +4,7 @@ import java.awt.image.BufferedImage;
 import java.util.List;
 
 import ch.systemsx.cisd.etlserver.registrator.api.v2.IDataSet;
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContent;
 import ch.systemsx.cisd.openbis.dss.etl.IImageProvider;
 import ch.systemsx.cisd.openbis.dss.etl.dto.api.impl.ImageDataSetInformation;
 
@@ -30,4 +31,9 @@ public interface IImageGenerationAlgorithm
      * {@link #generateImages(ImageDataSetInformation, List, IImageProvider)}. Note, all file names generated by this method have to be different.
      */
     public String getImageFileName(int index);
+
+    /**
+     * Sets content which is the source of images. If this isn't set {@link ImageDataSetInformation#getIncomingDirectory()} is used. 
+     */
+    public void setContent(IHierarchicalContent content);
 }

screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/EmptyImageCreationAlgorithm.java

@@ -6,6 +6,7 @@ import java.util.Collections;
 import java.util.List;
 
 import ch.systemsx.cisd.etlserver.registrator.api.v2.IDataSet;
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContent;
 import ch.systemsx.cisd.openbis.dss.etl.IImageProvider;
 import ch.systemsx.cisd.openbis.dss.etl.dto.api.IImageGenerationAlgorithm;
 import ch.systemsx.cisd.openbis.generic.shared.IServer;
@@ -33,4 +34,9 @@ public class EmptyImageCreationAlgorithm implements IImageGenerationAlgorithm, S
     {
         throw new UnsupportedOperationException();
     }
+
+    @Override
+    public void setContent(IHierarchicalContent content)
+    {
+    }
 }

screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/impl/MaximumIntensityProjectionGenerationAlgorithm.java

@@ -66,6 +66,8 @@ public class MaximumIntensityProjectionGenerationAlgorithm implements IImageGene
     private int height;
 
     private boolean useThumbnails;
+    
+    private IHierarchicalContent content;
 
     /**
      * Creates an instance for the specified data set type. The generated image will have the same size as the original images. The file name will be
@@ -234,8 +236,23 @@ public class MaximumIntensityProjectionGenerationAlgorithm implements IImageGene
     BufferedImage loadImage(IImageProvider imageProvider, File incomingDirectory, String imagePath,
             String identifier, ImageLibraryInfo library)
     {
-        File file = new File(incomingDirectory, imagePath);
-        return imageProvider.getImage(new FileBasedContentNode(file), identifier, library);
+        IHierarchicalContentNode contentNode = createContentNode(incomingDirectory, imagePath);
+        return imageProvider.getImage(contentNode, identifier, library);
+    }
+
+    private IHierarchicalContentNode createContentNode(File incomingDirectory, String imagePath)
+    {
+        if (content == null)
+        {
+            return new FileBasedContentNode(new File(incomingDirectory, imagePath));
+        }
+        return content.getNode(imagePath);
+    }
+
+    @Override
+    public void setContent(IHierarchicalContent content)
+    {
+        this.content = content;
     }
 
     /**

screening/source/java/ch/systemsx/cisd/openbis/dss/etl/jython/v2/ImagingDataSetRegistrationTransaction.java

@@ -32,10 +32,7 @@ import java.util.List;
 import java.util.Properties;
 import java.util.Set;
 
-import javax.imageio.ImageIO;
-
 import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
-import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
 import ch.systemsx.cisd.common.exceptions.UserFailureException;
 import ch.systemsx.cisd.common.properties.PropertyUtils;
 import ch.systemsx.cisd.etlserver.registrator.DataSetRegistrationDetails;
@@ -51,6 +48,7 @@ import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchical
 import ch.systemsx.cisd.openbis.dss.Constants;
 import ch.systemsx.cisd.openbis.dss.etl.Hdf5ThumbnailGenerator;
 import ch.systemsx.cisd.openbis.dss.etl.ImageCache;
+import ch.systemsx.cisd.openbis.dss.etl.Utils;
 import ch.systemsx.cisd.openbis.dss.etl.dto.ImageLibraryInfo;
 import ch.systemsx.cisd.openbis.dss.etl.dto.api.IImageGenerationAlgorithm;
 import ch.systemsx.cisd.openbis.dss.etl.dto.api.ImageFileInfo;
@@ -394,21 +392,7 @@ public class ImagingDataSetRegistrationTransaction extends DataSetRegistrationTr
             List<BufferedImage> images = algorithm.generateImages(imageDataSetInformation, thumbnailDatasets, imageCache);
             if (images.size() > 0)
             {
-                IDataSet representative = createNewDataSet(algorithm.getDataSetTypeCode(), DataSetKind.PHYSICAL);
-                for (int i = 0; i < images.size(); i++)
-                {
-                    BufferedImage imageData = images.get(i);
-                    String imageFile = createNewFile(representative, algorithm.getImageFileName(i));
-                    File f = new File(imageFile);
-                    try
-                    {
-                        ImageIO.write(imageData, "png", f);
-                    } catch (IOException e)
-                    {
-                        throw new EnvironmentFailureException("Can not save representative thumbnail to file '"
-                                + f + "': " + e, e);
-                    }
-                }
+                IDataSet representative = Utils.createDataSetAndImageFiles(this, algorithm, images);
                 containedDataSetCodes.add(representative.getDataSetCode());
                 thumbnailDatasets.add(representative);
             }

screening/source/java/ch/systemsx/cisd/openbis/dss/screening/server/plugins/jython/ScreeningJythonIngestionService.java

@@ -53,10 +53,7 @@ public class ScreeningJythonIngestionService extends JythonIngestionService
                 getScriptPathProperty(properties)));
     }
 
-    /**
-     * Constructor used in tests.
-     */
-    protected ScreeningJythonIngestionService(Properties properties, File storeRoot,
+    public ScreeningJythonIngestionService(Properties properties, File storeRoot,
             IPluginScriptRunnerFactory scriptRunnerFactory)
     {
         super(properties, storeRoot, scriptRunnerFactory);