SP-32, BIS-14: JythonBasedAggregationServiceReportingPluginTest introduced....

SP-32, BIS-14: JythonBasedAggregationServiceReportingPluginTest introduced. Few small refactorings, improved error handling.

SVN: 25325

datastore_server/resource/test-data/JythonBasedAggregationServiceReportingPluginTest/script.py

+from ch.systemsx.cisd.openbis.generic.shared.api.v1.dto import SearchCriteria
+from ch.systemsx.cisd.openbis.generic.shared.api.v1.dto import SearchSubCriteria
+from ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.SearchCriteria import MatchClause
+from ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.SearchCriteria import MatchClauseAttribute
+
+EXPERIMENT = "Experiment"
+CODE = "Data Set Code"
+NUMBER_OF_FILES = "Number of Files"
+NUMBER_OF_PROTEINS = "Number of Proteins"
+
+def countFiles(node):
+    sum = 1
+    if node.isDirectory():
+        for child in node.getChildNodes():
+            sum = sum + countFiles(child)
+    return sum
+
+def getNumberOfProteins(dataSetCode):
+    result = queryService.select("protein-db", "select count(*) as count from proteins where data_set = ?{1}", [dataSetCode])
+    return result[0].get("count")
+
+def aggregate(parameters, tableBuilder):
+    experimentCode = parameters.get('experiment-code')
+    searchCriteria = SearchCriteria()
+    subCriteria = SearchCriteria()
+    subCriteria.addMatchClause(MatchClause.createAttributeMatch(MatchClauseAttribute.CODE, experimentCode))
+    searchCriteria.addSubCriteria(SearchSubCriteria.createExperimentCriteria(subCriteria))
+    dataSets = searchService.searchForDataSets(searchCriteria)
+    tableBuilder.addHeader(EXPERIMENT)
+    tableBuilder.addHeader(CODE)
+    tableBuilder.addHeader(NUMBER_OF_FILES)
+    tableBuilder.addHeader(NUMBER_OF_PROTEINS)
+    for dataSet in dataSets:
+        dataSetCode = dataSet.getDataSetCode()
+        content = contentProvider.getContent(dataSetCode)
+        row = tableBuilder.addRow()
+        row.setCell(EXPERIMENT, dataSet.experiment.experimentIdentifier)
+        row.setCell(CODE, dataSetCode)
+        row.setCell(NUMBER_OF_FILES, countFiles(content.rootNode))
+        row.setCell(NUMBER_OF_PROTEINS, getNumberOfProteins(dataSetCode))

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/jython/PluginScriptRunnerFactory.java

@@ -161,12 +161,12 @@ public class PluginScriptRunnerFactory implements IPluginScriptRunnerFactory
         return evaluator;
     }
 
-    private static ISearchService createSearchService()
+    protected ISearchService createSearchService()
     {
         return ServiceProvider.getSearchService();
     }
 
-    private static IDataSourceQueryService createDataSourceQueryService()
+    protected IDataSourceQueryService createDataSourceQueryService()
     {
         return ServiceProvider.getDataSourceQueryService();
     }

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/jython/Utils.java

@@ -18,7 +18,6 @@ package ch.systemsx.cisd.openbis.dss.generic.server.plugins.jython;
 
 import org.apache.log4j.Logger;
 
-import ch.systemsx.cisd.common.evaluator.EvaluatorException;
 import ch.systemsx.cisd.common.exceptions.UserFailureException;
 import ch.systemsx.cisd.common.utilities.ExceptionUtils;
 import ch.systemsx.cisd.openbis.generic.shared.basic.dto.TableModel;
@@ -39,16 +38,6 @@ class Utils
             ISimpleTableModelBuilderAdaptor builder = SimpleTableModelBuilderAdaptor.create();
             tableModelCreator.create(builder);
             return (TableModel) builder.getTableModel();
-        } catch (EvaluatorException ex)
-        {
-            if (null != ex.getCause())
-            {
-                notifyLog.error(createErrorMessage(scriptPath), ex.getCause());
-            } else
-            {
-                notifyLog.error(createErrorMessage(scriptPath), ex);
-            }
-            throw createUserFailureException(ex);
         } catch (RuntimeException ex)
         {
             notifyLog.error(createErrorMessage(scriptPath), ex);
@@ -64,7 +53,7 @@ class Utils
     private static UserFailureException createUserFailureException(RuntimeException ex)
     {
         return new UserFailureException("Chosen plugin failed to create a report: "
-                + ExceptionUtils.getEndOfChain(ex));
+                + ExceptionUtils.getEndOfChain(ex), ex);
     }
 
 }

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractAggregationServiceReportingPlugin.java

@@ -49,13 +49,19 @@ public abstract class AbstractAggregationServiceReportingPlugin extends Abstract
 
     public TableModel createReport(List<DatasetDescription> datasets, DataSetProcessingContext context)
     {
-        throw new IllegalArgumentException(
-                "The method createReport is not supported by AGGREGATION_TABLE_MODEL tasks");
+        throw createException();
     }
 
     public LinkModel createLink(DatasetDescription dataset)
     {
-        throw new IllegalArgumentException(
-                "The method createLink is not supported by AGGREGATION_TABLE_MODEL tasks");
+        throw createException();
     }
+    
+    private IllegalArgumentException createException()
+    {
+        return new IllegalArgumentException(
+                "The method createReport is not supported by " + getReportingPluginType()
+                        + " tasks");
+    }
+    
 }

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/tasks/AbstractPluginTaskFactory.java

@@ -113,7 +113,6 @@ public abstract class AbstractPluginTaskFactory<T>
             servletPropertiesManager.addServletProperties(label, props);
         }
         String pluginKey = sectionProperties.getKey();
-        String[] datasetCodes = extractDatasetCodes(pluginProperties);
 
         this.className = PropertyUtils.getMandatoryProperty(pluginProperties, CLASS_PROPERTY_NAME);
         this.instanceParameters = extractInstanceParameters(pluginProperties);
@@ -124,11 +123,15 @@ public abstract class AbstractPluginTaskFactory<T>
         {
             ReportingPluginType type =
                     ((IReportingPluginTask) pluginInstance).getReportingPluginType();
+            String[] datasetCodes =
+                    type == ReportingPluginType.AGGREGATION_TABLE_MODEL ? new String[0]
+                            : extractDatasetCodes(pluginProperties);
             this.description =
                     DatastoreServiceDescription.reporting(pluginKey, label, datasetCodes,
                             datastoreCode, type);
         } else
         {
+            String[] datasetCodes = extractDatasetCodes(pluginProperties);
             this.description =
                     DatastoreServiceDescription.processing(pluginKey, label, datasetCodes,
                             datastoreCode);

datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/jython/JythonBasedAggregationServiceReportingPluginTest.java

+/*
+ * Copyright 2012 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.server.plugins.jython;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import net.lemnik.eodsql.DataSet;
+
+import org.jmock.Expectations;
+import org.jmock.Mockery;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.base.tests.AbstractFileSystemTestCase;
+import ch.systemsx.cisd.common.exceptions.UserFailureException;
+import ch.systemsx.cisd.common.io.hierarchical_content.api.IHierarchicalContent;
+import ch.systemsx.cisd.common.io.hierarchical_content.api.IHierarchicalContentNode;
+import ch.systemsx.cisd.common.mail.IMailClient;
+import ch.systemsx.cisd.common.test.RecordingMatcher;
+import ch.systemsx.cisd.etlserver.registrator.api.v1.impl.DataSetImmutable;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.IReportingPluginTask;
+import ch.systemsx.cisd.openbis.dss.generic.shared.DataSetProcessingContext;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.api.internal.v1.IDataSourceQueryService;
+import ch.systemsx.cisd.openbis.dss.generic.shared.api.internal.v1.ISearchService;
+import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.SearchCriteria;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.TableModel;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.builders.DataSetBuilder;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.builders.ExperimentBuilder;
+
+/**
+ * 
+ *
+ * @author Franz-Josef Elmer
+ */
+public class JythonBasedAggregationServiceReportingPluginTest extends AbstractFileSystemTestCase
+{
+    private Mockery context;
+    private ISearchService searchService;
+    private IDataSourceQueryService queryService;
+    private File store;
+    private File scriptFolder;
+    private DataSetProcessingContext processingContext;
+    private IMailClient mailClient;
+    private IHierarchicalContentProvider contentProvider;
+    private IHierarchicalContent content;
+    private IHierarchicalContentNode rootNode;
+    private DataSet<?> dbDataSet;
+
+    @BeforeMethod
+    public void setUpTest()
+    {
+        context = new Mockery();
+        searchService = context.mock(ISearchService.class);
+        queryService = context.mock(IDataSourceQueryService.class);
+        mailClient = context.mock(IMailClient.class);
+        contentProvider = context.mock(IHierarchicalContentProvider.class);
+        content = context.mock(IHierarchicalContent.class);
+        rootNode = context.mock(IHierarchicalContentNode.class);
+        dbDataSet = context.mock(DataSet.class);
+        processingContext =
+                new DataSetProcessingContext(contentProvider, null, new HashMap<String, String>(),
+                        mailClient, "test-user");
+        store = new File(workingDirectory, "store");
+        store.mkdirs();
+        scriptFolder = new File("resource/test-data/" + getClass().getSimpleName());
+    }
+
+    @AfterMethod
+    public void afterTest()
+    {
+        context.assertIsSatisfied();
+    }
+
+    @Test
+    public void testHappyCase()
+    {
+        final RecordingMatcher<SearchCriteria> searchCriteriaRecorder =
+                new RecordingMatcher<SearchCriteria>();
+        context.checking(new Expectations()
+            {
+                {
+                    one(searchService).searchForDataSets(with(searchCriteriaRecorder));
+                    will(returnValue(Arrays.asList(new DataSetImmutable(new DataSetBuilder()
+                            .code("ds1")
+                            .experiment(
+                                    new ExperimentBuilder().identifier("/A/B/ABC").getExperiment())
+                            .getDataSet(), null))));
+
+                    one(contentProvider).asContent("ds1");
+                    will(returnValue(content));
+
+                    one(content).getRootNode();
+                    will(returnValue(rootNode));
+
+                    one(rootNode).isDirectory();
+                    will(returnValue(false));
+
+                    one(content).close();
+
+                    one(queryService).select("protein-db",
+                            "select count(*) as count from proteins where data_set = ?{1}", "ds1");
+                    will(returnValue(dbDataSet));
+                    
+                    one(dbDataSet).size();
+                    will(returnValue(1));
+                    
+                    one(dbDataSet).get(0);
+                    will(returnValue(new ParametersBuilder().parameter("count", 42L).getParameters()));
+                }
+            });
+        Map<String, Object> parameters =
+                new ParametersBuilder().parameter("experiment-code", "ABC").getParameters();
+
+        IReportingPluginTask plugin = createPlugin("script.py");
+        TableModel tableModel = plugin.createAggregationReport(parameters, processingContext);
+
+        assertEquals("SearchCriteria[MATCH_ALL_CLAUSES,[],"
+                + "[SearchSubCriteria[EXPERIMENT,SearchCriteria[MATCH_ALL_CLAUSES,"
+                + "[SearchCriteria.AttributeMatchClause[ATTRIBUTE,CODE,ABC,EQUALS]],[]]]]]",
+                searchCriteriaRecorder.recordedObject().toString());
+        assertEquals("[Experiment, Data Set Code, Number of Files, Number of Proteins]", tableModel
+                .getHeader().toString());
+        assertEquals("[/A/B/ABC, ds1, 1, 42]", tableModel.getRows().get(0).getValues().toString());
+        assertEquals(1, tableModel.getRows().size());
+        context.assertIsSatisfied();
+    }
+    
+    
+    @Test
+    public void testSearchServiceThrowsException()
+    {
+        final RecordingMatcher<SearchCriteria> searchCriteriaRecorder =
+                new RecordingMatcher<SearchCriteria>();
+        context.checking(new Expectations()
+            {
+                {
+                    one(searchService).searchForDataSets(with(searchCriteriaRecorder));
+                    will(throwException(new IllegalArgumentException("Invalid")));
+                }
+            });
+        Map<String, Object> parameters =
+                new ParametersBuilder().getParameters();
+        
+        IReportingPluginTask plugin = createPlugin("script.py");
+        try
+        {
+            plugin.createAggregationReport(parameters, processingContext);
+            fail("UserFailureException expected");
+        } catch (UserFailureException ex)
+        {
+            assertEquals("Chosen plugin failed to create a report: "
+                    + "java.lang.IllegalArgumentException: Invalid", ex.getMessage());
+            String message = ex.getCause().getMessage();
+            String prefix =
+                    "Error occured in line 28 of the script when evaluating 'aggregate({}, ";
+            assertEquals("Message '" + message + "' doesn't starts with '" + prefix + "'.", true,
+                    message.startsWith(prefix));
+        }
+        
+        assertEquals("SearchCriteria[MATCH_ALL_CLAUSES,[],"
+                + "[SearchSubCriteria[EXPERIMENT,SearchCriteria[MATCH_ALL_CLAUSES,"
+                + "[SearchCriteria.AttributeMatchClause[ATTRIBUTE,CODE,<null>,EQUALS]],[]]]]]",
+                searchCriteriaRecorder.recordedObject().toString());
+        context.assertIsSatisfied();
+    }
+    
+    private static final class ParametersBuilder
+    {
+        private final Map<String, Object> parameters = new HashMap<String, Object>();
+        
+        public ParametersBuilder parameter(String name, Object value)
+        {
+            parameters.put(name, value);
+            return this;
+        }
+        
+        public Map<String, Object> getParameters()
+        {
+            return parameters;
+        }
+    }
+    
+    private IReportingPluginTask createPlugin(String scriptFile)
+    {
+        return new JythonBasedAggregationServiceReportingPlugin(new Properties(), store,
+                new PluginScriptRunnerFactory(new File(scriptFolder, scriptFile).getPath())
+                    {
+                        private static final long serialVersionUID = 1L;
+
+                        @Override
+                        protected ISearchService createSearchService()
+                        {
+                            return searchService;
+                        }
+
+                        @Override
+                        protected IDataSourceQueryService createDataSourceQueryService()
+                        {
+                            return queryService;
+                        }
+                    });
+    }
+
+}