LMS-1420 improve TSV plugins (comments, separator)

SVN: 14977

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractDataMergingReportingPlugin.java

@@ -17,22 +17,17 @@
 package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard;
 
 import java.io.File;
-import java.io.FileReader;
 import java.io.IOException;
-import java.io.Reader;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Properties;
 
-import org.apache.commons.io.IOUtils;
-
 import ch.systemsx.cisd.base.exceptions.IOExceptionUnchecked;
 import ch.systemsx.cisd.common.exceptions.UserFailureException;
 import ch.systemsx.cisd.common.filesystem.FileUtilities;
 import ch.systemsx.cisd.common.parser.ParserException;
 import ch.systemsx.cisd.common.parser.ParsingException;
 import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.DatasetFileLines;
-import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.IReportingPluginTask;
 import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.SimpleTableModelBuilder;
 import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ISerializableComparable;
 import ch.systemsx.cisd.openbis.generic.shared.basic.dto.StringTableCell;
@@ -44,9 +39,10 @@ import ch.systemsx.cisd.openbis.generic.shared.util.TableCellUtil;
  * 
  * @author Bernd Rinn
  */
-public abstract class AbstractDataMergingReportingPlugin extends AbstractDatastorePlugin implements
-        IReportingPluginTask
+public abstract class AbstractDataMergingReportingPlugin extends AbstractFileTableReportingPlugin
 {
+    private static final long serialVersionUID = 1L;
+
     private static final String FILE_INCLUDE_PATTERN = "file-include-pattern";
 
     private static final String FILE_EXCLUDE_PATTERN = "file-exclude-pattern";
@@ -58,17 +54,15 @@ public abstract class AbstractDataMergingReportingPlugin extends AbstractDatasto
 
     private final String includePatternOrNull;
 
-    private final String separator;
-
     protected AbstractDataMergingReportingPlugin(Properties properties, File storeRoot)
     {
-        this(properties, storeRoot, "\t");
+        this(properties, storeRoot, TAB_SEPARATOR);
     }
 
     protected AbstractDataMergingReportingPlugin(Properties properties, File storeRoot,
             String separator)
     {
-        super(properties, storeRoot);
+        super(properties, storeRoot, separator);
         final String excludePatternOrNull = properties.getProperty(FILE_EXCLUDE_PATTERN);
         if (excludePatternOrNull == null)
         {
@@ -78,7 +72,6 @@ public abstract class AbstractDataMergingReportingPlugin extends AbstractDatasto
             this.excludePattern = excludePatternOrNull;
         }
         this.includePatternOrNull = properties.getProperty(FILE_INCLUDE_PATTERN);
-        this.separator = separator;
     }
 
     protected String[] getHeaderTitles(DatasetDescription dataset)
@@ -162,48 +155,6 @@ public abstract class AbstractDataMergingReportingPlugin extends AbstractDatasto
         }
     }
 
-    /**
-     * Loads {@link DatasetFileLines} from the specified tab file.
-     * 
-     * @throws IOExceptionUnchecked if a {@link IOException} has occurred.
-     */
-    protected DatasetFileLines loadFromFile(DatasetDescription dataset, final File file)
-            throws ParserException, ParsingException, IllegalArgumentException,
-            IOExceptionUnchecked
-    {
-        assert file != null : "Given file must not be null";
-        assert file.isFile() : "Given file '" + file.getAbsolutePath() + "' is not a file.";
-
-        FileReader reader = null;
-        try
-        {
-            reader = new FileReader(file);
-            return load(dataset, reader, file);
-        } catch (final IOException ex)
-        {
-            throw new IOExceptionUnchecked(ex);
-        } finally
-        {
-            IOUtils.closeQuietly(reader);
-        }
-    }
-
-    /**
-     * Loads data from the specified reader.
-     * 
-     * @throws IOException
-     */
-    @SuppressWarnings("unchecked")
-    protected DatasetFileLines load(final DatasetDescription dataset, final Reader reader,
-            final File file) throws ParserException, ParsingException, IllegalArgumentException,
-            IOException
-    {
-        assert reader != null : "Unspecified reader";
-
-        final List<String> lines = IOUtils.readLines(reader);
-        return new DatasetFileLines(file, dataset, lines, separator);
-    }
-
     protected boolean isFileExcluded(File file)
     {
         if (includePatternOrNull != null)
@@ -214,7 +165,4 @@ public abstract class AbstractDataMergingReportingPlugin extends AbstractDatasto
             return file.getName().matches(excludePattern);
         }
     }
-
-    private static final long serialVersionUID = 1L;
-
 }

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractFileTableReportingPlugin.java

+/*
+ * Copyright 2010 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.Reader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.commons.io.IOUtils;
+
+import ch.systemsx.cisd.base.exceptions.IOExceptionUnchecked;
+import ch.systemsx.cisd.common.parser.ParserException;
+import ch.systemsx.cisd.common.parser.ParsingException;
+import ch.systemsx.cisd.common.utilities.PropertyUtils;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.DatasetFileLines;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.IReportingPluginTask;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.SimpleTableModelBuilder;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ISerializableComparable;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.TableModel;
+import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
+import ch.systemsx.cisd.openbis.generic.shared.util.TableCellUtil;
+
+/**
+ * Abstract class for reporting plugins which read from CSV or TSV files.
+ * 
+ * @author Tomasz Pylak
+ */
+abstract public class AbstractFileTableReportingPlugin extends AbstractDatastorePlugin implements
+        IReportingPluginTask
+{
+    protected static final String TAB_SEPARATOR = "\t";
+
+    protected static final String SEMICOLON_SEPARATOR = "\t";
+
+    private static final long serialVersionUID = 1L;
+
+    private static final String SEPARATOR_PROPERTY_KEY = "separator";
+
+    private static final String IGNORE_COMMENTS_PROPERTY_KEY = "ignore-comments";
+
+    // if the line starts with this character and comments should be ignored, the line is ignored
+    private static final String COMMENT = "#";
+
+    private final String separator;
+
+    private final boolean ignoreComments;
+
+    protected AbstractFileTableReportingPlugin(Properties properties, File storeRoot,
+            String defaultSeparator)
+    {
+        super(properties, storeRoot);
+        this.separator =
+                PropertyUtils.getProperty(properties, SEPARATOR_PROPERTY_KEY, defaultSeparator);
+        this.ignoreComments =
+                PropertyUtils.getBoolean(properties, IGNORE_COMMENTS_PROPERTY_KEY, true);
+
+    }
+
+    /**
+     * Loads {@link DatasetFileLines} from the specified tab file.
+     * 
+     * @throws IOExceptionUnchecked if a {@link IOException} has occurred.
+     */
+    protected DatasetFileLines loadFromFile(DatasetDescription dataset, final File file)
+            throws ParserException, ParsingException, IllegalArgumentException,
+            IOExceptionUnchecked
+    {
+        assert file != null : "Given file must not be null";
+        assert file.isFile() : "Given file '" + file.getAbsolutePath() + "' is not a file.";
+
+        FileReader reader = null;
+        try
+        {
+            reader = new FileReader(file);
+            return load(dataset, reader, file);
+        } catch (final IOException ex)
+        {
+            throw new IOExceptionUnchecked(ex);
+        } finally
+        {
+            IOUtils.closeQuietly(reader);
+        }
+    }
+
+    /**
+     * Loads data from the specified reader.
+     * 
+     * @throws IOException
+     */
+    @SuppressWarnings("unchecked")
+    protected DatasetFileLines load(final DatasetDescription dataset, final Reader reader,
+            final File file) throws ParserException, ParsingException, IllegalArgumentException,
+            IOException
+    {
+        assert reader != null : "Unspecified reader";
+
+        List<String> lines = IOUtils.readLines(reader);
+        if (ignoreComments)
+        {
+            lines = filterCommentedLines(lines);
+        }
+        return new DatasetFileLines(file, dataset, lines, separator);
+    }
+
+    private static List<String> filterCommentedLines(List<String> lines)
+    {
+        List<String> result = new ArrayList<String>();
+        for (String line : lines)
+        {
+            if (line.trim().startsWith(COMMENT) == false)
+            {
+                result.add(line);
+            }
+        }
+        return result;
+    }
+
+    protected static TableModel createTableModel(DatasetFileLines lines)
+    {
+        SimpleTableModelBuilder tableBuilder = new SimpleTableModelBuilder();
+        for (String title : lines.getHeaderTokens())
+        {
+            tableBuilder.addHeader(title);
+        }
+        for (String[] line : lines.getDataLines())
+        {
+            List<ISerializableComparable> row = new ArrayList<ISerializableComparable>();
+            for (String token : line)
+            {
+                row.add(TableCellUtil.createTableCell(token));
+            }
+            tableBuilder.addRow(row);
+        }
+        return tableBuilder.getTableModel();
+    }
+}

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/TSVViewReportingPlugin.java

@@ -17,33 +17,28 @@
 package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard;
 
 import java.io.File;
-import java.util.ArrayList;
 import java.util.List;
 import java.util.Properties;
 
 import ch.systemsx.cisd.common.exceptions.UserFailureException;
 import ch.systemsx.cisd.openbis.dss.generic.server.AutoResolveUtils;
 import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.DatasetFileLines;
-import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.IReportingPluginTask;
-import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.SimpleTableModelBuilder;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ISerializableComparable;
 import ch.systemsx.cisd.openbis.generic.shared.basic.dto.TableModel;
 import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
-import ch.systemsx.cisd.openbis.generic.shared.util.TableCellUtil;
 
 /**
- * Shows the data as a table if one 'main data set' file exists and is a 'tsv' file.
+ * Shows the data as a table if one 'main data set' file exists, matches the selected dataset
+ * pattern and is a 'tsv' file.
  * 
  * @author Izabela Adamczyk
  */
-public class TSVViewReportingPlugin extends AbstractDataMergingReportingPlugin implements
-        IReportingPluginTask
+public class TSVViewReportingPlugin extends AbstractFileTableReportingPlugin
 {
     private static final long serialVersionUID = 1L;
 
     public TSVViewReportingPlugin(Properties properties, File storeRoot)
     {
-        super(properties, storeRoot);
+        super(properties, storeRoot, TAB_SEPARATOR);
     }
 
     public TableModel createReport(List<DatasetDescription> datasets)
@@ -56,22 +51,8 @@ public class TSVViewReportingPlugin extends AbstractDataMergingReportingPlugin i
                         root);
         if (fileToOpenOrNull != null && fileToOpenOrNull.isFile() && fileToOpenOrNull.exists())
         {
-            SimpleTableModelBuilder tableBuilder = new SimpleTableModelBuilder();
             DatasetFileLines lines = loadFromFile(dataset, fileToOpenOrNull);
-            for (String title : lines.getHeaderTokens())
-            {
-                tableBuilder.addHeader(title);
-            }
-            for (String[] line : lines.getDataLines())
-            {
-                List<ISerializableComparable> row = new ArrayList<ISerializableComparable>();
-                for (String token : line)
-                {
-                    row.add(TableCellUtil.createTableCell(token));
-                }
-                tableBuilder.addRow(row);
-            }
-            return tableBuilder.getTableModel();
+            return createTableModel(lines);
         }
         throw UserFailureException.fromTemplate("Main TSV file could not be found.");
     }

openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/basic/dto/TableModelColumnHeader.java

@@ -99,4 +99,10 @@ public class TableModelColumnHeader implements IsSerializable, Serializable
     {
         this.entityKindOrNull = entityKind;
     }
+
+    @Override
+    public String toString()
+    {
+        return title;
+    }
 }
\ No newline at end of file

screening/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/ImageAnalysisMergedRowsReportingPlugin.java

@@ -17,7 +17,6 @@
 package ch.systemsx.cisd.openbis.dss.generic.server.plugins;
 
 import java.io.File;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Properties;
@@ -28,11 +27,7 @@ import ch.systemsx.cisd.common.exceptions.UserFailureException;
 import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.AbstractDataMergingReportingPlugin;
 import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.DatasetFileLines;
 import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.SimpleTableModelBuilder;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DoubleTableCell;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ISerializableComparable;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IntegerTableCell;
 import ch.systemsx.cisd.openbis.generic.shared.basic.dto.TableModel;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.TableModelRow;
 import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
 
 /**
@@ -51,57 +46,10 @@ public class ImageAnalysisMergedRowsReportingPlugin extends AbstractDataMergingR
 
     public ImageAnalysisMergedRowsReportingPlugin(Properties properties, File storeRoot)
     {
-        super(properties, storeRoot, ";");
+        super(properties, storeRoot, SEMICOLON_SEPARATOR);
     }
 
     public TableModel createReport(List<DatasetDescription> datasets)
-    {
-        TableModel stringModel = createStringModel(datasets);
-        List<TableModelRow> typedRows = asTypedRows(stringModel.getRows());
-        return new TableModel(stringModel.getHeader(), typedRows);
-    }
-
-    private static List<TableModelRow> asTypedRows(List<TableModelRow> rows)
-    {
-        List<TableModelRow> typedRows = new ArrayList<TableModelRow>();
-        for (TableModelRow row : rows)
-        {
-            typedRows.add(asTypedRow(row));
-        }
-        return typedRows;
-    }
-
-    private static TableModelRow asTypedRow(TableModelRow row)
-    {
-        List<ISerializableComparable> typedValues = new ArrayList<ISerializableComparable>();
-        for (ISerializableComparable value : row.getValues())
-        {
-            typedValues.add(asTypedValue(value));
-        }
-        return new TableModelRow(typedValues);
-    }
-
-    private static ISerializableComparable asTypedValue(ISerializableComparable value)
-    {
-        String text = value.toString();
-        try
-        {
-            Double doubleValue = Double.parseDouble(text);
-            if (doubleValue.intValue() == doubleValue)
-            {
-                return new IntegerTableCell(doubleValue.intValue());
-            } else
-            {
-                return new DoubleTableCell(doubleValue);
-            }
-        } catch (NumberFormatException e)
-        {
-            return value;
-        }
-    }
-
-    // all columns will be strings
-    private TableModel createStringModel(List<DatasetDescription> datasets)
     {
         SimpleTableModelBuilder builder = new SimpleTableModelBuilder();
         builder.addHeader("Data Set Code");

screening/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/ImageAnalysisMergedRowsReportingPluginTest.java

+/*
+ * Copyright 2009 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.server.plugins;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.commons.lang.StringUtils;
+import org.testng.AssertJUnit;
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.common.exceptions.UserFailureException;
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.IReportingPluginTask;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.TableModel;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.TableModelRow;
+import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
+
+/**
+ * Test cases for the {@link ImageAnalysisMergedRowsReportingPlugin}.
+ * 
+ * @author Tomasz Pylak
+ */
+public class ImageAnalysisMergedRowsReportingPluginTest extends AssertJUnit
+{
+
+    private final static File dir =
+            new File("targets/unit-test-wd/ImageAnalysisMergedRowsReportingPluginTest");
+
+    @Test
+    public void testMerge()
+    {
+        String separator = ",";
+        Properties props = createProperties(separator, true);
+        TableModel model = createReport(separator, props);
+
+        assertEquals(4, model.getHeader().size());
+        assertEquals("Data Set Code#Plate#key#val", StringUtils.join(model.getHeader(), '#'));
+
+        List<TableModelRow> rows = model.getRows();
+        assertEquals(2, rows.size());
+        assertEquals("datasetCode-a#fa.txt#one#1", StringUtils.join(rows.get(0).getValues(), '#'));
+        assertEquals("datasetCode-b#fb.txt#two#2", StringUtils.join(rows.get(1).getValues(), '#'));
+    }
+
+    @Test
+    // do not ignore comments - should fail when comment is treated as a header
+    public void testMergeNotIgnoringCommentsFailed()
+    {
+
+        boolean failedBecauseOfComment = false;
+        try
+        {
+            String separator = ";";
+            Properties props = createProperties(separator, false);
+            createReport(separator, props);
+        } catch (UserFailureException ex)
+        {
+            failedBecauseOfComment = true;
+        }
+        assertTrue("should fail when comment is treated as a header in file a.txt",
+                failedBecauseOfComment);
+    }
+
+    @Test
+    // the same test, but now treat ';' as separator - two columns should be merged
+    public void testMergeCheckSeparator()
+    {
+        Properties props = createProperties(";", true);
+        TableModel model = createReport(",", props);
+
+        assertEquals(3, model.getHeader().size());
+        assertEquals(2, model.getRows().size());
+    }
+
+    private TableModel createReport(String separator, Properties props)
+    {
+        final File dirA = new File(dir, "a");
+        final File dirB = new File(dir, "b");
+        dirA.mkdirs();
+        dirB.mkdirs();
+        final File f1 = new File(dirA, "fa.txt");
+        f1.deleteOnExit();
+        final File f2 = new File(dirB, "fb.txt");
+        f2.deleteOnExit();
+        FileUtilities.writeToFile(f1, "# any comment\n" + "key" + separator + "val\n" + "one"
+                + separator + "1\n");
+        FileUtilities.writeToFile(f2, "key" + separator + "val\n" + "two" + separator + "2\n");
+
+        IReportingPluginTask plugin = new ImageAnalysisMergedRowsReportingPlugin(props, dir);
+        List<DatasetDescription> datasets =
+                Arrays.asList(createDatasetDescription(dirA.getName()),
+                        createDatasetDescription(dirB.getName()));
+        TableModel model = plugin.createReport(datasets);
+        return model;
+    }
+
+    private Properties createProperties(String separator, boolean ignoreComments)
+    {
+        Properties props = new Properties();
+        props.put("separator", separator);
+        props.put("sub-directory-name", "");
+        props.put("ignore-comments", "" + ignoreComments);
+        return props;
+    }
+
+    private DatasetDescription createDatasetDescription(String location)
+    {
+        return new DatasetDescription("datasetCode-" + location, location, "", "", "", "", null,
+                null);
+    }
+
+}

screening/sourceTest/java/tests.xml

@@ -7,6 +7,7 @@
     </groups>
     <packages>
       <package name="ch.systemsx.cisd.openbis.plugin.screening.*" />
+      <package name="ch.systemsx.cisd.openbis.dss.*" />
    </packages>
  </test>
 </suite>