test and bug fixing of TSVViewReportingPlugin

SVN: 16320

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/AutoResolveUtils.java

@@ -148,15 +148,23 @@ public class AutoResolveUtils
             return;
         } else
         {
-            for (File f : startingPoint.listFiles(filter))
+            File[] filteredFiles = startingPoint.listFiles(filter);
+            if (filteredFiles != null)
             {
-                result.add(f);
+                for (File f : filteredFiles)
+                {
+                    result.add(f);
+                }
             }
-            for (File d : startingPoint.listFiles())
+            File[] files = startingPoint.listFiles();
+            if (files != null)
             {
-                if (d.isDirectory())
+                for (File d : files)
                 {
-                    findFiles(d, filter, result);
+                    if (d.isDirectory())
+                    {
+                        findFiles(d, filter, result);
+                    }
                 }
             }
         }

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractFileTableReportingPlugin.java

@@ -55,9 +55,9 @@ abstract public class AbstractFileTableReportingPlugin extends AbstractDatastore
 
     private static final String SEPARATOR_PROPERTY_KEY = "separator";
 
-    private static final String IGNORE_COMMENTS_PROPERTY_KEY = "ignore-comments";
+    public static final String IGNORE_COMMENTS_PROPERTY_KEY = "ignore-comments";
     
-    private static final String IGNORE_TRAILING_EMPTY_CELLS_PROPERTY_KEY = "ignore-trailing-empty-cells";
+    public static final String IGNORE_TRAILING_EMPTY_CELLS_PROPERTY_KEY = "ignore-trailing-empty-cells";
 
     // if the line starts with this character and comments should be ignored, the line is ignored
     private static final char COMMENT = '#';
@@ -147,14 +147,13 @@ abstract public class AbstractFileTableReportingPlugin extends AbstractDatastore
         {
             lines.add(reader.getValues());
         }
-        return new DatasetFileLines(file, dataset, lines);
+        return new DatasetFileLines(file, dataset.getDatasetCode(), lines, ignoreTrailingEmptyCells);
     }
 
     protected TableModel createTableModel(DatasetFileLines lines)
     {
         SimpleTableModelBuilder tableBuilder = new SimpleTableModelBuilder();
-        String[] headerTokens = lines.getHeaderTokens();
-        for (String title : headerTokens)
+        for (String title : lines.getHeaderTokens())
         {
             tableBuilder.addHeader(title);
         }
@@ -165,18 +164,6 @@ abstract public class AbstractFileTableReportingPlugin extends AbstractDatastore
             {
                 row.add(TableCellUtil.createTableCell(token));
             }
-            if (ignoreTrailingEmptyCells)
-            {
-                while (row.size() > headerTokens.length)
-                {
-                    ISerializableComparable cell = row.get(row.size() - 1);
-                    if (cell.toString().length() > 0)
-                    {
-                        break;
-                    }
-                    row.remove(row.size() - 1);
-                }
-            }
             tableBuilder.addRow(row);
         }
         return tableBuilder.getTableModel();

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/tasks/DatasetFileLines.java

@@ -20,6 +20,8 @@ import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.lang.StringUtils;
+
 import ch.systemsx.cisd.common.exceptions.UserFailureException;
 import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
 
@@ -42,6 +44,11 @@ public class DatasetFileLines
     }
 
     public DatasetFileLines(File file, String datasetCode, List<String[]> lines)
+    {
+        this(file, datasetCode, lines, false);
+    }
+    
+    public DatasetFileLines(File file, String datasetCode, List<String[]> lines, boolean ignoreTrailingEmptyCells)
     {
         this.file = file;
         if (lines.size() < 2)
@@ -54,18 +61,35 @@ public class DatasetFileLines
         dataLines = new ArrayList<String[]>(lines.size());
         for (int i = 1; i < lines.size(); i++)
         {
-            String[] dataTokens = lines.get(i);
+            String[] dataTokens = getTokens(lines.get(i), ignoreTrailingEmptyCells);
             if (headerTokens.length != dataTokens.length)
             {
                 throw UserFailureException.fromTemplate(
                         "Number of columns in header (%s) does not match number of columns "
-                                + "in %d data row (%s) in Data Set '%s' file.",
+                                + "in %d. data row (%s) in Data Set '%s' file.",
                         headerTokens.length, i, dataTokens.length, datasetCode);
             }
             dataLines.add(dataTokens);
         }
     }
 
+    private String[] getTokens(String[] dataTokens, boolean ignoreTrailingEmptyCells)
+    {
+        if (ignoreTrailingEmptyCells)
+        {
+            int indexOfLastNonEmptyCell = dataTokens.length - 1;
+            while (indexOfLastNonEmptyCell >= headerTokens.length
+                    && StringUtils.isBlank(dataTokens[indexOfLastNonEmptyCell]))
+            {
+                indexOfLastNonEmptyCell--;
+            }
+            String[] newDataTopkens = new String[indexOfLastNonEmptyCell + 1];
+            System.arraycopy(dataTokens, 0, newDataTopkens, 0, newDataTopkens.length);
+            return newDataTopkens;
+        }
+        return dataTokens;
+    }
+
     public final File getFile()
     {
         return file;

datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/TSVViewReportingPluginTest.java

+/*
+ * Copyright 2010 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard;
+
+import static ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.AbstractFileTableReportingPlugin.IGNORE_COMMENTS_PROPERTY_KEY;
+import static ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.AbstractFileTableReportingPlugin.IGNORE_TRAILING_EMPTY_CELLS_PROPERTY_KEY;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.base.tests.AbstractFileSystemTestCase;
+import ch.systemsx.cisd.common.exceptions.UserFailureException;
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.TableModel;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.TableModelRow;
+import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
+
+/**
+ * 
+ *
+ * @author Franz-Josef Elmer
+ */
+public class TSVViewReportingPluginTest extends AbstractFileSystemTestCase
+{
+    private static final String TEST_FILE = "test.txt";
+    
+    private File store;
+
+    private File dataSetInStore;
+
+    private DatasetDescription datasetDescription;
+    
+    @BeforeMethod
+    public void beforeMethod()
+    {
+        store = new File(workingDirectory, "store");
+        store.mkdirs();
+        dataSetInStore = new File(store, "dataset");
+        dataSetInStore.mkdirs();
+        datasetDescription = new DatasetDescription();
+        datasetDescription.setDatasetCode("ds1");
+        datasetDescription.setMainDataSetPattern(".*");
+        datasetDescription.setDataSetLocation(dataSetInStore.getName());
+    }
+
+    @Test
+    public void testCreateReport()
+    {
+        FileUtilities.writeToFile(new File(dataSetInStore, TEST_FILE), "a\tb\n1\t2\n\t4");
+        TSVViewReportingPlugin plugin = new TSVViewReportingPlugin(new Properties(), store);
+        TableModel tableModel = plugin.createReport(Arrays.asList(datasetDescription));
+        
+        assertEquals("[a, b]", tableModel.getHeader().toString());
+        List<TableModelRow> rows = tableModel.getRows();
+        assertEquals("[1, 2]", rows.get(0).getValues().toString());
+        assertEquals("[, 4]", rows.get(1).getValues().toString());
+        assertEquals(2, rows.size());
+    }
+    
+    @Test
+    public void testCreateReportFailingBecauseOfInvalidNumberOfCells()
+    {
+        FileUtilities.writeToFile(new File(dataSetInStore, TEST_FILE), "a\tb\n1\t2\t3\t\n");
+        Properties properties = new Properties();
+        properties.setProperty(IGNORE_TRAILING_EMPTY_CELLS_PROPERTY_KEY, "true");
+        TSVViewReportingPlugin plugin = new TSVViewReportingPlugin(properties, store);
+        
+        try
+        {
+            plugin.createReport(Arrays.asList(datasetDescription));
+            fail("UserFailureException expected");
+        } catch (UserFailureException ex)
+        {
+            assertEquals("Number of columns in header (2) does not match number of columns in "
+                    + "1. data row (3) in Data Set 'ds1' file.", ex.getMessage());
+        }
+    }
+    
+    @Test
+    public void testIgnoringCommentAndTrailingEmptyCells()
+    {
+        FileUtilities.writeToFile(new File(dataSetInStore, TEST_FILE), 
+                "a\tb\n" +
+                "#comment\n" +
+                "1\ta\t\t\n" +
+                "2\tb\t\n" +
+                "3\tc\n" +
+        "4\t\n");
+        Properties properties = new Properties();
+        properties.setProperty(IGNORE_COMMENTS_PROPERTY_KEY, "true");
+        properties.setProperty(IGNORE_TRAILING_EMPTY_CELLS_PROPERTY_KEY, "true");
+        TSVViewReportingPlugin plugin = new TSVViewReportingPlugin(properties, store);
+        TableModel tableModel = plugin.createReport(Arrays.asList(datasetDescription));
+        
+        assertEquals("[a, b]", tableModel.getHeader().toString());
+        List<TableModelRow> rows = tableModel.getRows();
+        assertEquals("[1, a]", rows.get(0).getValues().toString());
+        assertEquals("[2, b]", rows.get(1).getValues().toString());
+        assertEquals("[3, c]", rows.get(2).getValues().toString());
+        assertEquals("[4, ]", rows.get(3).getValues().toString());
+        assertEquals(4, rows.size());
+    }
+}