From 9fdfd8966657fa0ad403d369eb0f275795f8f12c Mon Sep 17 00:00:00 2001
From: tpylak <tpylak>
Date: Tue, 25 Aug 2009 08:29:53 +0000
Subject: [PATCH] LMS-1106 Can not import multiple type samples from an Excel
sheet
SVN: 12259
---
.../cisd/common/parser/TabFileLoader.java | 118 ++++++++++++++++--
.../cisd/common/parser/TabFileLoaderTest.java | 26 ++++
.../parser/SampleUploadSectionsParser.java | 29 ++++-
3 files changed, 159 insertions(+), 14 deletions(-)
diff --git a/common/source/java/ch/systemsx/cisd/common/parser/TabFileLoader.java b/common/source/java/ch/systemsx/cisd/common/parser/TabFileLoader.java
index ba4b9765f89..396a691051b 100644
--- a/common/source/java/ch/systemsx/cisd/common/parser/TabFileLoader.java
+++ b/common/source/java/ch/systemsx/cisd/common/parser/TabFileLoader.java
@@ -21,7 +21,6 @@ import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
@@ -32,6 +31,7 @@ import org.apache.commons.io.LineIterator;
import org.apache.commons.lang.StringUtils;
import ch.systemsx.cisd.base.exceptions.IOExceptionUnchecked;
+import ch.systemsx.cisd.common.exceptions.NotImplementedException;
import ch.systemsx.cisd.common.parser.filter.AlwaysAcceptLineFilter;
import ch.systemsx.cisd.common.parser.filter.ILineFilter;
@@ -69,6 +69,8 @@ import ch.systemsx.cisd.common.parser.filter.ILineFilter;
public class TabFileLoader<T>
{
+ private static final String TOKENS_SEPARATOR = "\t";
+
public static final String COMMENT_PREFIX = "#";
private final IParserObjectFactoryFactory<T> factory;
@@ -135,10 +137,9 @@ public class TabFileLoader<T>
break;
}
}
- final List<T> result = new ArrayList<T>();
if (line == null) // no lines present
{
- return result;
+ return new ArrayList<T>();
}
final String headerLine;
@@ -151,18 +152,119 @@ public class TabFileLoader<T>
}
final DefaultParser<T> parser = new DefaultParser<T>();
- final String[] tokens = StringUtils.split(headerLine, "\t");
+ final String[] tokens = StringUtils.split(headerLine, TOKENS_SEPARATOR);
+ int lastEmptyHeadersToSkip = countLastEmptyTokens(headerLine);
final int headerLength = tokens.length;
notUnique(tokens);
+
final IPropertyMapper propertyMapper = new DefaultPropertyMapper(tokens);
parser.setObjectFactory(factory.createFactory(propertyMapper));
+
+ Line firstContentLine = previousLineHasColumnHeaders ? line : null;
+ Iterator<Line> contentLineIterator =
+ createContentIterator(firstContentLine, lineIterator, lastEmptyHeadersToSkip);
final ILineFilter filter = AlwaysAcceptLineFilter.INSTANCE;
- if (previousLineHasColumnHeaders)
+ return parser.parse(contentLineIterator, filter, headerLength);
+ }
+
+ /**
+ * @param firstContentLineOrNull if not null, it will be returned as the first iterator element,
+ * followed by all iterator elements from the second parameter
+ * @param lastEmptyTokensToSkip the number of token separators which will be removed form the
+ * end of each iterated line
+ */
+ private static Iterator<Line> createContentIterator(final Line firstContentLineOrNull,
+ final Iterator<Line> lineIterator, final int lastEmptyTokensToSkip)
+ {
+
+ final String suffixToDelete = multiply(lastEmptyTokensToSkip, TOKENS_SEPARATOR);
+ return new Iterator<Line>()
+ {
+ private Line firstLineOrNull = firstContentLineOrNull;
+
+ public boolean hasNext()
+ {
+ return firstLineOrNull != null || lineIterator.hasNext();
+ }
+
+ public Line next()
+ {
+ return trim(nextUntrimmed());
+ }
+
+ private Line trim(Line line)
+ {
+ if (lastEmptyTokensToSkip == 0)
+ {
+ return line;
+ }
+ String text = trim(line.getText(), line.getNumber());
+ return new Line(line.getNumber(), text);
+ }
+
+ private String trim(String text, int lineNumber)
+ {
+ if (text.endsWith(suffixToDelete))
+ {
+ return text.substring(0, text.length() - suffixToDelete.length());
+ } else
+ {
+ throw new ParsingException(new String[]
+ { text }, lineNumber)
+ {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public final String getMessage()
+ {
+ return super.getMessage()
+ + " The line was expected to be followed by as many separators as the header.";
+ }
+ };
+ }
+ }
+
+ private Line nextUntrimmed()
+ {
+ if (firstLineOrNull != null)
+ {
+ Line line = firstLineOrNull;
+ firstLineOrNull = null;
+ return line;
+ } else
+ {
+ return lineIterator.next();
+ }
+ }
+
+ public void remove()
+ {
+ throw new NotImplementedException();
+ }
+ };
+ }
+
+ private static String multiply(int number, String text)
+ {
+ StringBuffer sb = new StringBuffer();
+ for (int i = 0; i < number; i++)
+ {
+ sb.append(text);
+ }
+ return sb.toString();
+ }
+
+ // how many tokens at the end of the array are blank?
+ private static int countLastEmptyTokens(String text)
+ {
+ String rest = text;
+ int counter = 0;
+ while (rest.endsWith(TOKENS_SEPARATOR))
{
- result.addAll(parser.parse(Arrays.asList(line).iterator(), filter, headerLength));
+ rest = rest.substring(0, rest.length() - TOKENS_SEPARATOR.length());
+ counter++;
}
- result.addAll(parser.parse(lineIterator, filter, headerLength));
- return result;
+ return counter;
}
private Iterator<Line> createLineIterator(final Reader reader)
diff --git a/common/sourceTest/java/ch/systemsx/cisd/common/parser/TabFileLoaderTest.java b/common/sourceTest/java/ch/systemsx/cisd/common/parser/TabFileLoaderTest.java
index eb65021aa15..ac67902ff1f 100644
--- a/common/sourceTest/java/ch/systemsx/cisd/common/parser/TabFileLoaderTest.java
+++ b/common/sourceTest/java/ch/systemsx/cisd/common/parser/TabFileLoaderTest.java
@@ -127,6 +127,32 @@ public final class TabFileLoaderTest
assertEquals(list.toString(), 0, list.size());
}
+ @Test
+ public void testSkipAdditionalLineSeparators()
+ {
+ TabFileLoader<ABC> loader = new TabFileLoader<ABC>(new ABCFactoryFactory());
+ String additionalSeparators = "\t\t\t";
+ String header = "A\tB\tC" + additionalSeparators;
+ String values = header;
+ List<ABC> list = loader.load(new StringReader(header + "\n" + values));
+
+ assertEquals(list.toString(), 1, list.size());
+ ABC row = list.get(0);
+ assertEquals("A", row.getA());
+ assertEquals("B", row.getB());
+ assertEquals("C", row.getC());
+ }
+
+ @Test(expectedExceptions = ParsingException.class)
+ public void testDifferentNumberOfAdditionalLineSeparatorsFails()
+ {
+ TabFileLoader<ABC> loader = new TabFileLoader<ABC>(new ABCFactoryFactory());
+ String additionalSeparators = "\t\t\t";
+ String header = "A\tB\tC" + additionalSeparators;
+ String values = header + "\t";
+ loader.load(new StringReader(header + "\n" + values));
+ }
+
@Test
public void testFirstLineHasHeadersWithoutHashSymbol()
{
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/plugin/generic/client/web/server/parser/SampleUploadSectionsParser.java b/openbis/source/java/ch/systemsx/cisd/openbis/plugin/generic/client/web/server/parser/SampleUploadSectionsParser.java
index f10af4e64df..f0ec2d3e086 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/plugin/generic/client/web/server/parser/SampleUploadSectionsParser.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/plugin/generic/client/web/server/parser/SampleUploadSectionsParser.java
@@ -153,8 +153,6 @@ public class SampleUploadSectionsParser
private static List<FileSection> extractSections(IUncheckedMultipartFile multipartFile)
{
- final String beginSection = "[";
- final String endSection = "]";
List<FileSection> sections = new ArrayList<FileSection>();
InputStreamReader reader = new InputStreamReader(multipartFile.getInputStream());
try
@@ -165,13 +163,14 @@ public class SampleUploadSectionsParser
while (it.hasNext())
{
String line = it.nextLine();
- if (line != null && line.startsWith(beginSection) && line.endsWith(endSection))
+ String newSectionName = tryGetSectionName(line);
+ if (newSectionName != null)
{
if (sectionName != null && sb != null)
+ {
sections.add(new FileSection(sb.toString(), sectionName));
- sectionName =
- line.substring(line.indexOf(beginSection) + 1, line
- .lastIndexOf(endSection));
+ }
+ sectionName = newSectionName;
sb = new StringBuilder();
} else if (sectionName == null || sb == null)
{
@@ -196,6 +195,24 @@ public class SampleUploadSectionsParser
return sections;
}
+ private static String tryGetSectionName(String line)
+ {
+ final String beginSection = "[";
+ final String endSection = "]";
+ if (line == null)
+ {
+ return null;
+ }
+ String trimmedLine = line.trim();
+ if (trimmedLine.startsWith(beginSection) && trimmedLine.endsWith(endSection))
+ {
+ return trimmedLine.substring(1, trimmedLine.length() - 1);
+ } else
+ {
+ return null;
+ }
+ }
+
private static List<BatchRegistrationResult> loadSamplesFromFiles(
UploadedFilesBean uploadedFiles, SampleType sampleType, boolean isAutoGenerateCodes,
final List<NewSamplesWithTypes> newSamples, boolean allowExperiments)
--
GitLab