From 1c48060bbcd1a9cb406d16b216d0c2a18c380b79 Mon Sep 17 00:00:00 2001
From: pkupczyk <pkupczyk>
Date: Mon, 4 Feb 2013 13:41:35 +0000
Subject: [PATCH] SP-484 / BIS-255 : SFTP problems: "Downloading is very slow
and slows down until OutOfMemoryError is thrown. There is also a lot of log
activity on the remote DSS" fixed
SVN: 28276
---
.../common/io/FullLengthReadingStream.java | 59 ++++++++++
.../io/FullLengthReadingStreamTest.java | 111 ++++++++++++++++++
.../generic/shared/content/ContentCache.java | 3 +-
3 files changed, 172 insertions(+), 1 deletion(-)
create mode 100644 common/source/java/ch/systemsx/cisd/common/io/FullLengthReadingStream.java
create mode 100644 common/sourceTest/java/ch/systemsx/cisd/common/io/FullLengthReadingStreamTest.java
diff --git a/common/source/java/ch/systemsx/cisd/common/io/FullLengthReadingStream.java b/common/source/java/ch/systemsx/cisd/common/io/FullLengthReadingStream.java
new file mode 100644
index 00000000000..84c96d44dcd
--- /dev/null
+++ b/common/source/java/ch/systemsx/cisd/common/io/FullLengthReadingStream.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2013 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.common.io;
+
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * A stream decorator that always returns a requested number of bytes even though the decorated
+ * stream may return this information in smaller chunks (i.e. read method always reads as many bytes
+ * as were specified by the length parameter unless the end of the stream is reached).
+ *
+ * @author pkupczyk
+ */
+public class FullLengthReadingStream extends FilterInputStream
+{
+
+ public FullLengthReadingStream(InputStream in)
+ {
+ super(in);
+ }
+
+ @Override
+ public int read(byte[] b, int offset, int length) throws IOException
+ {
+ int read = 0;
+
+ while (read < length)
+ {
+ int count = in.read(b, offset + read, length - read);
+
+ if (count < 0)
+ {
+ return read > 0 ? read : -1;
+ } else
+ {
+ read += count;
+ }
+ }
+
+ return read;
+ }
+
+}
diff --git a/common/sourceTest/java/ch/systemsx/cisd/common/io/FullLengthReadingStreamTest.java b/common/sourceTest/java/ch/systemsx/cisd/common/io/FullLengthReadingStreamTest.java
new file mode 100644
index 00000000000..9cdad79b0cc
--- /dev/null
+++ b/common/sourceTest/java/ch/systemsx/cisd/common/io/FullLengthReadingStreamTest.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2013 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.common.io;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+
+import org.testng.AssertJUnit;
+import org.testng.annotations.Test;
+
+/**
+ * @author pkupczyk
+ */
+public class FullLengthReadingStreamTest extends AssertJUnit
+{
+
+ @Test
+ public void testHalfReadingStream() throws IOException
+ {
+ HalfReadingStream stream = new HalfReadingStream(new byte[]
+ { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 });
+
+ byte[] bytes = new byte[10];
+ int count = stream.read(bytes, 3, 11);
+
+ assertEquals(5, count);
+ assertEquals(new byte[]
+ { 0, 0, 0, 0, 1, 2, 3, 4, 0, 0 }, bytes);
+
+ bytes = new byte[10];
+ count = stream.read(bytes, 8, 4);
+
+ assertEquals(2, count);
+ assertEquals(new byte[]
+ { 0, 0, 0, 0, 0, 0, 0, 0, 5, 6 }, bytes);
+
+ bytes = new byte[10];
+ count = stream.read(bytes, 0, 10);
+
+ assertEquals(3, count);
+ assertEquals(new byte[]
+ { 7, 8, 9, 0, 0, 0, 0, 0, 0, 0 }, bytes);
+
+ bytes = new byte[10];
+ count = stream.read(bytes, 0, 10);
+
+ assertEquals(-1, count);
+ assertEquals(new byte[]
+ { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, bytes);
+ }
+
+ @Test
+ public void testFullReadingStream() throws IOException
+ {
+ HalfReadingStream stream = new HalfReadingStream(new byte[]
+ { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 });
+ FullLengthReadingStream fullStream = new FullLengthReadingStream(stream);
+
+ byte[] bytes = new byte[6];
+ int count = fullStream.read(bytes);
+
+ assertEquals(6, count);
+ assertEquals(new byte[]
+ { 0, 1, 2, 3, 4, 5 }, bytes);
+
+ bytes = new byte[6];
+ count = fullStream.read(bytes);
+
+ assertEquals(4, count);
+ assertEquals(new byte[]
+ { 6, 7, 8, 9, 0, 0 }, bytes);
+
+ bytes = new byte[6];
+ count = fullStream.read(bytes);
+
+ assertEquals(-1, count);
+ assertEquals(new byte[]
+ { 0, 0, 0, 0, 0, 0 }, bytes);
+ }
+
+ private class HalfReadingStream extends ByteArrayInputStream
+ {
+
+ public HalfReadingStream(byte[] bytes)
+ {
+ super(bytes);
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len)
+ {
+ return super.read(b, off, Math.max(1, len / 2));
+ }
+
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/content/ContentCache.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/content/ContentCache.java
index 4114cd2e089..643aa182d41 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/content/ContentCache.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/content/ContentCache.java
@@ -54,6 +54,7 @@ import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
import ch.systemsx.cisd.common.filesystem.FileOperations;
import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.common.filesystem.IFileOperations;
+import ch.systemsx.cisd.common.io.FullLengthReadingStream;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.properties.PropertyUtils;
@@ -461,7 +462,7 @@ public class ContentCache implements IContentCache, InitializingBean
try
{
openStream = url.openStream();
- return openStream;
+ return new FullLengthReadingStream(openStream);
} catch (IOException ex)
{
IOUtils.closeQuietly(openStream);
--
GitLab