SSDM-6374 : Microservice server merged from SVN

microservice-server/build.gradle

@@ -6,6 +6,7 @@ task wrapper(type: Wrapper) {
 }
 
 repositories {
+    mavenCentral()
     ivy {
         ivyPattern "http://svnsis.ethz.ch/repos/cisd/ivy-repository/trunk/[organisation]/[module]/[revision]/ivy.xml"
         artifactPattern "http://svnsis.ethz.ch/repos/cisd/ivy-repository/trunk/[organisation]/[module]/[revision]/[artifact]-[revision](-[classifier]).[ext]"
@@ -24,7 +25,7 @@ dependencies {
 }
 
 
-mainClassName = "ch.ethz.sis.microservices.server.startup.Main"
+mainClassName = "ch.ethz.sis.microservices.download.server.startup.Main"
 
 distZip {
     into(project.name) {

microservice-server/conf/config.json

@@ -2,7 +2,7 @@
 	"port" : 8080,
 	"services" : [
 		{	
-			"className" : "ch.ethz.sis.microservices.server.services.store.FileInfoHandler", 
+			"className" : "ch.ethz.sis.microservices.download.server.services.store.FileInfoHandler", 
 			"url" : "/file-information",
 			"parameters" : {
 				"openbis-url" : "http://localhost:8888/openbis/openbis/rmi-application-server-v3",
@@ -13,7 +13,7 @@
 			}
 		}, 
 		{ 
-			"className" : "ch.ethz.sis.microservices.server.services.store.DownloadHandler", 
+			"className" : "ch.ethz.sis.microservices.download.server.services.store.DownloadHandler", 
 			"url" : "/download",
 			"parameters" : { 
 				"openbis-url" : "http://localhost:8888/openbis/openbis/rmi-application-server-v3",

microservice-server/config.json

+{
+	"port" : 8080,
+	"services" : [
+		{	
+			"className" : "ch.ethz.sis.microservices.download.server.services.store.FileInfoHandler", 
+			"url" : "/file-information",
+			"parameters" : {
+				"openbis-url" : "http://localhost:8888/openbis/openbis/rmi-application-server-v3",
+				"datastore-url" : "http://localhost:8889/datastore_server/rmi-data-store-server-v3",
+				"services-timeout" : "10000",
+				"allowedExternalDMSCode" : "ADMIN-BS-MBPR28.D.ETHZ.CH-E96954A7",
+				"allowedContentCopyPath" : "/Users/localadmin/obis_data/"
+			}
+		}, 
+		{ 
+			"className" : "ch.ethz.sis.microservices.download.server.services.store.DownloadHandler", 
+			"url" : "/download",
+			"parameters" : { 
+				"openbis-url" : "http://localhost:8888/openbis/openbis/rmi-application-server-v3",
+				"datastore-url" : "http://localhost:8889/datastore_server/rmi-data-store-server-v3",
+				"services-timeout" : "10000",
+				"allowedExternalDMSCode" : "ADMIN-BS-MBPR28.D.ETHZ.CH-E96954A7",
+				"allowedContentCopyPath" : "/Users/localadmin/obis_data/"
+			}
+		} 
+	]
+}
\ No newline at end of file

microservice-server/gradle/wrapper/gradle-wrapper.properties

@@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=http\://svnsis.ethz.ch/repos/cisd/ivy-repository/trunk/gradle/distribution/4.5/gradle-4.5-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.5-bin.zip

microservice-server/src/main/java/ch/ethz/sis/microservices/api/configuration/Config.java → microservice-server/src/main/java/ch/ethz/sis/microservices/download/api/configuration/Config.java

-package ch.ethz.sis.microservices.api.configuration;
+package ch.ethz.sis.microservices.download.api.configuration;
 
 import lombok.Data;
 import lombok.NoArgsConstructor;

microservice-server/src/main/java/ch/ethz/sis/microservices/api/configuration/ServiceConfig.java → microservice-server/src/main/java/ch/ethz/sis/microservices/download/api/configuration/ServiceConfig.java

-package ch.ethz.sis.microservices.api.configuration;
+package ch.ethz.sis.microservices.download.api.configuration;
 
 import java.util.HashMap;
 

microservice-server/src/main/java/ch/ethz/sis/microservices/server/json/JSONObjectMapper.java → microservice-server/src/main/java/ch/ethz/sis/microservices/download/server/json/JSONObjectMapper.java

-package ch.ethz.sis.microservices.server.json;
+package ch.ethz.sis.microservices.download.server.json;
 
 import java.io.FileInputStream;
 

microservice-server/src/main/java/ch/ethz/sis/microservices/server/json/jackson/JacksonObjectMapper.java → microservice-server/src/main/java/ch/ethz/sis/microservices/download/server/json/jackson/JacksonObjectMapper.java

-package ch.ethz.sis.microservices.server.json.jackson;
+package ch.ethz.sis.microservices.download.server.json.jackson;
 
 import java.io.FileInputStream;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
 
-import ch.ethz.sis.microservices.server.json.JSONObjectMapper;
+import ch.ethz.sis.microservices.download.server.json.JSONObjectMapper;
 
 public class JacksonObjectMapper implements JSONObjectMapper
 {

microservice-server/src/main/java/ch/ethz/sis/microservices/server/logging/LogFactory.java → microservice-server/src/main/java/ch/ethz/sis/microservices/download/server/logging/LogFactory.java

-package ch.ethz.sis.microservices.server.logging;
+package ch.ethz.sis.microservices.download.server.logging;
 
 public interface LogFactory {
     <T> Logger getLogger(Class<T> clazz);

microservice-server/src/main/java/ch/ethz/sis/microservices/server/logging/LogFactoryFactory.java → microservice-server/src/main/java/ch/ethz/sis/microservices/download/server/logging/LogFactoryFactory.java

-package ch.ethz.sis.microservices.server.logging;
+package ch.ethz.sis.microservices.download.server.logging;
 
 public class LogFactoryFactory {
     public LogFactory create(String logFactoryClass) throws Exception {

microservice-server/src/main/java/ch/ethz/sis/microservices/server/logging/LogManager.java → microservice-server/src/main/java/ch/ethz/sis/microservices/download/server/logging/LogManager.java

-package ch.ethz.sis.microservices.server.logging;
+package ch.ethz.sis.microservices.download.server.logging;
 
 public class LogManager {
     private static LogFactory factory;

microservice-server/src/main/java/ch/ethz/sis/microservices/server/logging/Logger.java → microservice-server/src/main/java/ch/ethz/sis/microservices/download/server/logging/Logger.java

-package ch.ethz.sis.microservices.server.logging;
+package ch.ethz.sis.microservices.download.server.logging;
 
 
 public interface Logger {

microservice-server/src/main/java/ch/ethz/sis/microservices/server/logging/log4j/Log4J2LogFactory.java → microservice-server/src/main/java/ch/ethz/sis/microservices/download/server/logging/log4j/Log4J2LogFactory.java

-package ch.ethz.sis.microservices.server.logging.log4j;
+package ch.ethz.sis.microservices.download.server.logging.log4j;
 
 import org.apache.logging.log4j.LogManager;
 
-import ch.ethz.sis.microservices.server.logging.LogFactory;
-import ch.ethz.sis.microservices.server.logging.Logger;
+import ch.ethz.sis.microservices.download.server.logging.LogFactory;
+import ch.ethz.sis.microservices.download.server.logging.Logger;
 
 public class Log4J2LogFactory implements LogFactory
 {

microservice-server/src/main/java/ch/ethz/sis/microservices/server/logging/log4j/Log4JLogger.java → microservice-server/src/main/java/ch/ethz/sis/microservices/download/server/logging/log4j/Log4JLogger.java

-package ch.ethz.sis.microservices.server.logging.log4j;
+package ch.ethz.sis.microservices.download.server.logging.log4j;
 
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.spi.AbstractLogger;
 import org.apache.logging.log4j.spi.ExtendedLoggerWrapper;
 
-class Log4JLogger extends ExtendedLoggerWrapper implements ch.ethz.sis.microservices.server.logging.Logger
+class Log4JLogger extends ExtendedLoggerWrapper implements ch.ethz.sis.microservices.download.server.logging.Logger
 {
 
     private final String FQCN;

microservice-server/src/main/java/ch/ethz/sis/microservices/server/services/Service.java → microservice-server/src/main/java/ch/ethz/sis/microservices/download/server/services/Service.java

-package ch.ethz.sis.microservices.server.services;
+package ch.ethz.sis.microservices.download.server.services;
 
 import javax.servlet.http.HttpServlet;
 
-import ch.ethz.sis.microservices.api.configuration.ServiceConfig;
+import ch.ethz.sis.microservices.download.api.configuration.ServiceConfig;
 
 public abstract class Service extends HttpServlet
 {

microservice-server/src/main/java/ch/ethz/sis/microservices/server/services/store/AbstractFileServiceHandler.java → microservice-server/src/main/java/ch/ethz/sis/microservices/download/server/services/store/AbstractFileServiceHandler.java

-package ch.ethz.sis.microservices.server.services.store;
+package ch.ethz.sis.microservices.download.server.services.store;
 
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Arrays;
-import java.util.Deque;
-import java.util.List;
 import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedDeque;
 
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
-import ch.ethz.sis.microservices.server.logging.LogManager;
-import ch.ethz.sis.microservices.server.logging.Logger;
-import ch.ethz.sis.microservices.server.services.Service;
+import ch.ethz.sis.microservices.download.server.logging.LogManager;
+import ch.ethz.sis.microservices.download.server.logging.Logger;
+import ch.ethz.sis.microservices.download.server.services.Service;
 import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.search.SearchResult;
 import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.ContentCopy;
 import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet;
 import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.fetchoptions.DataSetFetchOptions;
@@ -29,7 +26,6 @@ import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile;
 import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.fetchoptions.DataSetFileFetchOptions;
 import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.search.DataSetFileSearchCriteria;
 import ch.systemsx.cisd.common.spring.HttpInvokerUtils;
-import lombok.Value;
 
 public abstract class AbstractFileServiceHandler extends Service
 {
@@ -70,10 +66,10 @@ public abstract class AbstractFileServiceHandler extends Service
         Path pathToFile = getPathToFile(contentCopyPath, datasetPathToFile);
         if (isFileAccessible)
         {
-            success(pathToFile, response);
+            success(pathToFile, request, response);
         } else
         {
-            failure(pathToFile, response);
+            failure(pathToFile, request, response);
         }
     }
 
@@ -85,9 +81,9 @@ public abstract class AbstractFileServiceHandler extends Service
         doAction(request, response);
     }
 
-    protected abstract void success(Path pathToFile, HttpServletResponse response) throws ServletException, IOException;
+    protected abstract void success(Path pathToFile, HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException;
 
-    protected abstract void failure(Path pathToFile, HttpServletResponse response) throws ServletException, IOException;
+    protected abstract void failure(Path pathToFile, HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException;
 
     private DataSet getDataset(String openbisURL, int openbisTimeout, String sessionToken, String datasetPermId, String externalDMSCode,
             String contentCopyPath)
@@ -110,68 +106,17 @@ public abstract class AbstractFileServiceHandler extends Service
         }
         return null;
     }
-    
-    /*
-     * Less recently used cache
-     * Keeps maxSize items
-     */
-    @Value
-    private static class LRUCache<KEY, VALUE> {
-
-        int maxSize;
-        Map<KEY, VALUE> cacheByKey = new ConcurrentHashMap<>();
-        Deque<KEY> cachePriority = new ConcurrentLinkedDeque<>();
-
-        public LRUCache(int maxSize) {
-            this.maxSize = maxSize;
-        }
-
-        public void add(KEY key, VALUE value) {
-            // Remove if existing
-            if(cacheByKey.containsKey(key)) {
-                cacheByKey.remove(key);
-                cachePriority.remove(key);
-            }
-
-            // Add again
-            cacheByKey.put(key, value);
-            cachePriority.addFirst(key);
-
-            // Remove oldest if the list grow too much
-            if(cachePriority.size() > maxSize) {
-                cacheByKey.remove(cachePriority.removeLast());
-            }
-        }
-
-        public VALUE get(KEY key) {
-            // Refresh Prio
-            if(cacheByKey.containsKey(key)) {
-                cachePriority.remove(key);
-                cachePriority.addFirst(key);
-            }
-
-            return cacheByKey.get(key);
-        }
-
-    }
-
-    private static LRUCache<String, List<DataSetFile>> cache = new LRUCache<>(1000);
 
     private DataSetFile getDatasetFile(String datastoreURL, int datastoreTimeout, String sessionToken, String datasetPermId,
             String datasetPathToFile)
     {
-        List<DataSetFile> files = cache.get(datasetPermId);
-        
-        if(files == null) {
-            IDataStoreServerApi v3Dss = HttpInvokerUtils.createServiceStub(IDataStoreServerApi.class, datastoreURL, datastoreTimeout);
-            DataSetFileSearchCriteria searchCriteria = new DataSetFileSearchCriteria();
-            searchCriteria.withDataSet().withCode().thatEquals(datasetPermId);
-            DataSetFileFetchOptions fetchOptions = new DataSetFileFetchOptions();
-            files = v3Dss.searchFiles(sessionToken, searchCriteria, fetchOptions).getObjects();
-            cache.add(datasetPermId, files);
-        }
-        
-        for (DataSetFile file : files)
+        IDataStoreServerApi v3Dss = HttpInvokerUtils.createServiceStub(IDataStoreServerApi.class, datastoreURL, datastoreTimeout);
+
+        DataSetFileSearchCriteria searchCriteria = new DataSetFileSearchCriteria();
+        searchCriteria.withDataSet().withCode().thatEquals(datasetPermId);
+        DataSetFileFetchOptions fetchOptions = new DataSetFileFetchOptions();
+        SearchResult<DataSetFile> files = v3Dss.searchFiles(sessionToken, searchCriteria, fetchOptions);
+        for (DataSetFile file : files.getObjects())
         {
             if (file.getPath().equals(datasetPathToFile))
             {

microservice-server/src/test/java/ch/ethz/sis/microservices/server/services/store/AbstractFileServiceTest.java → microservice-server/src/main/java/ch/ethz/sis/microservices/download/server/services/store/AbstractFileServiceTest.java

-package ch.ethz.sis.microservices.server.services.store;
+package ch.ethz.sis.microservices.download.server.services.store;
 
 import java.util.HashMap;
 import java.util.Map;
 
-import ch.ethz.sis.microservices.server.logging.LogManager;
-import ch.ethz.sis.microservices.server.logging.log4j.Log4J2LogFactory;
-import ch.ethz.sis.microservices.util.HttpClient;
+import ch.ethz.sis.microservices.download.server.logging.LogManager;
+import ch.ethz.sis.microservices.download.server.logging.log4j.Log4J2LogFactory;
+import ch.ethz.sis.microservices.download.server.startup.HttpClient;
 import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi;
 import ch.systemsx.cisd.common.spring.HttpInvokerUtils;
 
@@ -17,13 +17,13 @@ public class AbstractFileServiceTest
         LogManager.setLogFactory(new Log4J2LogFactory());
     }
 
-    public static void test(String openbisURL, String serviceURL, String user, String pass) throws Exception
+    public static void test(String openbisURL, String serviceURL, String user, String pass, Long offset) throws Exception
     {
         // Service
         String externalDMSCode = "ADMIN-BS-MBPR28.D.ETHZ.CH-E96954A7";
-        String contentCopyPath = "/Users/localadmin/obis_data/data";
-        String datasetPermId = "20180131090547942-3";
-        String datasetPathToFile = "file";
+        String contentCopyPath = "/Users/localadmin/obis_data/data1";
+        String datasetPermId = "20180523115921026-50";
+        String datasetPathToFile = "openBIS-installation-standard-technologies-SNAPSHOT-r1526484921.tar.gz";
 
         // Obtain session token from openBIS
         int timeout = 10000;
@@ -36,11 +36,15 @@ public class AbstractFileServiceTest
         parameters.put("externalDMSCode", externalDMSCode);
         parameters.put("contentCopyPath", contentCopyPath);
         parameters.put("datasetPathToFile", datasetPathToFile);
+        if(offset != null) {
+        		parameters.put("offset", offset.toString());
+        }
+        
 
         long start = System.currentTimeMillis();
         byte[] response = HttpClient.doGet(serviceURL, parameters);
         long end = System.currentTimeMillis();
-        System.out.println(new String(response));
+        System.out.println("Response Size: " + response.length);
         System.out.println("Time: " + (end - start));
     }
 }

microservice-server/src/main/java/ch/ethz/sis/microservices/server/services/store/DownloadHandler.java → microservice-server/src/main/java/ch/ethz/sis/microservices/download/server/services/store/DownloadHandler.java

-package ch.ethz.sis.microservices.server.services.store;
+package ch.ethz.sis.microservices.download.server.services.store;
 
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 
 import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
-import ch.ethz.sis.microservices.server.logging.LogManager;
-import ch.ethz.sis.microservices.server.logging.Logger;
+import ch.ethz.sis.microservices.download.server.logging.LogManager;
+import ch.ethz.sis.microservices.download.server.logging.Logger;
 
 public class DownloadHandler extends AbstractFileServiceHandler
 {
     private static Logger logger = LogManager.getLogger(DownloadHandler.class);
 
     @Override
-    protected void success(Path pathToFile, HttpServletResponse response) throws ServletException, IOException
+    protected void success(Path pathToFile, HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
     {
+    		// Request parameters
+    		String offsetP = request.getParameter("offset");
+    		long offset = 0;
+    		if(offsetP != null) {
+    			offset = Long.parseLong(offsetP);
+    		}
+        
+    		long size = Files.size(pathToFile) - offset;
+    		if(size < 0) {
+    			throw new IllegalArgumentException("offset to read starts beyond end of file");
+    		}
+    		
+    		// Response
         response.setContentType("application/octet-stream");
         response.setHeader("Content-Disposition", "attachment; filename=" + pathToFile.getFileName().toString());
         response.setHeader("Content-Length", Long.toString(Files.size(pathToFile)));
-        Files.copy(pathToFile, response.getOutputStream());
+        
+		InputStream is = Files.newInputStream(pathToFile);
+		is.skip(offset);
+		copy(is, response.getOutputStream(), size);
+        
         response.setStatus(HttpServletResponse.SC_OK);
     }
+    
+    private static final int BUFFER_SIZE = 1024;
+    private static void copy(InputStream source, OutputStream destination, long sourceSize) throws IOException {
+    		byte[] buf = new byte[BUFFER_SIZE];
+    		int readed;
+    		while ((readed = source.read(buf)) > 0) {
+    			destination.write(buf, 0, readed);
+    		}
+    }
 
     @Override
-    protected void failure(Path pathToFile, HttpServletResponse response) throws ServletException, IOException
+    protected void failure(Path pathToFile, HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
     {
         response.setStatus(HttpServletResponse.SC_NOT_FOUND);
     }

microservice-server/src/test/java/ch/ethz/sis/microservices/server/services/store/DownloadHandlerTest.java → microservice-server/src/main/java/ch/ethz/sis/microservices/download/server/services/store/DownloadHandlerTest.java

-package ch.ethz.sis.microservices.server.services.store;
+package ch.ethz.sis.microservices.download.server.services.store;
 
 public class DownloadHandlerTest
 {
@@ -9,6 +9,7 @@ public class DownloadHandlerTest
                 "http://localhost:8888/openbis/openbis/rmi-application-server-v3",
                 "http://localhost:8080/download",
                 "admin",
-                "admin");
+                "admin",
+                null);
     }
 }