SSDM-13578 : 2PT : Database and V3 Implementation - integration tests

server-application-server/source/java/ch/ethz/sis/openbis/generic/server/asapi/v3/TransactionConfiguration.java

@@ -16,42 +16,42 @@ import ch.systemsx.cisd.common.spring.ExposablePropertyPlaceholderConfigurer;
 public class TransactionConfiguration
 {
 
-    private static final String TRANSACTION_ENABLED_PROPERTY_NAME = "api.v3.transaction.enabled";
+    public static final String TRANSACTION_ENABLED_PROPERTY_NAME = "api.v3.transaction.enabled";
 
-    private static final boolean TRANSACTION_ENABLED_DEFAULT = false;
+    public static final boolean TRANSACTION_ENABLED_DEFAULT = false;
 
-    private static final String TRANSACTION_TIMEOUT_PROPERTY_NAME = "api.v3.transaction.transaction-timeout";
+    public static final String TRANSACTION_TIMEOUT_PROPERTY_NAME = "api.v3.transaction.transaction-timeout";
 
-    private static final int TRANSACTION_TIMEOUT_DEFAULT = 3600;
+    public static final int TRANSACTION_TIMEOUT_DEFAULT = 3600;
 
-    private static final String FINISH_TRANSACTIONS_INTERVAL_PROPERTY_NAME = "api.v3.transaction.finish-transactions-interval";
+    public static final String FINISH_TRANSACTIONS_INTERVAL_PROPERTY_NAME = "api.v3.transaction.finish-transactions-interval";
 
-    private static final int FINISH_TRANSACTIONS_INTERVAL_DEFAULT = 600;
+    public static final int FINISH_TRANSACTIONS_INTERVAL_DEFAULT = 600;
 
-    private static final String INTERACTIVE_SESSION_KEY_PROPERTY_NAME = "api.v3.transaction.interactive-session-key";
+    public static final String INTERACTIVE_SESSION_KEY_PROPERTY_NAME = "api.v3.transaction.interactive-session-key";
 
-    private static final String COORDINATOR_KEY_PROPERTY_NAME = "api.v3.transaction.coordinator-key";
+    public static final String COORDINATOR_KEY_PROPERTY_NAME = "api.v3.transaction.coordinator-key";
 
-    private static final String TRANSACTION_LOG_FOLDER_PATH_PROPERTY_NAME = "api.v3.transaction.transaction-log-folder-path";
+    public static final String TRANSACTION_LOG_FOLDER_PATH_PROPERTY_NAME = "api.v3.transaction.transaction-log-folder-path";
 
-    private static final String TRANSACTION_LOG_FOLDER_PATH_DEFAULT = "transaction-logs";
+    public static final String TRANSACTION_LOG_FOLDER_PATH_DEFAULT = "transaction-logs";
 
-    private static final String TRANSACTION_COUNT_LIMIT_PROPERTY_NAME = "api.v3.transaction.transaction-count-limit";
+    public static final String TRANSACTION_COUNT_LIMIT_PROPERTY_NAME = "api.v3.transaction.transaction-count-limit";
 
-    private static final int TRANSACTION_COUNT_LIMIT_DEFAULT = 10;
+    public static final int TRANSACTION_COUNT_LIMIT_DEFAULT = 10;
 
-    private static final String APPLICATION_SERVER_URL_PROPERTY_NAME = "api.v3.transaction.participant.application-server.url";
+    public static final String APPLICATION_SERVER_URL_PROPERTY_NAME = "api.v3.transaction.participant.application-server.url";
 
-    private static final String APPLICATION_SERVER_TIMEOUT_PROPERTY_NAME =
+    public static final String APPLICATION_SERVER_TIMEOUT_PROPERTY_NAME =
             "api.v3.transaction.participant.application-server.timeout";
 
-    private static final int APPLICATION_SERVER_TIMEOUT_DEFAULT = 3600;
+    public static final int APPLICATION_SERVER_TIMEOUT_DEFAULT = 3600;
 
-    private static final String AFS_SERVER_URL_PROPERTY_NAME = "api.v3.transaction.participant.afs-server.url";
+    public static final String AFS_SERVER_URL_PROPERTY_NAME = "api.v3.transaction.participant.afs-server.url";
 
-    private static final String AFS_SERVER_TIMEOUT_PROPERTY_NAME = "api.v3.transaction.participant.afs-server.timeout";
+    public static final String AFS_SERVER_TIMEOUT_PROPERTY_NAME = "api.v3.transaction.participant.afs-server.timeout";
 
-    private static final int AFS_SERVER_TIMEOUT_DEFAULT = 3600;
+    public static final int AFS_SERVER_TIMEOUT_DEFAULT = 3600;
 
     @Resource(name = ExposablePropertyPlaceholderConfigurer.PROPERTY_CONFIGURER_BEAN_NAME)
     private ExposablePropertyPlaceholderConfigurer configurer;

server-application-server/sourceTest/java/ch/systemsx/cisd/openbis/generic/shared/util/TestInstanceHostUtils.java

@@ -32,6 +32,11 @@ public class TestInstanceHostUtils
         return OPENBIS_URL + ":" + getOpenBISPort();
     }
 
+    public static String getOpenBISPath()
+    {
+        return "/openbis/openbis";
+    }
+
     public static int getDSSPort()
     {
         return 8800 + getProjectNumber() + 9;
@@ -42,6 +47,26 @@ public class TestInstanceHostUtils
         return OPENBIS_URL + ":" + getDSSPort();
     }
 
+    public static String getDSSPath()
+    {
+        return "/datastore_server";
+    }
+
+    public static int getAFSPort()
+    {
+        return 8800 + getProjectNumber() + 7;
+    }
+
+    public static String getAFSUrl()
+    {
+        return OPENBIS_URL + ":" + getAFSPort();
+    }
+
+    public static String getAFSPath()
+    {
+        return "/data-store-server";
+    }
+
     private static int getProjectNumber()
     {
         String projectName = System.getProperty("ant.project.name", "");

test-integration/build.gradle

 apply from: '../build/javaproject.gradle'
 
 dependencies {
-    implementation project(':lib-transactional-file-system'), project(':server-application-server'), project(path: ':server-application-server', configuration: 'tests'), project(':server-data-store')
+    implementation project(path: ':lib-commonbase', configuration: 'tests'),
+            project(':lib-transactional-file-system'),
+            project(':server-application-server'),
+            project(path: ':server-application-server', configuration: 'tests'),
+            project(':server-data-store')
 }
 
 test {

test-integration/sourceTest/java/ch/ethz/sis/openbis/systemtests/Integration2PCTest.java

+package ch.ethz.sis.openbis.systemtests;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.fail;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.util.List;
+
+import org.testng.annotations.Test;
+
+import ch.ethz.sis.openbis.generic.OpenBIS;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.entitytype.id.EntityTypePermId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.create.ExperimentCreation;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.project.create.ProjectCreation;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.project.id.ProjectPermId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.create.SpaceCreation;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.id.SpacePermId;
+import ch.ethz.sis.openbis.systemtests.common.AbstractIntegrationTest;
+import ch.ethz.sis.openbis.systemtests.common.IntegrationTestOpenBIS;
+import ch.systemsx.cisd.common.test.AssertionUtil;
+
+public class Integration2PCTest extends AbstractIntegrationTest
+{
+
+    @Test
+    public void test()
+    {
+        OpenBIS openBIS = new IntegrationTestOpenBIS();
+        openBIS.setInteractiveSessionKey(TEST_INTERACTIVE_SESSION_KEY);
+
+        openBIS.login("test", "admin");
+
+        openBIS.beginTransaction();
+
+        byte[] bytes = "Hello World!".getBytes(StandardCharsets.UTF_8);
+        openBIS.getAfsServerFacade().write("another-sample", "anotherdir/anotherfile", 0L, bytes, calculateMD5(bytes));
+
+        SpaceCreation spaceCreation = new SpaceCreation();
+        spaceCreation.setCode("2PT_TEST");
+
+        SpacePermId spaceId = openBIS.createSpaces(List.of(spaceCreation)).get(0);
+
+        ProjectCreation projectCreation = new ProjectCreation();
+        projectCreation.setSpaceId(spaceId);
+        projectCreation.setCode("2PT_TEST");
+
+        ProjectPermId projectId = openBIS.createProjects(List.of(projectCreation)).get(0);
+
+        ExperimentCreation experimentCreation = new ExperimentCreation();
+        experimentCreation.setProjectId(projectId);
+        experimentCreation.setCode("2PT_TEST");
+
+        try
+        {
+            openBIS.createExperiments(List.of(experimentCreation));
+            fail();
+        }catch(Exception e){
+            assertEquals(e.getMessage(), "Operation 'createExperiments' failed.");
+            AssertionUtil.assertContains("Type id cannot be null", e.getCause().getMessage());
+        }
+
+        ExperimentCreation experimentCreation2 = new ExperimentCreation();
+        experimentCreation2.setTypeId(new EntityTypePermId("UNKNOWN"));
+        experimentCreation2.setProjectId(projectId);
+        experimentCreation2.setCode("2PT_TEST");
+
+        openBIS.createExperiments(List.of(experimentCreation2));
+
+        openBIS.commitTransaction();
+    }
+
+    public static byte[] calculateMD5(byte[] data)
+    {
+        try
+        {
+            MessageDigest md = MessageDigest.getInstance("MD5");
+            md.update(data);
+            return md.digest();
+        } catch (Exception e)
+        {
+            throw new RuntimeException("Checksum calculation failed", e);
+        }
+    }
+
+}

test-integration/sourceTest/java/ch/ethz/sis/openbis/systemtests/IntegrationTest.java

-package ch.ethz.sis.openbis.systemtests;
-
-import java.nio.charset.StandardCharsets;
-import java.security.MessageDigest;
-
-import org.testng.annotations.Test;
-
-import ch.ethz.sis.openbis.generic.OpenBIS;
-import ch.systemsx.cisd.openbis.generic.shared.util.TestInstanceHostUtils;
-
-public class IntegrationTest extends AbstractIntegrationTest
-{
-
-    @Test
-    public void helloWorld()
-    {
-        OpenBIS openBIS = new OpenBIS("http://localhost:" + TestInstanceHostUtils.getOpenBISPort() + "/openbis/openbis", "OLD_DSS_IS_NOT_USED", "http://localhost:8085/data-store-server");
-
-        String sessionToken = openBIS.login("test","password");
-
-        openBIS.getAfsServerFacade().write("another-sample", "anotherdir/anotherfile", 0L, "Hello World!".getBytes(StandardCharsets.UTF_8),
-                calculateMD5("Hello World!".getBytes(StandardCharsets.UTF_8)));
-
-        System.out.println("Session token: " + sessionToken);
-    }
-
-    public static byte[] calculateMD5(byte[] data)
-    {
-        try
-        {
-            MessageDigest md = MessageDigest.getInstance("MD5");
-            md.update(data);
-            return md.digest();
-        } catch (Exception e)
-        {
-            throw new RuntimeException("Checksum calculation failed", e);
-        }
-    }
-
-}

test-integration/sourceTest/java/ch/ethz/sis/openbis/systemtests/AbstractIntegrationTest.java → test-integration/sourceTest/java/ch/ethz/sis/openbis/systemtests/common/AbstractIntegrationTest.java

@@ -13,10 +13,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package ch.ethz.sis.openbis.systemtests;
+package ch.ethz.sis.openbis.systemtests.common;
 
+import java.io.File;
+import java.io.FileInputStream;
 import java.lang.reflect.Method;
+import java.nio.file.Files;
 import java.util.List;
+import java.util.Properties;
 
 import org.eclipse.jetty.server.HttpConfiguration;
 import org.eclipse.jetty.server.HttpConnectionFactory;
@@ -31,13 +35,17 @@ import org.springframework.web.context.WebApplicationContext;
 import org.springframework.web.context.support.GenericWebApplicationContext;
 import org.springframework.web.servlet.DispatcherServlet;
 import org.testng.annotations.AfterMethod;
+import org.testng.annotations.AfterSuite;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.BeforeSuite;
 
+import ch.ethz.sis.afs.manager.TransactionConnection;
 import ch.ethz.sis.afsserver.server.observer.impl.DummyServerObserver;
 import ch.ethz.sis.afsserver.startup.AtomicFileSystemServerParameter;
+import ch.ethz.sis.openbis.generic.server.asapi.v3.TransactionConfiguration;
 import ch.ethz.sis.shared.startup.Configuration;
-import ch.systemsx.cisd.openbis.generic.server.util.TestInitializer;
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
+import ch.systemsx.cisd.common.logging.LogInitializer;
 import ch.systemsx.cisd.openbis.generic.shared.util.TestInstanceHostUtils;
 
 /**
@@ -45,18 +53,49 @@ import ch.systemsx.cisd.openbis.generic.shared.util.TestInstanceHostUtils;
  */
 public abstract class AbstractIntegrationTest
 {
+    public static final String TEST_TRANSACTION_COORDINATOR_KEY = "integration-test-transaction-coordinator-key";
+
+    public static final String TEST_INTERACTIVE_SESSION_KEY = "integration-test-interactive-session-key";
+
+    private static Server applicationServer;
+
+    private static ch.ethz.sis.afsserver.server.Server<TransactionConnection, Object> afsServer;
+
     protected static GenericWebApplicationContext applicationServerSpringContext;
 
-    protected static TestLogger logger = new TestLogger();
+    protected static IntegrationTestLogger logger = new IntegrationTestLogger();
 
     @BeforeSuite
     public void beforeSuite() throws Exception
     {
-        TestInitializer.initWithIndex();
+        LogInitializer.init();
+
+        cleanupApplicationServerFolders();
+        cleanupAfsServerFolders();
+
         startApplicationServer();
         startAfsServer();
     }
 
+    @AfterSuite
+    public void afterSuite() throws Exception
+    {
+        shutdownApplicationServer();
+        shutdownAfsServer();
+    }
+
+    private void shutdownApplicationServer()
+    {
+        applicationServer.setStopAtShutdown(true);
+        logger.log("Shut down application server.");
+    }
+
+    private void shutdownAfsServer() throws Exception
+    {
+        afsServer.shutdown(false);
+        logger.log("Shut down afs server.");
+    }
+
     @BeforeMethod
     public void beforeTest(Method method)
     {
@@ -69,8 +108,62 @@ public abstract class AbstractIntegrationTest
         logger.log("AFTER  " + method.getDeclaringClass().getName() + "." + method.getName());
     }
 
+    private void cleanupApplicationServerFolders() throws Exception
+    {
+        Properties configuration = getApplicationServerConfiguration();
+
+        String transactionLogFolder = configuration.getProperty(TransactionConfiguration.TRANSACTION_LOG_FOLDER_PATH_PROPERTY_NAME);
+        cleanupFolderSafely(transactionLogFolder);
+    }
+
+    private void cleanupAfsServerFolders() throws Exception
+    {
+        Configuration configuration = getAfsServerConfiguration();
+
+        String writeAheadLogFolder = configuration.getStringProperty(AtomicFileSystemServerParameter.writeAheadLogRoot);
+        cleanupFolderSafely(writeAheadLogFolder);
+
+        String storageRoot = configuration.getStringProperty(AtomicFileSystemServerParameter.storageRoot);
+        cleanupFolderSafely(storageRoot);
+    }
+
+    private void cleanupFolderSafely(String folderPath) throws Exception
+    {
+        if (!new File(folderPath).exists())
+        {
+            return;
+        }
+
+        File safetyRoot = new File("../").getCanonicalFile();
+        File folderParent = new File(folderPath).getCanonicalFile();
+
+        while (folderParent != null && !Files.isSameFile(safetyRoot.toPath(), folderParent.toPath()))
+        {
+            folderParent = folderParent.getParentFile();
+        }
+
+        if (folderParent == null)
+        {
+            throw new RuntimeException(
+                    "Folder " + new File(folderPath).getAbsolutePath() + " is outside of " + safetyRoot.getAbsolutePath()
+                            + " therefore cannot be safely deleted.");
+        } else
+        {
+            FileUtilities.deleteRecursively(new File(folderPath));
+            logger.log("Deleted folder: " + new File(folderPath).getAbsolutePath());
+        }
+    }
+
     private void startApplicationServer() throws Exception
     {
+        Properties configuration = getApplicationServerConfiguration();
+
+        for (Object key : configuration.keySet())
+        {
+            Object value = configuration.get(key);
+            System.setProperty(String.valueOf(key), String.valueOf(value));
+        }
+
         Server server = new Server();
         HttpConfiguration httpConfig = new HttpConfiguration();
         ServerConnector connector =
@@ -96,15 +189,47 @@ public abstract class AbstractIntegrationTest
                 new ServletContextHandler(server, "/", ServletContextHandler.SESSIONS);
         servletContext.addServlet(new ServletHolder(dispatcherServlet), "/*");
         server.start();
+
+        AbstractIntegrationTest.applicationServer = server;
     }
 
     private void startAfsServer() throws Exception
+    {
+        Configuration configuration = getAfsServerConfiguration();
+        DummyServerObserver dummyServerObserver = new DummyServerObserver();
+
+        AbstractIntegrationTest.afsServer = new ch.ethz.sis.afsserver.server.Server<>(configuration, dummyServerObserver, dummyServerObserver);
+    }
+
+    private Properties getApplicationServerConfiguration() throws Exception
+    {
+        Properties configuration = new Properties();
+        configuration.load(new FileInputStream("../server-application-server/source/java/service.properties"));
+        configuration.setProperty("database.create-from-scratch", "true");
+        configuration.setProperty("database.kind", "integration");
+        configuration.setProperty("script-folder", "../server-application-server/source");
+        configuration.setProperty(TransactionConfiguration.COORDINATOR_KEY_PROPERTY_NAME, TEST_TRANSACTION_COORDINATOR_KEY);
+        configuration.setProperty(TransactionConfiguration.INTERACTIVE_SESSION_KEY_PROPERTY_NAME, TEST_INTERACTIVE_SESSION_KEY);
+        configuration.setProperty(TransactionConfiguration.TRANSACTION_LOG_FOLDER_PATH_PROPERTY_NAME, "./targets/transaction-logs");
+        configuration.setProperty(TransactionConfiguration.TRANSACTION_TIMEOUT_PROPERTY_NAME, "15");
+        configuration.setProperty(TransactionConfiguration.APPLICATION_SERVER_URL_PROPERTY_NAME, TestInstanceHostUtils.getOpenBISUrl());
+        configuration.setProperty(TransactionConfiguration.AFS_SERVER_URL_PROPERTY_NAME,
+                TestInstanceHostUtils.getAFSUrl() + TestInstanceHostUtils.getAFSPath());
+        return configuration;
+    }
+
+    private Configuration getAfsServerConfiguration()
     {
         Configuration configuration = new Configuration(List.of(AtomicFileSystemServerParameter.class),
                 "../server-data-store/src/main/resources/server-data-store-config.properties");
         configuration.setProperty(AtomicFileSystemServerParameter.logConfigFile, "etc/log4j2.xml");
-        DummyServerObserver dummyServerObserver = new DummyServerObserver();
-        new ch.ethz.sis.afsserver.server.Server<>(configuration, dummyServerObserver, dummyServerObserver);
+        configuration.setProperty(AtomicFileSystemServerParameter.writeAheadLogRoot, "./targets/afs/transaction-logs");
+        configuration.setProperty(AtomicFileSystemServerParameter.storageRoot, "./targets/afs/storage");
+        configuration.setProperty(AtomicFileSystemServerParameter.apiServerTransactionManagerKey, TEST_TRANSACTION_COORDINATOR_KEY);
+        configuration.setProperty(AtomicFileSystemServerParameter.apiServerInteractiveSessionKey, TEST_INTERACTIVE_SESSION_KEY);
+        configuration.setProperty(AtomicFileSystemServerParameter.httpServerPort, String.valueOf(TestInstanceHostUtils.getAFSPort()));
+        configuration.setProperty(AtomicFileSystemServerParameter.httpServerUri, TestInstanceHostUtils.getAFSPath());
+        return configuration;
     }
 
 }

test-integration/sourceTest/java/ch/ethz/sis/openbis/systemtests/TestLogger.java → test-integration/sourceTest/java/ch/ethz/sis/openbis/systemtests/common/IntegrationTestLogger.java

-package ch.ethz.sis.openbis.systemtests;
+package ch.ethz.sis.openbis.systemtests.common;
 
-public class TestLogger
+public class IntegrationTestLogger
 {
 
     public void log(String message)

test-integration/sourceTest/java/ch/ethz/sis/openbis/systemtests/common/IntegrationTestOpenBIS.java

+package ch.ethz.sis.openbis.systemtests.common;
+
+import ch.ethz.sis.openbis.generic.OpenBIS;
+import ch.systemsx.cisd.openbis.generic.shared.util.TestInstanceHostUtils;
+
+public class IntegrationTestOpenBIS extends OpenBIS
+{
+
+    public IntegrationTestOpenBIS()
+    {
+        super(TestInstanceHostUtils.getOpenBISUrl() + TestInstanceHostUtils.getOpenBISPath(),
+                TestInstanceHostUtils.getDSSUrl() + TestInstanceHostUtils.getDSSPath(),
+                TestInstanceHostUtils.getAFSUrl() + TestInstanceHostUtils.getAFSPath());
+    }
+}