From 02daa63bb4afcabc6da9e4232f4ac4dd412a58a2 Mon Sep 17 00:00:00 2001
From: pkupczyk <piotr.kupczyk@id.ethz.ch>
Date: Mon, 18 Mar 2024 15:54:14 +0100
Subject: [PATCH] SSDM-13578 : 2PT : Database and V3 Implementation - make sure
 all the resources are properly released

---
 .../asapi/v3/TransactionParticipantApi.java   |  45 +++--
 .../asapi/v3/AbstractTransactionTest.java     |  39 ++++-
 .../asapi/v3/Transaction1PCTest.java          | 160 ++++++++++--------
 .../asapi/v3/Transaction2PCTest.java          | 139 ++++++++-------
 4 files changed, 223 insertions(+), 160 deletions(-)

diff --git a/server-application-server/source/java/ch/ethz/sis/openbis/generic/server/asapi/v3/TransactionParticipantApi.java b/server-application-server/source/java/ch/ethz/sis/openbis/generic/server/asapi/v3/TransactionParticipantApi.java
index 290709fa540..d7790ca5c09 100644
--- a/server-application-server/source/java/ch/ethz/sis/openbis/generic/server/asapi/v3/TransactionParticipantApi.java
+++ b/server-application-server/source/java/ch/ethz/sis/openbis/generic/server/asapi/v3/TransactionParticipantApi.java
@@ -23,7 +23,6 @@ import org.springframework.transaction.support.DefaultTransactionDefinition;
 import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi;
 import ch.ethz.sis.openbis.generic.asapi.v3.ITransactionCoordinatorApi;
 import ch.ethz.sis.openbis.generic.asapi.v3.ITransactionParticipantApi;
-import ch.ethz.sis.transaction.AbstractTransaction;
 import ch.ethz.sis.transaction.IDatabaseTransactionProvider;
 import ch.ethz.sis.transaction.ISessionTokenProvider;
 import ch.ethz.sis.transaction.ITransactionLog;
@@ -275,12 +274,13 @@ public class TransactionParticipantApi extends AbstractTransactionNodeApi implem
                             TransactionStatus transactionStatus = (TransactionStatus) transaction;
                             if (!transactionStatus.isCompleted())
                             {
+                                // The prepared transaction already got rolled back in the database (see above), here we are just releasing the resources (e.g. database connection).
                                 transactionManager.rollback((TransactionStatus) transaction);
                             }
                         } catch (Exception e)
                         {
                             operationLog.warn(
-                                    "Prepared database transaction '" + transactionId + "' could not be rolled back in the transaction manager.",
+                                    "Prepared database transaction '" + transactionId + "' could not be closed in the transaction manager.",
                                     e);
                         }
                     }
@@ -304,20 +304,41 @@ public class TransactionParticipantApi extends AbstractTransactionNodeApi implem
         {
             if (isTwoPhaseTransaction)
             {
-                if (isTransactionPreparedInDatabase(transactionId))
+                try
                 {
-                    try (Connection connection = databaseContext.getDataSource().getConnection();
-                            PreparedStatement commitStatement = connection.prepareStatement("COMMIT PREPARED '" + transactionId + "'"))
+                    if (isTransactionPreparedInDatabase(transactionId))
+                    {
+                        try (Connection connection = databaseContext.getDataSource().getConnection();
+                                PreparedStatement commitStatement = connection.prepareStatement("COMMIT PREPARED '" + transactionId + "'"))
+                        {
+                            commitStatement.execute();
+                            operationLog.info("Prepared database transaction '" + transactionId + "' was committed.");
+                        }
+                    } else
                     {
-                        commitStatement.execute();
-                        operationLog.info("Prepared database transaction '" + transactionId + "' was committed.");
+                        throw new IllegalStateException(
+                                "Prepared database transaction '" + transactionId + "' was not found in the database and could not be committed.");
                     }
-
-                    // Calling transactionManager.commit() after the prepared transaction got committed above is not allowed, therefore we skip it.
-                } else
+                } finally
                 {
-                    throw new IllegalStateException(
-                            "Prepared database transaction '" + transactionId + "' was not found in the database and could not be committed.");
+                    if (transaction != null)
+                    {
+                        try
+                        {
+                            TransactionStatus transactionStatus = (TransactionStatus) transaction;
+                            if (!transactionStatus.isCompleted())
+                            {
+                                // The prepared transaction already got committed in the database (see above), here we are just releasing the resources (e.g. database connection).
+                                // We are intentionally calling rollback as the second commit would fail (the already committed data is safe - it won't be rolled back).
+                                transactionManager.rollback((TransactionStatus) transaction);
+                            }
+                        } catch (Exception e)
+                        {
+                            operationLog.warn(
+                                    "Prepared database transaction '" + transactionId + "' could not be closed in the transaction manager.",
+                                    e);
+                        }
+                    }
                 }
             } else
             {
diff --git a/server-application-server/sourceTest/java/ch/ethz/sis/openbis/systemtest/asapi/v3/AbstractTransactionTest.java b/server-application-server/sourceTest/java/ch/ethz/sis/openbis/systemtest/asapi/v3/AbstractTransactionTest.java
index ba3b3eae293..3fc0e121ab8 100644
--- a/server-application-server/sourceTest/java/ch/ethz/sis/openbis/systemtest/asapi/v3/AbstractTransactionTest.java
+++ b/server-application-server/sourceTest/java/ch/ethz/sis/openbis/systemtest/asapi/v3/AbstractTransactionTest.java
@@ -211,6 +211,28 @@ public class AbstractTransactionTest extends AbstractTest
                     transactionCountLimit);
         }
 
+        public void close()
+        {
+            String sessionToken = v3api.loginAsSystem();
+
+            for (TransactionCoordinator.Transaction transaction : getTransactionMap().values())
+            {
+                try
+                {
+                    if (TransactionStatus.COMMIT_STARTED.equals(transaction.getTransactionStatus()))
+                    {
+                        commitTransaction(transaction.getTransactionId(), sessionToken, TEST_INTERACTIVE_SESSION_KEY);
+                    } else
+                    {
+                        rollbackTransaction(transaction.getTransactionId(), sessionToken, TEST_INTERACTIVE_SESSION_KEY);
+                    }
+                } catch (Exception e)
+                {
+                    operationLog.warn("Could not close transaction '" + transaction.getTransactionId() + "'.", e);
+                }
+            }
+        }
+
     }
 
     public class TestTransactionParticipantApi extends TransactionParticipantApi
@@ -379,15 +401,22 @@ public class AbstractTransactionTest extends AbstractTest
 
         public void close()
         {
-            for (TransactionParticipant.Transaction transaction : super.getTransactionMap().values())
+            String sessionToken = v3api.loginAsSystem();
+
+            getDatabaseTransactionProvider().setRollbackAction(null);
+            getDatabaseTransactionProvider().setCommitAction(null);
+
+            for (TransactionParticipant.Transaction transaction : getTransactionMap().values())
             {
                 try
                 {
-                    transaction.lockOrFail(() ->
+                    if (TransactionStatus.COMMIT_STARTED.equals(transaction.getTransactionStatus()))
+                    {
+                        commitTransaction(transaction.getTransactionId(), sessionToken, TEST_INTERACTIVE_SESSION_KEY);
+                    } else
                     {
-                        transaction.close();
-                        return null;
-                    }, false);
+                        rollbackTransaction(transaction.getTransactionId(), sessionToken, TEST_INTERACTIVE_SESSION_KEY);
+                    }
                 } catch (Exception e)
                 {
                     operationLog.warn("Could not close transaction '" + transaction.getTransactionId() + "'.", e);
diff --git a/server-application-server/sourceTest/java/ch/ethz/sis/openbis/systemtest/asapi/v3/Transaction1PCTest.java b/server-application-server/sourceTest/java/ch/ethz/sis/openbis/systemtest/asapi/v3/Transaction1PCTest.java
index 39d5f632538..b8a79d6a056 100644
--- a/server-application-server/sourceTest/java/ch/ethz/sis/openbis/systemtest/asapi/v3/Transaction1PCTest.java
+++ b/server-application-server/sourceTest/java/ch/ethz/sis/openbis/systemtest/asapi/v3/Transaction1PCTest.java
@@ -47,11 +47,11 @@ public class Transaction1PCTest extends AbstractTransactionTest
     {
         if (participant.getTransactionConfiguration().isEnabled())
         {
-            rollbackPreparedDatabaseTransactions();
-            deleteCreatedSpacesAndProjects();
-
             participant.close();
         }
+
+        rollbackPreparedDatabaseTransactions();
+        deleteCreatedSpacesAndProjects();
     }
 
     @Test
@@ -645,79 +645,95 @@ public class Transaction1PCTest extends AbstractTransactionTest
     @Test
     public void testRecovery()
     {
-        TestTransactionParticipantApi participantBeforeCrash =
-                createParticipant(createConfiguration(true, 60, 10), TEST_PARTICIPANT_1_ID, TRANSACTION_LOG_PARTICIPANT_1_FOLDER);
-
-        // "commit" and "rollback" should fail
-        RuntimeException commitException = new RuntimeException("Test commit exception");
-        RuntimeException rollbackException = new RuntimeException("Test rollback exception");
-
-        participantBeforeCrash.getDatabaseTransactionProvider().setCommitAction(() ->
-        {
-            throw commitException;
-        });
-        participantBeforeCrash.getDatabaseTransactionProvider().setRollbackAction(() ->
-        {
-            throw rollbackException;
-        });
-
-        assertTransactions(participantBeforeCrash.getTransactionMap());
-
-        UUID transactionId1 = UUID.randomUUID();
-        UUID transactionId2 = UUID.randomUUID();
-
-        String sessionToken1 = v3api.login(TEST_USER, PASSWORD);
-        String sessionToken2 = v3api.login(TEST_USER, PASSWORD);
-
-        // begin both transactions
-        participantBeforeCrash.beginTransaction(transactionId1, sessionToken1, TEST_INTERACTIVE_SESSION_KEY, null);
-        participantBeforeCrash.beginTransaction(transactionId2, sessionToken2, TEST_INTERACTIVE_SESSION_KEY, null);
-
-        // create a space in tr1
-        SpaceCreation spaceCreation1 = new SpaceCreation();
-        spaceCreation1.setCode(CODE_PREFIX + UUID.randomUUID());
-
-        participantBeforeCrash.executeOperation(transactionId1, sessionToken1, TEST_INTERACTIVE_SESSION_KEY,
-                OPERATION_CREATE_SPACES, new Object[] { sessionToken2, Collections.singletonList(spaceCreation1) });
+        TestTransactionParticipantApi participantBeforeCrash = null;
+        TestTransactionParticipantApi participantAfterCrash = null;
 
-        // create a space in tr2
-        SpaceCreation spaceCreation2 = new SpaceCreation();
-        spaceCreation2.setCode(CODE_PREFIX + UUID.randomUUID());
-
-        participantBeforeCrash.executeOperation(transactionId2, sessionToken2, TEST_INTERACTIVE_SESSION_KEY,
-                OPERATION_CREATE_SPACES, new Object[] { sessionToken2, Collections.singletonList(spaceCreation2) });
-
-        // failed commit of tr1
         try
         {
-            participantBeforeCrash.commitTransaction(transactionId1, sessionToken1, TEST_INTERACTIVE_SESSION_KEY);
-            fail();
-        } catch (Exception e)
-        {
-            assertEquals(e.getMessage(), "Commit transaction '" + transactionId1 + "' failed.");
-            assertEquals(e.getCause(), commitException);
+            participantBeforeCrash =
+                    createParticipant(createConfiguration(true, 60, 10), TEST_PARTICIPANT_1_ID, TRANSACTION_LOG_PARTICIPANT_1_FOLDER);
+
+            // "commit" and "rollback" should fail
+            RuntimeException commitException = new RuntimeException("Test commit exception");
+            RuntimeException rollbackException = new RuntimeException("Test rollback exception");
+
+            participantBeforeCrash.getDatabaseTransactionProvider().setCommitAction(() ->
+            {
+                throw commitException;
+            });
+            participantBeforeCrash.getDatabaseTransactionProvider().setRollbackAction(() ->
+            {
+                throw rollbackException;
+            });
+
+            assertTransactions(participantBeforeCrash.getTransactionMap());
+
+            UUID transactionId1 = UUID.randomUUID();
+            UUID transactionId2 = UUID.randomUUID();
+
+            String sessionToken1 = v3api.login(TEST_USER, PASSWORD);
+            String sessionToken2 = v3api.login(TEST_USER, PASSWORD);
+
+            // begin both transactions
+            participantBeforeCrash.beginTransaction(transactionId1, sessionToken1, TEST_INTERACTIVE_SESSION_KEY, null);
+            participantBeforeCrash.beginTransaction(transactionId2, sessionToken2, TEST_INTERACTIVE_SESSION_KEY, null);
+
+            // create a space in tr1
+            SpaceCreation spaceCreation1 = new SpaceCreation();
+            spaceCreation1.setCode(CODE_PREFIX + UUID.randomUUID());
+
+            participantBeforeCrash.executeOperation(transactionId1, sessionToken1, TEST_INTERACTIVE_SESSION_KEY,
+                    OPERATION_CREATE_SPACES, new Object[] { sessionToken2, Collections.singletonList(spaceCreation1) });
+
+            // create a space in tr2
+            SpaceCreation spaceCreation2 = new SpaceCreation();
+            spaceCreation2.setCode(CODE_PREFIX + UUID.randomUUID());
+
+            participantBeforeCrash.executeOperation(transactionId2, sessionToken2, TEST_INTERACTIVE_SESSION_KEY,
+                    OPERATION_CREATE_SPACES, new Object[] { sessionToken2, Collections.singletonList(spaceCreation2) });
+
+            // failed commit of tr1
+            try
+            {
+                participantBeforeCrash.commitTransaction(transactionId1, sessionToken1, TEST_INTERACTIVE_SESSION_KEY);
+                fail();
+            } catch (Exception e)
+            {
+                assertEquals(e.getMessage(), "Commit transaction '" + transactionId1 + "' failed.");
+                assertEquals(e.getCause(), commitException);
+            }
+
+            assertTransactions(participantBeforeCrash.getTransactionMap(), new TestTransaction(transactionId1, TransactionStatus.ROLLBACK_STARTED),
+                    new TestTransaction(transactionId2, TransactionStatus.BEGIN_FINISHED));
+
+            // new participant
+            participantAfterCrash =
+                    createParticipant(createConfiguration(true, 60, 10), TEST_PARTICIPANT_1_ID, TRANSACTION_LOG_PARTICIPANT_1_FOLDER);
+
+            assertTransactions(participantAfterCrash.getTransactionMap());
+
+            // only 2PC transactions are recovered
+            participantAfterCrash.recoverTransactionsFromTransactionLog();
+
+            assertTransactions(participantAfterCrash.getTransactionMap());
+
+            Map<ISpaceId, Space> createdSpaces1 = v3api.getSpaces(sessionToken1,
+                    Collections.singletonList(new SpacePermId(spaceCreation2.getCode())), new SpaceFetchOptions());
+            Map<ISpaceId, Space> createdSpaces2 = v3api.getSpaces(sessionToken2,
+                    Collections.singletonList(new SpacePermId(spaceCreation2.getCode())), new SpaceFetchOptions());
+            assertEquals(createdSpaces1.size(), 0);
+            assertEquals(createdSpaces2.size(), 0);
+        } finally
+        {
+            if (participantBeforeCrash != null)
+            {
+                participantBeforeCrash.close();
+            }
+            if (participantAfterCrash != null)
+            {
+                participantAfterCrash.close();
+            }
         }
-
-        assertTransactions(participantBeforeCrash.getTransactionMap(), new TestTransaction(transactionId1, TransactionStatus.ROLLBACK_STARTED),
-                new TestTransaction(transactionId2, TransactionStatus.BEGIN_FINISHED));
-
-        // new participant
-        TestTransactionParticipantApi participantAfterCrash =
-                createParticipant(createConfiguration(true, 60, 10), TEST_PARTICIPANT_1_ID, TRANSACTION_LOG_PARTICIPANT_1_FOLDER);
-
-        assertTransactions(participantAfterCrash.getTransactionMap());
-
-        // only 2PC transactions are recovered
-        participantAfterCrash.recoverTransactionsFromTransactionLog();
-
-        assertTransactions(participantAfterCrash.getTransactionMap());
-
-        Map<ISpaceId, Space> createdSpaces1 = v3api.getSpaces(sessionToken1,
-                Collections.singletonList(new SpacePermId(spaceCreation2.getCode())), new SpaceFetchOptions());
-        Map<ISpaceId, Space> createdSpaces2 = v3api.getSpaces(sessionToken2,
-                Collections.singletonList(new SpacePermId(spaceCreation2.getCode())), new SpaceFetchOptions());
-        assertEquals(createdSpaces1.size(), 0);
-        assertEquals(createdSpaces2.size(), 0);
     }
 
 }
diff --git a/server-application-server/sourceTest/java/ch/ethz/sis/openbis/systemtest/asapi/v3/Transaction2PCTest.java b/server-application-server/sourceTest/java/ch/ethz/sis/openbis/systemtest/asapi/v3/Transaction2PCTest.java
index 07c3329c770..24e11f2e519 100644
--- a/server-application-server/sourceTest/java/ch/ethz/sis/openbis/systemtest/asapi/v3/Transaction2PCTest.java
+++ b/server-application-server/sourceTest/java/ch/ethz/sis/openbis/systemtest/asapi/v3/Transaction2PCTest.java
@@ -27,7 +27,6 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.fetchoptions.SpaceFetchOpt
 import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.id.ISpaceId;
 import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.id.SpacePermId;
 import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.search.SpaceSearchCriteria;
-import ch.ethz.sis.transaction.AbstractTransaction;
 import ch.ethz.sis.transaction.ITransactionParticipant;
 import ch.ethz.sis.transaction.TransactionOperationException;
 import ch.ethz.sis.transaction.TransactionStatus;
@@ -88,25 +87,13 @@ public class Transaction2PCTest extends AbstractTransactionTest
     {
         if (coordinator.getTransactionConfiguration().isEnabled())
         {
-            rollbackPreparedDatabaseTransactions();
-
-            String sessionToken = v3api.loginAsSystem();
-
-            for (AbstractTransaction transaction : coordinator.getTransactionMap().values())
-            {
-                try
-                {
-                    coordinator.rollbackTransaction(transaction.getTransactionId(), sessionToken, TEST_INTERACTIVE_SESSION_KEY);
-                } catch (Exception ignored)
-                {
-                }
-            }
-
-            deleteCreatedSpacesAndProjects();
-
+            coordinator.close();
             participant1.close();
             participant2.close();
         }
+
+        rollbackPreparedDatabaseTransactions();
+        deleteCreatedSpacesAndProjects();
     }
 
     @Test
@@ -1041,81 +1028,91 @@ public class Transaction2PCTest extends AbstractTransactionTest
     @Test
     public void testRecoveryOfParticipantWithTransactionToCommit()
     {
-        List<ITransactionParticipant> participants = new ArrayList<>();
-        participants.add(participant1);
-        participants.add(participant2);
+        TestTransactionParticipantApi participant1AfterCrash = null;
 
-        TestTransactionCoordinatorApi coordinator =
-                createCoordinator(createConfiguration(true, 60, 10), participants, TRANSACTION_LOG_COORDINATOR_FOLDER);
+        try
+        {
+            List<ITransactionParticipant> participants = new ArrayList<>();
+            participants.add(participant1);
+            participants.add(participant2);
 
-        assertTransactions(coordinator.getTransactionMap());
-        assertTransactions(participant1.getTransactionMap());
-        assertTransactions(participant2.getTransactionMap());
+            coordinator = createCoordinator(createConfiguration(true, 60, 10), participants, TRANSACTION_LOG_COORDINATOR_FOLDER);
 
-        String sessionToken = v3api.login(TEST_USER, PASSWORD);
+            assertTransactions(coordinator.getTransactionMap());
+            assertTransactions(participant1.getTransactionMap());
+            assertTransactions(participant2.getTransactionMap());
 
-        coordinator.beginTransaction(coordinatorTrId, sessionToken, TEST_INTERACTIVE_SESSION_KEY);
+            String sessionToken = v3api.login(TEST_USER, PASSWORD);
 
-        SpaceCreation spaceCreation = new SpaceCreation();
-        spaceCreation.setCode(CODE_PREFIX + UUID.randomUUID());
+            coordinator.beginTransaction(coordinatorTrId, sessionToken, TEST_INTERACTIVE_SESSION_KEY);
 
-        coordinator.executeOperation(coordinatorTrId, sessionToken, TEST_INTERACTIVE_SESSION_KEY, participant1.getParticipantId(),
-                OPERATION_CREATE_SPACES, new Object[] { sessionToken, Collections.singletonList(spaceCreation) });
+            SpaceCreation spaceCreation = new SpaceCreation();
+            spaceCreation.setCode(CODE_PREFIX + UUID.randomUUID());
 
-        // "prepare" should fail
-        RuntimeException exception = new RuntimeException("Test commit exception");
-        participant1.getDatabaseTransactionProvider().setCommitAction(() ->
-        {
-            throw exception;
-        });
+            coordinator.executeOperation(coordinatorTrId, sessionToken, TEST_INTERACTIVE_SESSION_KEY, participant1.getParticipantId(),
+                    OPERATION_CREATE_SPACES, new Object[] { sessionToken, Collections.singletonList(spaceCreation) });
 
-        coordinator.commitTransaction(coordinatorTrId, sessionToken, TEST_INTERACTIVE_SESSION_KEY);
+            // "prepare" should fail
+            RuntimeException exception = new RuntimeException("Test commit exception");
+            participant1.getDatabaseTransactionProvider().setCommitAction(() ->
+            {
+                throw exception;
+            });
 
-        assertTransactions(coordinator.getTransactionMap(), new TestTransaction(coordinatorTrId, TransactionStatus.COMMIT_STARTED));
-        assertTransactions(participant1.getTransactionMap(), new TestTransaction(coordinatorTrId, TransactionStatus.COMMIT_STARTED));
-        assertTransactions(participant2.getTransactionMap());
+            coordinator.commitTransaction(coordinatorTrId, sessionToken, TEST_INTERACTIVE_SESSION_KEY);
 
-        TestTransactionParticipantApi participant1AfterCrash =
-                createParticipant(createConfiguration(true, 60, 10), TEST_PARTICIPANT_1_ID, TRANSACTION_LOG_PARTICIPANT_1_FOLDER);
-        participant1AfterCrash.setTestTransactionMapping(Map.of(coordinatorTrId, participant1TrId));
-        // replace original participant with a new instance
-        participants.set(0, participant1AfterCrash);
+            assertTransactions(coordinator.getTransactionMap(), new TestTransaction(coordinatorTrId, TransactionStatus.COMMIT_STARTED));
+            assertTransactions(participant1.getTransactionMap(), new TestTransaction(coordinatorTrId, TransactionStatus.COMMIT_STARTED));
+            assertTransactions(participant2.getTransactionMap());
 
-        assertTransactions(coordinator.getTransactionMap(), new TestTransaction(coordinatorTrId, TransactionStatus.COMMIT_STARTED));
-        assertTransactions(participant1AfterCrash.getTransactionMap());
-        assertTransactions(participant2.getTransactionMap());
+            participant1AfterCrash =
+                    createParticipant(createConfiguration(true, 60, 10), TEST_PARTICIPANT_1_ID, TRANSACTION_LOG_PARTICIPANT_1_FOLDER);
+            participant1AfterCrash.setTestTransactionMapping(Map.of(coordinatorTrId, participant1TrId));
+            // replace original participant with a new instance
+            participants.set(0, participant1AfterCrash);
 
-        Map<ISpaceId, Space> createdSpaces = v3api.getSpaces(sessionToken,
-                Collections.singletonList(new SpacePermId(spaceCreation.getCode())), new SpaceFetchOptions());
-        assertEquals(createdSpaces.size(), 0);
+            assertTransactions(coordinator.getTransactionMap(), new TestTransaction(coordinatorTrId, TransactionStatus.COMMIT_STARTED));
+            assertTransactions(participant1AfterCrash.getTransactionMap());
+            assertTransactions(participant2.getTransactionMap());
 
-        participant1AfterCrash.recoverTransactionsFromTransactionLog();
+            Map<ISpaceId, Space> createdSpaces = v3api.getSpaces(sessionToken,
+                    Collections.singletonList(new SpacePermId(spaceCreation.getCode())), new SpaceFetchOptions());
+            assertEquals(createdSpaces.size(), 0);
 
-        assertTransactions(coordinator.getTransactionMap(), new TestTransaction(coordinatorTrId, TransactionStatus.COMMIT_STARTED));
-        assertTransactions(participant1AfterCrash.getTransactionMap(), new TestTransaction(coordinatorTrId, TransactionStatus.COMMIT_STARTED));
-        assertTransactions(participant2.getTransactionMap());
+            participant1AfterCrash.recoverTransactionsFromTransactionLog();
 
-        participant1AfterCrash.finishFailedOrAbandonedTransactions();
+            assertTransactions(coordinator.getTransactionMap(), new TestTransaction(coordinatorTrId, TransactionStatus.COMMIT_STARTED));
+            assertTransactions(participant1AfterCrash.getTransactionMap(), new TestTransaction(coordinatorTrId, TransactionStatus.COMMIT_STARTED));
+            assertTransactions(participant2.getTransactionMap());
 
-        createdSpaces = v3api.getSpaces(sessionToken,
-                Collections.singletonList(new SpacePermId(spaceCreation.getCode())), new SpaceFetchOptions());
-        assertEquals(createdSpaces.size(), 1);
+            participant1AfterCrash.finishFailedOrAbandonedTransactions();
 
-        assertTransactions(coordinator.getTransactionMap(), new TestTransaction(coordinatorTrId, TransactionStatus.COMMIT_STARTED));
-        assertTransactions(participant1AfterCrash.getTransactionMap());
-        assertTransactions(participant2.getTransactionMap());
+            createdSpaces = v3api.getSpaces(sessionToken,
+                    Collections.singletonList(new SpacePermId(spaceCreation.getCode())), new SpaceFetchOptions());
+            assertEquals(createdSpaces.size(), 1);
 
-        coordinator.finishFailedOrAbandonedTransactions();
+            assertTransactions(coordinator.getTransactionMap(), new TestTransaction(coordinatorTrId, TransactionStatus.COMMIT_STARTED));
+            assertTransactions(participant1AfterCrash.getTransactionMap());
+            assertTransactions(participant2.getTransactionMap());
 
-        assertTransactions(coordinator.getTransactionMap());
-        assertTransactions(participant1AfterCrash.getTransactionMap());
-        assertTransactions(participant2.getTransactionMap());
+            coordinator.finishFailedOrAbandonedTransactions();
 
-        createdSpaces = v3api.getSpaces(sessionToken,
-                Collections.singletonList(new SpacePermId(spaceCreation.getCode())), new SpaceFetchOptions());
-        assertEquals(createdSpaces.size(), 1);
+            assertTransactions(coordinator.getTransactionMap());
+            assertTransactions(participant1AfterCrash.getTransactionMap());
+            assertTransactions(participant2.getTransactionMap());
 
-        participant1AfterCrash.close();
+            createdSpaces = v3api.getSpaces(sessionToken,
+                    Collections.singletonList(new SpacePermId(spaceCreation.getCode())), new SpaceFetchOptions());
+            assertEquals(createdSpaces.size(), 1);
+
+            participant1AfterCrash.close();
+        } finally
+        {
+            if (participant1AfterCrash != null)
+            {
+                participant1AfterCrash.close();
+            }
+        }
     }
 
 }
-- 
GitLab