From 118daf79f55bfb2ec5e8a6818f7ba32840053fb5 Mon Sep 17 00:00:00 2001
From: buczekp <buczekp>
Date: Tue, 27 Apr 2010 08:34:25 +0000
Subject: [PATCH] [LMS-1507] improved speed of loading datasets by codes;
 improved speed of updating data set statuses

SVN: 15653
---
 .../generic/server/AbstractCommonServer.java  |  4 +-
 .../openbis/generic/server/CommonServer.java  |  8 +-
 .../server/business/bo/ExternalDataTable.java | 18 ++---
 .../business/bo/IExternalDataTable.java       |  9 ++-
 .../server/dataaccess/IExternalDataDAO.java   |  5 ++
 .../server/dataaccess/IGenericDAO.java        | 12 ++-
 .../db/AbstractGenericEntityDAO.java          | 28 +++++--
 .../server/dataaccess/db/ExternalDataDAO.java | 52 +++++++++++-
 .../dataaccess/db/VocabularyTermDAO.java      |  6 --
 .../openbis/generic/shared/dto/DataPE.java    |  6 +-
 .../generic/shared/dto/ExternalDataPE.java    |  6 +-
 .../generic/server/CommonServerTest.java      |  8 +-
 .../business/bo/ExternalDataTableTest.java    | 81 ++++++++++++-------
 13 files changed, 166 insertions(+), 77 deletions(-)

diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/AbstractCommonServer.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/AbstractCommonServer.java
index eaf986d88e7..50c67a19d50 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/AbstractCommonServer.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/AbstractCommonServer.java
@@ -101,7 +101,7 @@ abstract class AbstractCommonServer<T extends IServer> extends AbstractServer<T>
         Session session = getSession(sessionToken);
         IExternalDataTable externalDataTable =
                 businessObjectFactory.createExternalDataTable(session);
-        externalDataTable.loadByDataSetCodes(datasetCodes, true);
+        externalDataTable.loadByDataSetCodes(datasetCodes, false, true);
         return externalDataTable.archiveDatasets();
     }
 
@@ -110,7 +110,7 @@ abstract class AbstractCommonServer<T extends IServer> extends AbstractServer<T>
         Session session = getSession(sessionToken);
         IExternalDataTable externalDataTable =
                 businessObjectFactory.createExternalDataTable(session);
-        externalDataTable.loadByDataSetCodes(datasetCodes, true);
+        externalDataTable.loadByDataSetCodes(datasetCodes, false, true);
         return externalDataTable.unarchiveDatasets();
     }
 
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/CommonServer.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/CommonServer.java
index 4e0f937ed58..b4264fbfd74 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/CommonServer.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/CommonServer.java
@@ -972,7 +972,7 @@ public final class CommonServer extends AbstractCommonServer<ICommonServer> impl
         {
             IExternalDataTable externalDataTable =
                     businessObjectFactory.createExternalDataTable(session);
-            externalDataTable.loadByDataSetCodes(dataSetCodes, false);
+            externalDataTable.loadByDataSetCodes(dataSetCodes, false, false);
             List<ExternalDataPE> dataSets = externalDataTable.getExternalData();
             Map<DataSetTypePE, List<ExternalDataPE>> groupedDataSets =
                     new LinkedHashMap<DataSetTypePE, List<ExternalDataPE>>();
@@ -1223,7 +1223,7 @@ public final class CommonServer extends AbstractCommonServer<ICommonServer> impl
         {
             IExternalDataTable externalDataTable =
                     businessObjectFactory.createExternalDataTable(session);
-            externalDataTable.loadByDataSetCodes(dataSetCodes, false);
+            externalDataTable.loadByDataSetCodes(dataSetCodes, true, false);
             return externalDataTable.uploadLoadedDataSetsToCIFEX(uploadContext);
         } catch (final DataAccessException ex)
         {
@@ -1852,7 +1852,7 @@ public final class CommonServer extends AbstractCommonServer<ICommonServer> impl
         Session session = getSession(sessionToken);
         IExternalDataTable externalDataTable =
                 businessObjectFactory.createExternalDataTable(session);
-        externalDataTable.loadByDataSetCodes(datasetCodes, true);
+        externalDataTable.loadByDataSetCodes(datasetCodes, false, true);
         return externalDataTable.lockDatasets();
     }
 
@@ -1861,7 +1861,7 @@ public final class CommonServer extends AbstractCommonServer<ICommonServer> impl
         Session session = getSession(sessionToken);
         IExternalDataTable externalDataTable =
                 businessObjectFactory.createExternalDataTable(session);
-        externalDataTable.loadByDataSetCodes(datasetCodes, true);
+        externalDataTable.loadByDataSetCodes(datasetCodes, false, true);
         return externalDataTable.unlockDatasets();
     }
 
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ExternalDataTable.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ExternalDataTable.java
index 90c50a64cc9..05260ac6020 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ExternalDataTable.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ExternalDataTable.java
@@ -157,19 +157,14 @@ public final class ExternalDataTable extends AbstractExternalDataBusinessObject
         this.externalData = externalData;
     }
 
-    public void loadByDataSetCodes(List<String> dataSetCodes, boolean lockForUpdate)
+    public void loadByDataSetCodes(List<String> dataSetCodes, boolean withProperties,
+            boolean lockForUpdate)
     {
         IExternalDataDAO externalDataDAO = getExternalDataDAO();
+
         externalData = new ArrayList<ExternalDataPE>();
-        for (String dataSetCode : dataSetCodes)
-        {
-            ExternalDataPE dataSet =
-                    externalDataDAO.tryToFindFullDataSetByCode(dataSetCode, true, lockForUpdate);
-            if (dataSet != null)
-            {
-                externalData.add(dataSet);
-            }
-        }
+        externalData.addAll(externalDataDAO.tryToFindFullDataSetByCodes(dataSetCodes,
+                withProperties, lockForUpdate));
     }
 
     public final void loadBySampleTechId(final TechId sampleId)
@@ -566,11 +561,12 @@ public final class ExternalDataTable extends AbstractExternalDataBusinessObject
                 } else
                 {
                     dataSet.setStatus(newStatus);
-                    externalDataDAO.validateAndSaveUpdatedEntity(dataSet);
+                    externalDataDAO.validate(dataSet);
                     counter++;
                 }
             }
         }
+        externalDataDAO.flush();
         return counter;
     }
 
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/IExternalDataTable.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/IExternalDataTable.java
index 3cafd6b129c..936be1bc1ac 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/IExternalDataTable.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/IExternalDataTable.java
@@ -33,11 +33,12 @@ public interface IExternalDataTable
 {
     /**
      * Loads data sets specified by their codes. Data set codes will be ignored if no
-     * {@link ExternalDataPE} could be found. Optionally if <var>lockForUpdate</var> is
-     * <var>true</var> a all updates to loaded data sets from other transactions will be blocked for
-     * until current transaction is finished.
+     * {@link ExternalDataPE} could be found. Properties will be loaded too depending on
+     * <var>withProperties</var> value. Optionally if <var>lockForUpdate</var> is <var>true</var>
+     * all updates to loaded data sets from other transactions will be blocked until current
+     * transaction is finished.
      */
-    void loadByDataSetCodes(List<String> dataSetCodes, boolean lockForUpdate);
+    void loadByDataSetCodes(List<String> dataSetCodes, boolean withProperties, boolean lockForUpdate);
 
     /**
      * Loads data sets which are linked to the sample with given <var>sampleId</var>.
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/IExternalDataDAO.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/IExternalDataDAO.java
index 773e831a466..00b4cf18765 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/IExternalDataDAO.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/IExternalDataDAO.java
@@ -80,6 +80,11 @@ public interface IExternalDataDAO extends IGenericDAO<ExternalDataPE>
     public ExternalDataPE tryToFindFullDataSetByCode(String dataSetCode, boolean withPropertyTypes,
             boolean lockForUpdate);
 
+    /**
+     * Tries to get the full data set for the specified code with optional locking.
+     */
+    public List<ExternalDataPE> tryToFindFullDataSetByCodes(Collection<String> dataSetCodes,
+            boolean withPropertyTypes, boolean lockForUpdate);
 
     /**
      * Sets status of dataset with given code.
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/IGenericDAO.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/IGenericDAO.java
index 5127f4c70b8..4d0ae91d6fe 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/IGenericDAO.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/IGenericDAO.java
@@ -63,6 +63,16 @@ public interface IGenericDAO<T extends IIdHolder>
      */
     public void validateAndSaveUpdatedEntity(T entity);
 
+    // Hibernate specific
+
+    public void validate(T entity);
+
+    public void flush();
+
+    public void clearSession();
+
+    //
+
     /**
      * Persists given entity.
      * 
@@ -84,6 +94,4 @@ public interface IGenericDAO<T extends IIdHolder>
      */
     public List<T> listAllEntities() throws DataAccessException;
 
-    public void clearSession();
-
 }
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/AbstractGenericEntityDAO.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/AbstractGenericEntityDAO.java
index 74c19b0ee30..ab201d4d2f8 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/AbstractGenericEntityDAO.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/AbstractGenericEntityDAO.java
@@ -125,6 +125,29 @@ public abstract class AbstractGenericEntityDAO<T extends IIdHolder> extends Abst
         getHibernateTemplate().flush();
     }
 
+    public final void validate(T entity)
+    {
+        assert entity != null : "entity is null";
+
+        // as long as CODE cannot be edited we don't have to translate it with a converter here
+        // because the code set in updated entity should be the one already translated during save
+        // but if we allow it this will have to be changed for entities with codes e.g.
+        // like for experiment:
+        // experiment.setCode(CodeConverter.tryToDatabase(experiment.getCode()));
+
+        validatePE(entity);
+    }
+
+    public void clearSession()
+    {
+        getHibernateTemplate().clear();
+    }
+
+    public final void flush()
+    {
+        getHibernateTemplate().flush();
+    }
+
     public void persist(T entity)
     {
         assert entity != null : "entity unspecified";
@@ -162,9 +185,4 @@ public abstract class AbstractGenericEntityDAO<T extends IIdHolder> extends Abst
         return cast(getHibernateTemplate().loadAll(getEntityClass()));
     }
 
-    public void clearSession()
-    {
-        getHibernateTemplate().clear();
-    }
-
 }
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/ExternalDataDAO.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/ExternalDataDAO.java
index db57995523b..e6e07ce9b5f 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/ExternalDataDAO.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/ExternalDataDAO.java
@@ -184,11 +184,17 @@ final class ExternalDataDAO extends AbstractGenericEntityDAO<ExternalDataPE> imp
     {
         assert dataSetCode != null : "Unspecified data set code.";
 
-        String name = ENTITY_SUPER_CLASS.getSimpleName();
-        String hql = String.format("select e from %s e where e.code = ?", name);
-        String normalizedCode = CodeConverter.tryToDatabase(dataSetCode);
-        final List<DataPE> list = cast(getHibernateTemplate().find(hql, toArray(normalizedCode)));
+        final String mangledCode = CodeConverter.tryToDatabase(dataSetCode);
+        final Criterion codeEq = Restrictions.eq("code", mangledCode);
+
+        final DetachedCriteria criteria = DetachedCriteria.forClass(ENTITY_SUPER_CLASS);
+        criteria.add(codeEq);
+        criteria.setFetchMode("dataSetType", FetchMode.JOIN);
+        criteria.setFetchMode("dataStore", FetchMode.JOIN);
+        criteria.setResultTransformer(DetachedCriteria.DISTINCT_ROOT_ENTITY);
+        final List<DataPE> list = cast(getHibernateTemplate().findByCriteria(criteria));
         final DataPE entity = tryFindEntity(list, "data set");
+
         if (operationLog.isDebugEnabled())
         {
             String methodName = MethodUtils.getCurrentMethod().getName();
@@ -197,6 +203,39 @@ final class ExternalDataDAO extends AbstractGenericEntityDAO<ExternalDataPE> imp
         return entity;
     }
 
+    public List<ExternalDataPE> tryToFindFullDataSetByCodes(Collection<String> dataSetCodes,
+            boolean withPropertyTypes, boolean lockForUpdate)
+    {
+        assert dataSetCodes != null : "Unspecified collection";
+
+        final Criterion codeIn = Restrictions.in("code", dataSetCodes);
+
+        final DetachedCriteria criteria = DetachedCriteria.forClass(ENTITY_CLASS);
+        criteria.add(codeIn);
+        criteria.setFetchMode("dataSetType", FetchMode.SELECT);
+        criteria.setFetchMode("dataStore", FetchMode.SELECT);
+        criteria.setFetchMode("experimentInternal", FetchMode.SELECT);
+        criteria.setFetchMode("sampleInternal", FetchMode.SELECT);
+        criteria.setFetchMode("fileFormat", FetchMode.SELECT);
+        if (withPropertyTypes)
+        {
+            criteria.setFetchMode("dataSetType.dataSetTypePropertyTypesInternal", FetchMode.JOIN);
+        }
+        criteria.setResultTransformer(DetachedCriteria.DISTINCT_ROOT_ENTITY);
+        if (lockForUpdate)
+        {
+            criteria.setLockMode(LockMode.UPGRADE);
+        }
+        final List<ExternalDataPE> list = cast(getHibernateTemplate().findByCriteria(criteria));
+
+        if (operationLog.isDebugEnabled())
+        {
+            operationLog.debug(String.format("Found '%s' data sets for codes '%s'.", list.size(),
+                    dataSetCodes));
+        }
+        return list;
+    }
+
     public ExternalDataPE tryToFindFullDataSetByCode(String dataSetCode, boolean withPropertyTypes,
             boolean lockForUpdate)
     {
@@ -207,6 +246,11 @@ final class ExternalDataDAO extends AbstractGenericEntityDAO<ExternalDataPE> imp
 
         final DetachedCriteria criteria = DetachedCriteria.forClass(ENTITY_CLASS);
         criteria.add(codeEq);
+        criteria.setFetchMode("dataSetType", FetchMode.SELECT);
+        criteria.setFetchMode("dataStore", FetchMode.SELECT);
+        criteria.setFetchMode("experimentInternal", FetchMode.SELECT);
+        criteria.setFetchMode("sampleInternal", FetchMode.SELECT);
+        criteria.setFetchMode("fileFormat", FetchMode.SELECT);
         if (withPropertyTypes)
         {
             criteria.setFetchMode("dataSetType.dataSetTypePropertyTypesInternal", FetchMode.JOIN);
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/VocabularyTermDAO.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/VocabularyTermDAO.java
index b73b8bb09f1..d0143779a22 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/VocabularyTermDAO.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/VocabularyTermDAO.java
@@ -51,12 +51,6 @@ final class VocabularyTermDAO extends AbstractGenericEntityDAO<VocabularyTermPE>
         super(sessionFactory, databaseInstance, VocabularyTermPE.class);
     }
 
-    public void validate(VocabularyTermPE pe)
-    {
-        assert pe != null : "pe is null";
-        validatePE(pe);
-    }
-
     public void increaseVocabularyTermOrdinals(VocabularyPE vocabulary, Long fromOrdinal,
             int increment)
     {
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/dto/DataPE.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/dto/DataPE.java
index b2199359841..f8cca9d9b23 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/dto/DataPE.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/dto/DataPE.java
@@ -113,7 +113,7 @@ public class DataPE extends AbstractIdAndCodeHolder<DataPE> implements IEntityPr
 
     private DataStorePE dataStore;
 
-    @ManyToOne(fetch = FetchType.EAGER)
+    @ManyToOne(fetch = FetchType.LAZY)
     @NotNull(message = ValidationMessages.DATA_STORE_NOT_NULL_MESSAGE)
     @JoinColumn(name = ColumnNames.DATA_STORE_COLUMN, updatable = false)
     public DataStorePE getDataStore()
@@ -138,7 +138,7 @@ public class DataPE extends AbstractIdAndCodeHolder<DataPE> implements IEntityPr
         this.registrationDate = registrationDate;
     }
 
-    @ManyToOne(fetch = FetchType.EAGER)
+    @ManyToOne(fetch = FetchType.LAZY)
     @NotNull(message = ValidationMessages.DATA_SET_TYPE_NOT_NULL_MESSAGE)
     @JoinColumn(name = ColumnNames.DATA_SET_TYPE_COLUMN)
     @IndexedEmbedded(prefix = SearchFieldConstants.PREFIX_ENTITY_TYPE)
@@ -229,7 +229,7 @@ public class DataPE extends AbstractIdAndCodeHolder<DataPE> implements IEntityPr
         return getSampleInternal();
     }
 
-    @ManyToOne(fetch = FetchType.EAGER)
+    @ManyToOne(fetch = FetchType.LAZY)
     @JoinColumn(name = ColumnNames.SAMPLE_COLUMN)
     private SamplePE getSampleInternal()
     {
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/dto/ExternalDataPE.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/dto/ExternalDataPE.java
index e8543298e5b..079aa5e6b90 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/dto/ExternalDataPE.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/dto/ExternalDataPE.java
@@ -77,7 +77,7 @@ public final class ExternalDataPE extends DataPE
      * Returns the id of the locator type of the location of this external data, or
      * <code>null</code> if not yet set.
      */
-    @ManyToOne(fetch = FetchType.EAGER)
+    @ManyToOne(fetch = FetchType.LAZY)
     @NotNull(message = ValidationMessages.LOCATOR_TYPE_NOT_NULL_MESSAGE)
     @JoinColumn(name = ColumnNames.LOCATOR_TYPE_COLUMN, updatable = false)
     public final LocatorTypePE getLocatorType()
@@ -106,7 +106,7 @@ public final class ExternalDataPE extends DataPE
         this.location = location;
     }
 
-    @ManyToOne(fetch = FetchType.EAGER)
+    @ManyToOne(fetch = FetchType.LAZY)
     @NotNull(message = ValidationMessages.STORAGE_FORMAT_NOT_NULL_MESSAGE)
     @JoinColumn(name = ColumnNames.STORAGE_FORMAT_COLUMN, updatable = false)
     public VocabularyTermPE getStorageFormatVocabularyTerm()
@@ -129,7 +129,7 @@ public final class ExternalDataPE extends DataPE
     }
 
     /** Returns <code>fileFormatType</code>. */
-    @ManyToOne(fetch = FetchType.EAGER)
+    @ManyToOne(fetch = FetchType.LAZY)
     @NotNull(message = ValidationMessages.FILE_FORMAT_TYPE_NOT_NULL_MESSAGE)
     @JoinColumn(name = ColumnNames.FILE_FORMAT_TYPE, updatable = true)
     @IndexedEmbedded(prefix = SearchFieldConstants.PREFIX_FILE_FORMAT_TYPE)
diff --git a/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/CommonServerTest.java b/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/CommonServerTest.java
index d7d25f68b2a..25dad538ad8 100644
--- a/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/CommonServerTest.java
+++ b/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/CommonServerTest.java
@@ -1303,7 +1303,7 @@ public final class CommonServerTest extends AbstractServerTestCase
                     one(commonBusinessObjectFactory).createExternalDataTable(SESSION);
                     will(returnValue(externalDataTable));
 
-                    one(externalDataTable).loadByDataSetCodes(dataSetCodes, false);
+                    one(externalDataTable).loadByDataSetCodes(dataSetCodes, false, false);
                     one(externalDataTable).getExternalData();
                     ExternalDataPE ds1 = createDataSet("ds1", "type1");
                     ExternalDataPE ds2 = createDataSet("ds2", "type1");
@@ -1345,7 +1345,7 @@ public final class CommonServerTest extends AbstractServerTestCase
                     one(commonBusinessObjectFactory).createExternalDataTable(SESSION);
                     will(returnValue(externalDataTable));
 
-                    one(externalDataTable).loadByDataSetCodes(dataSetCodes, false);
+                    one(externalDataTable).loadByDataSetCodes(dataSetCodes, true, false);
                     one(externalDataTable).uploadLoadedDataSetsToCIFEX(uploadContext);
                 }
             });
@@ -1366,7 +1366,7 @@ public final class CommonServerTest extends AbstractServerTestCase
                     one(commonBusinessObjectFactory).createExternalDataTable(SESSION);
                     will(returnValue(externalDataTable));
 
-                    one(externalDataTable).loadByDataSetCodes(dataSetCodes, true);
+                    one(externalDataTable).loadByDataSetCodes(dataSetCodes, false, true);
                     one(externalDataTable).archiveDatasets();
                 }
             });
@@ -1387,7 +1387,7 @@ public final class CommonServerTest extends AbstractServerTestCase
                     one(commonBusinessObjectFactory).createExternalDataTable(SESSION);
                     will(returnValue(externalDataTable));
 
-                    one(externalDataTable).loadByDataSetCodes(dataSetCodes, true);
+                    one(externalDataTable).loadByDataSetCodes(dataSetCodes, false, true);
                     one(externalDataTable).unarchiveDatasets();
                 }
             });
diff --git a/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ExternalDataTableTest.java b/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ExternalDataTableTest.java
index e557135a017..be23f8f109a 100644
--- a/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ExternalDataTableTest.java
+++ b/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ExternalDataTableTest.java
@@ -210,7 +210,8 @@ public final class ExternalDataTableTest extends AbstractBOTest
         prepareFindFullDataset(d2, false);
 
         ExternalDataTable externalDataTable = createExternalDataTable();
-        externalDataTable.loadByDataSetCodes(Arrays.asList(d1.getCode(), d2.getCode()), false);
+        externalDataTable.loadByDataSetCodes(Arrays.asList(d1.getCode(), d2.getCode()), false,
+                false);
 
         assertEquals(1, externalDataTable.getExternalData().size());
         assertSame(d1, externalDataTable.getExternalData().get(0));
@@ -229,7 +230,7 @@ public final class ExternalDataTableTest extends AbstractBOTest
         context.checking(new Expectations()
             {
                 {
-                    one(externalDataDAO).tryToFindFullDataSetByCode(result.getCode(), true,
+                    one(externalDataDAO).tryToFindFullDataSetByCode(result.getCode(), false,
                             lockForUpdate);
                     will(returnValue(found ? result : null));
                 }
@@ -254,7 +255,8 @@ public final class ExternalDataTableTest extends AbstractBOTest
             });
 
         ExternalDataTable externalDataTable = createExternalDataTable();
-        externalDataTable.loadByDataSetCodes(Arrays.asList(d1.getCode(), d2.getCode()), false);
+        externalDataTable.loadByDataSetCodes(Arrays.asList(d1.getCode(), d2.getCode()), false,
+                false);
         try
         {
             externalDataTable.deleteLoadedDataSets("");
@@ -291,7 +293,8 @@ public final class ExternalDataTableTest extends AbstractBOTest
             });
 
         ExternalDataTable externalDataTable = createExternalDataTable();
-        externalDataTable.loadByDataSetCodes(Code.extractCodes(Arrays.asList(allDataSets)), false);
+        externalDataTable.loadByDataSetCodes(Code.extractCodes(Arrays.asList(allDataSets)), false,
+                false);
         try
         {
             externalDataTable.deleteLoadedDataSets("");
@@ -334,7 +337,8 @@ public final class ExternalDataTableTest extends AbstractBOTest
             });
 
         ExternalDataTable externalDataTable = createExternalDataTable();
-        externalDataTable.loadByDataSetCodes(Arrays.asList(d1.getCode(), d2.getCode()), false);
+        externalDataTable.loadByDataSetCodes(Arrays.asList(d1.getCode(), d2.getCode()), false,
+                false);
         externalDataTable.deleteLoadedDataSets(reason);
 
         context.assertIsSatisfied();
@@ -391,7 +395,8 @@ public final class ExternalDataTableTest extends AbstractBOTest
             });
 
         ExternalDataTable externalDataTable = createExternalDataTable();
-        externalDataTable.loadByDataSetCodes(Arrays.asList(d1PE.getCode(), d2PE.getCode()), false);
+        externalDataTable.loadByDataSetCodes(Arrays.asList(d1PE.getCode(), d2PE.getCode()), true,
+                false);
         String message = externalDataTable.uploadLoadedDataSetsToCIFEX(uploadContext);
 
         assertEquals(
@@ -428,7 +433,8 @@ public final class ExternalDataTableTest extends AbstractBOTest
             });
 
         ExternalDataTable externalDataTable = createExternalDataTable();
-        externalDataTable.loadByDataSetCodes(Code.extractCodes(Arrays.asList(allDataSets)), false);
+        externalDataTable.loadByDataSetCodes(Code.extractCodes(Arrays.asList(allDataSets)), true,
+                false);
         try
         {
             externalDataTable.uploadLoadedDataSetsToCIFEX(uploadContext);
@@ -564,13 +570,17 @@ public final class ExternalDataTableTest extends AbstractBOTest
                     prepareUpdateDatasetStatus(d2Available2, ARCHIVE_PENDING);
                     prepareUpdateDatasetStatus(d3Available, ARCHIVE_PENDING);
 
+                    prepareFlush();
+
                     prepareArchiving(dataStoreService2, dss2, d2Available1, d2Available2);
                     prepareArchiving(dataStoreService3, dss3, d3Available);
                 }
+
             });
 
         ExternalDataTable externalDataTable = createExternalDataTable();
-        externalDataTable.loadByDataSetCodes(Code.extractCodes(Arrays.asList(allDataSets)), true);
+        externalDataTable.loadByDataSetCodes(Code.extractCodes(Arrays.asList(allDataSets)), false,
+                true);
         int archived = externalDataTable.archiveDatasets();
         assertEquals(3, archived);
 
@@ -602,13 +612,16 @@ public final class ExternalDataTableTest extends AbstractBOTest
                     prepareUpdateDatasetStatus(d2Archived2, UNARCHIVE_PENDING);
                     prepareUpdateDatasetStatus(d3Archived, UNARCHIVE_PENDING);
 
+                    prepareFlush();
+
                     prepareUnarchiving(dataStoreService2, dss2, d2Archived1, d2Archived2);
                     prepareUnarchiving(dataStoreService3, dss3, d3Archived);
                 }
             });
 
         ExternalDataTable externalDataTable = createExternalDataTable();
-        externalDataTable.loadByDataSetCodes(Code.extractCodes(Arrays.asList(allDataSets)), true);
+        externalDataTable.loadByDataSetCodes(Code.extractCodes(Arrays.asList(allDataSets)), false,
+                true);
         int unarchived = externalDataTable.unarchiveDatasets();
         assertEquals(3, unarchived);
 
@@ -622,30 +635,40 @@ public final class ExternalDataTableTest extends AbstractBOTest
             {
                 {
 
-                    one(externalDataDAO).validateAndSaveUpdatedEntity(
-                            with(new BaseMatcher<ExternalDataPE>()
-                                {
-
-                                    public boolean matches(Object item)
-                                    {
-                                        if (item instanceof ExternalDataPE)
-                                        {
-                                            ExternalDataPE actualDataSet = (ExternalDataPE) item;
-                                            return dataSet.equals(actualDataSet)
-                                                    && newStatus == actualDataSet.getStatus();
-                                        }
-                                        return false;
-                                    }
+                    one(externalDataDAO).validate(with(new BaseMatcher<ExternalDataPE>()
+                        {
 
-                                    public void describeTo(Description description)
-                                    {
-                                        description.appendValue("Dataset " + dataSet.getCode()
-                                                + " with status " + newStatus);
-                                    }
+                            public boolean matches(Object item)
+                            {
+                                if (item instanceof ExternalDataPE)
+                                {
+                                    ExternalDataPE actualDataSet = (ExternalDataPE) item;
+                                    return dataSet.equals(actualDataSet)
+                                            && newStatus == actualDataSet.getStatus();
+                                }
+                                return false;
+                            }
+
+                            public void describeTo(Description description)
+                            {
+                                description.appendValue("Dataset " + dataSet.getCode()
+                                        + " with status " + newStatus);
+                            }
+
+                        }));
+                }
+            });
+    }
 
-                                }));
+    private void prepareFlush()
+    {
+        context.checking(new Expectations()
+            {
+                {
+                    one(externalDataDAO).flush();
                 }
             });
+
     }
 
     @SuppressWarnings("unchecked")
-- 
GitLab