From 60e1d875bd6f4adc359ceaff2155c994152044e9 Mon Sep 17 00:00:00 2001
From: pkupczyk <pkupczyk>
Date: Wed, 5 Mar 2014 13:05:46 +0000
Subject: [PATCH] SP-1226 / BIS-677 : Data Set Size Filling Maintenance Task -
fill in sizes in chunks
SVN: 30819
---
...penbisDBFromPathInfoDBMaintenanceTask.java | 85 ++++++++---
.../server/EncapsulatedOpenBISService.java | 4 +-
.../shared/IEncapsulatedOpenBISService.java | 2 +-
...isDBFromPathInfoDBMaintenanceTaskTest.java | 144 ++++++++++++++++--
.../server/ServiceForDataStoreServer.java | 4 +-
.../ServiceForDataStoreServerLogger.java | 4 +-
.../bo/datasetlister/DatasetLister.java | 7 +-
.../bo/datasetlister/IDatasetLister.java | 2 +-
.../datasetlister/IDatasetListingQuery.java | 4 +-
.../shared/IServiceForDataStoreServer.java | 2 +-
.../server/ServiceForDataStoreServerTest.java | 9 +-
11 files changed, 213 insertions(+), 54 deletions(-)
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.java
index 1b5abd2317d..5289a865bf3 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.java
@@ -30,6 +30,10 @@ import org.apache.log4j.Logger;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.maintenance.IMaintenanceTask;
+import ch.systemsx.cisd.common.properties.PropertyUtils;
+import ch.systemsx.cisd.common.time.DateTimeUtils;
+import ch.systemsx.cisd.common.utilities.ITimeProvider;
+import ch.systemsx.cisd.common.utilities.SystemTimeProvider;
import ch.systemsx.cisd.etlserver.path.IPathsInfoDAO;
import ch.systemsx.cisd.etlserver.path.PathEntryDTO;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
@@ -46,64 +50,103 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask impl
private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.class);
+ static final String CHUNK_SIZE_KEY = "data-set-chunk-size";
+
+ static final String TIME_LIMIT_KEY = "time-limit";
+
+ static final int DEFAULT_CHUNK_SIZE = 100;
+
+ static final int DEFAULT_TIME_LIMIT = 1000 * 60 * 60;
+
private IEncapsulatedOpenBISService service;
private IPathsInfoDAO dao;
+ private ITimeProvider timeProvider;
+
+ private int chunkSize;
+
+ private long timeLimit;
+
public FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask()
{
+ service = ServiceProvider.getOpenBISService();
+ dao = createDAO();
+ timeProvider = SystemTimeProvider.SYSTEM_TIME_PROVIDER;
}
- public FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask(IEncapsulatedOpenBISService service, IPathsInfoDAO dao)
+ public FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask(IEncapsulatedOpenBISService service, IPathsInfoDAO dao,
+ ITimeProvider timeProvider)
{
this.service = service;
this.dao = dao;
+ this.timeProvider = timeProvider;
}
@Override
public void setUp(String pluginName, Properties properties)
{
- service = ServiceProvider.getOpenBISService();
- dao = createDAO();
+ chunkSize = PropertyUtils.getInt(properties, CHUNK_SIZE_KEY, DEFAULT_CHUNK_SIZE);
+ timeLimit = DateTimeUtils.getDurationInMillis(properties, TIME_LIMIT_KEY, DEFAULT_TIME_LIMIT);
+ operationLog.info(pluginName + " initialized with chunk size = " + chunkSize + ", time limit = " + DateTimeUtils.renderDuration(timeLimit));
}
+ @SuppressWarnings("null")
@Override
public void execute()
{
operationLog.info("Start filling.");
- List<SimpleDataSetInformationDTO> dataSets = service.listPhysicalDataSetsWithUnknownSize();
+ List<SimpleDataSetInformationDTO> dataSets = null;
- operationLog.info("Found " + (dataSets != null ? dataSets.size() : 0) + " dataset(s) with unknown size in openbis database.");
+ long startTime = timeProvider.getTimeInMilliseconds();
+ boolean foundDataSets = false;
+ boolean reachedTimeLimit = false;
+ int chunkIndex = 1;
- if (dataSets != null && false == dataSets.isEmpty())
+ do
{
- Set<String> codes = new HashSet<String>();
+ dataSets = service.listPhysicalDataSetsWithUnknownSize(chunkSize);
+ foundDataSets = dataSets != null && false == dataSets.isEmpty();
- for (SimpleDataSetInformationDTO dataSet : dataSets)
+ if (foundDataSets)
{
- codes.add(dataSet.getDataSetCode());
- }
+ operationLog.info("Found " + dataSets.size() + " dataset(s) with unknown size in openbis database (chunk: " + chunkIndex + ").");
- List<PathEntryDTO> pathInfoEntries = dao.listDataSetsSize(codes.toArray(new String[codes.size()]));
+ Set<String> codes = new HashSet<String>();
- if (pathInfoEntries != null && false == pathInfoEntries.isEmpty())
- {
- Map<String, Long> sizeMap = new HashMap<String, Long>();
+ for (SimpleDataSetInformationDTO dataSet : dataSets)
+ {
+ codes.add(dataSet.getDataSetCode());
+ }
+
+ List<PathEntryDTO> pathInfoEntries = dao.listDataSetsSize(codes.toArray(new String[codes.size()]));
- for (PathEntryDTO pathInfoEntry : pathInfoEntries)
+ if (pathInfoEntries != null && false == pathInfoEntries.isEmpty())
{
- if (pathInfoEntry.getSizeInBytes() != null)
+ Map<String, Long> sizeMap = new HashMap<String, Long>();
+
+ for (PathEntryDTO pathInfoEntry : pathInfoEntries)
{
- sizeMap.put(pathInfoEntry.getDataSetCode(), pathInfoEntry.getSizeInBytes());
+ if (pathInfoEntry.getSizeInBytes() != null)
+ {
+ sizeMap.put(pathInfoEntry.getDataSetCode(), pathInfoEntry.getSizeInBytes());
+ }
}
- }
- operationLog.info("Found sizes for " + sizeMap.size() + " dataset(s) in pathinfo database.");
+ operationLog.info("Found sizes for " + sizeMap.size() + " dataset(s) in pathinfo database (chunk: " + chunkIndex + ").");
- service.updatePhysicalDataSetsSize(sizeMap);
+ service.updatePhysicalDataSetsSize(sizeMap);
+ }
+ } else
+ {
+ operationLog.info("Did not find any datasets with unknown size in openbis database (chunk: " + chunkIndex + ").");
}
- }
+
+ reachedTimeLimit = timeProvider.getTimeInMilliseconds() > startTime + timeLimit;
+ chunkIndex++;
+
+ } while (foundDataSets && false == reachedTimeLimit);
operationLog.info("Filling finished.");
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/EncapsulatedOpenBISService.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/EncapsulatedOpenBISService.java
index 17353cce39b..6130696f4f5 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/EncapsulatedOpenBISService.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/EncapsulatedOpenBISService.java
@@ -665,11 +665,11 @@ public final class EncapsulatedOpenBISService implements IEncapsulatedOpenBISSer
}
@Override
- public List<SimpleDataSetInformationDTO> listPhysicalDataSetsWithUnknownSize() throws UserFailureException
+ public List<SimpleDataSetInformationDTO> listPhysicalDataSetsWithUnknownSize(int chunkSize) throws UserFailureException
{
List<SimpleDataSetInformationDTO> dataSets =
service.listPhysicalDataSetsWithUnknownSize(session.getSessionToken(),
- session.getDataStoreCode());
+ session.getDataStoreCode(), chunkSize);
return injectDefaultShareIdIfMissing(dataSets);
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IEncapsulatedOpenBISService.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IEncapsulatedOpenBISService.java
index db800139a67..cae0efa585a 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IEncapsulatedOpenBISService.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IEncapsulatedOpenBISService.java
@@ -284,7 +284,7 @@ public interface IEncapsulatedOpenBISService extends IEncapsulatedBasicOpenBISSe
* Returns informations about physical data sets with unknown size that belong to the calling data store server.
*/
@ManagedAuthentication
- public List<SimpleDataSetInformationDTO> listPhysicalDataSetsWithUnknownSize() throws UserFailureException;
+ public List<SimpleDataSetInformationDTO> listPhysicalDataSetsWithUnknownSize(int chunkSize) throws UserFailureException;
/**
* Updates sizes of the specified physical data sets (map key: data set code, map value: data set size).
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest.java
index bc3f464a72f..7c9f25ddb8c 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest.java
@@ -17,10 +17,12 @@
package ch.systemsx.cisd.etlserver.plugins;
import static ch.systemsx.cisd.common.test.ArrayContainsExactlyMatcher.containsExactly;
+import static ch.systemsx.cisd.etlserver.plugins.FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.*;
import java.util.Arrays;
import java.util.Collections;
import java.util.Map;
+import java.util.Properties;
import org.apache.commons.collections.map.HashedMap;
import org.jmock.Expectations;
@@ -29,6 +31,7 @@ import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
+import ch.systemsx.cisd.common.utilities.ITimeProvider;
import ch.systemsx.cisd.etlserver.path.IPathsInfoDAO;
import ch.systemsx.cisd.etlserver.path.PathEntryDTO;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
@@ -46,6 +49,8 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
private IPathsInfoDAO dao;
+ private ITimeProvider timeProvider;
+
private SimpleDataSetInformationDTO dataSet1;
private SimpleDataSetInformationDTO dataSet2;
@@ -62,6 +67,7 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
context = new Mockery();
service = context.mock(IEncapsulatedOpenBISService.class);
dao = context.mock(IPathsInfoDAO.class);
+ timeProvider = context.mock(ITimeProvider.class);
dataSet1 = new SimpleDataSetInformationDTO();
dataSet1.setDataSetCode("DS_1");
@@ -90,12 +96,15 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
context.checking(new Expectations()
{
{
- one(service).listPhysicalDataSetsWithUnknownSize();
+ allowing(timeProvider).getTimeInMilliseconds();
+ will(returnValue(0L));
+
+ one(service).listPhysicalDataSetsWithUnknownSize(DEFAULT_CHUNK_SIZE);
will(returnValue(Collections.emptyList()));
}
});
- execute();
+ execute(null, null);
}
@Test
@@ -104,12 +113,15 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
context.checking(new Expectations()
{
{
- one(service).listPhysicalDataSetsWithUnknownSize();
+ allowing(timeProvider).getTimeInMilliseconds();
+ will(returnValue(0L));
+
+ one(service).listPhysicalDataSetsWithUnknownSize(DEFAULT_CHUNK_SIZE);
will(returnValue(null));
}
});
- execute();
+ execute(null, null);
}
@Test
@@ -118,15 +130,21 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
context.checking(new Expectations()
{
{
- one(service).listPhysicalDataSetsWithUnknownSize();
+ allowing(timeProvider).getTimeInMilliseconds();
+ will(returnValue(0L));
+
+ one(service).listPhysicalDataSetsWithUnknownSize(DEFAULT_CHUNK_SIZE);
will(returnValue(Arrays.asList(dataSet1)));
one(dao).listDataSetsSize(new String[] { dataSet1.getDataSetCode() });
will(returnValue(Collections.emptyList()));
+
+ one(service).listPhysicalDataSetsWithUnknownSize(DEFAULT_CHUNK_SIZE);
+ will(returnValue(Collections.emptyList()));
}
});
- execute();
+ execute(null, null);
}
@Test
@@ -135,24 +153,33 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
context.checking(new Expectations()
{
{
- one(service).listPhysicalDataSetsWithUnknownSize();
+ allowing(timeProvider).getTimeInMilliseconds();
+ will(returnValue(0L));
+
+ one(service).listPhysicalDataSetsWithUnknownSize(DEFAULT_CHUNK_SIZE);
will(returnValue(Arrays.asList(dataSet1)));
one(dao).listDataSetsSize(new String[] { dataSet1.getDataSetCode() });
will(returnValue(null));
+
+ one(service).listPhysicalDataSetsWithUnknownSize(DEFAULT_CHUNK_SIZE);
+ will(returnValue(Collections.emptyList()));
}
});
- execute();
+ execute(null, null);
}
@Test
- public void testExecute()
+ public void testExecuteWithOneChunk()
{
context.checking(new Expectations()
{
{
- one(service).listPhysicalDataSetsWithUnknownSize();
+ allowing(timeProvider).getTimeInMilliseconds();
+ will(returnValue(0L));
+
+ one(service).listPhysicalDataSetsWithUnknownSize(DEFAULT_CHUNK_SIZE);
will(returnValue(Arrays.asList(dataSet1, dataSet2, dataSet3)));
one(dao).listDataSetsSize(
@@ -163,16 +190,107 @@ public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTaskTest
Map<String, Long> sizeMap = new HashedMap<String, Long>();
sizeMap.put("DS_1", 123L);
one(service).updatePhysicalDataSetsSize(sizeMap);
+
+ one(service).listPhysicalDataSetsWithUnknownSize(DEFAULT_CHUNK_SIZE);
+ will(returnValue(Collections.emptyList()));
}
});
- execute();
+ execute(null, null);
}
- private void execute()
+ @Test
+ public void testExecuteWithMultipleChunks()
+ {
+ final int chunkSize = 2;
+
+ context.checking(new Expectations()
+ {
+ {
+ allowing(timeProvider).getTimeInMilliseconds();
+ will(returnValue(0L));
+
+ one(service).listPhysicalDataSetsWithUnknownSize(chunkSize);
+ will(returnValue(Arrays.asList(dataSet1, dataSet2)));
+
+ one(dao).listDataSetsSize(
+ with(containsExactly(dataSet1.getDataSetCode(), dataSet2.getDataSetCode())));
+ will(returnValue(Arrays.asList(entry1, entry2)));
+
+ Map<String, Long> sizeMap = new HashedMap<String, Long>();
+ sizeMap.put("DS_1", 123L);
+ one(service).updatePhysicalDataSetsSize(sizeMap);
+
+ one(service).listPhysicalDataSetsWithUnknownSize(chunkSize);
+ will(returnValue(Arrays.asList(dataSet3)));
+
+ one(dao).listDataSetsSize(new String[] { dataSet3.getDataSetCode() });
+ will(returnValue(Collections.emptyList()));
+
+ one(service).listPhysicalDataSetsWithUnknownSize(chunkSize);
+ will(returnValue(Collections.emptyList()));
+ }
+ });
+
+ execute(null, chunkSize);
+ }
+
+ @Test
+ public void testExecuteWithTimeLimit()
+ {
+ final long timeLimit = 10L;
+ final int chunkSize = 1;
+
+ context.checking(new Expectations()
+ {
+ {
+ one(timeProvider).getTimeInMilliseconds();
+ will(returnValue(0L));
+
+ one(service).listPhysicalDataSetsWithUnknownSize(chunkSize);
+ will(returnValue(Arrays.asList(dataSet1)));
+
+ one(dao).listDataSetsSize(new String[] { dataSet1.getDataSetCode() });
+ will(returnValue(Arrays.asList(entry1)));
+
+ Map<String, Long> sizeMap = new HashedMap<String, Long>();
+ sizeMap.put("DS_1", 123L);
+ one(service).updatePhysicalDataSetsSize(sizeMap);
+
+ one(timeProvider).getTimeInMilliseconds();
+ will(returnValue(8L));
+
+ one(service).listPhysicalDataSetsWithUnknownSize(chunkSize);
+ will(returnValue(Arrays.asList(dataSet2)));
+
+ one(dao).listDataSetsSize(new String[] { dataSet2.getDataSetCode() });
+ will(returnValue(Collections.emptyList()));
+
+ one(timeProvider).getTimeInMilliseconds();
+ will(returnValue(12L));
+ }
+ });
+
+ execute(timeLimit, chunkSize);
+ }
+
+ private void execute(Long timeLimit, Integer chunkSize)
{
FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask task =
- new FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask(service, dao);
+ new FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask(service, dao, timeProvider);
+
+ Properties properties = new Properties();
+
+ if (timeLimit != null)
+ {
+ properties.setProperty(TIME_LIMIT_KEY, timeLimit.toString() + " ms");
+ }
+ if (chunkSize != null)
+ {
+ properties.setProperty(CHUNK_SIZE_KEY, chunkSize.toString());
+ }
+
+ task.setUp("fill-unknown-sizes", properties);
task.execute();
}
}
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ServiceForDataStoreServer.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ServiceForDataStoreServer.java
index b8f12dd6c98..ac1616f1c75 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ServiceForDataStoreServer.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ServiceForDataStoreServer.java
@@ -1292,13 +1292,13 @@ public class ServiceForDataStoreServer extends AbstractCommonServer<IServiceForD
@Override
@RolesAllowed(RoleWithHierarchy.SPACE_ETL_SERVER)
- public List<SimpleDataSetInformationDTO> listPhysicalDataSetsWithUnknownSize(String sessionToken, String dataStoreCode)
+ public List<SimpleDataSetInformationDTO> listPhysicalDataSetsWithUnknownSize(String sessionToken, String dataStoreCode, int chunkSize)
{
final Session session = getSession(sessionToken);
final DataStorePE dataStore = loadDataStore(session, dataStoreCode);
final IDatasetLister datasetLister = businessObjectFactory.createDatasetLister(session);
final List<AbstractExternalData> dataSets =
- datasetLister.listByDataStoreWithUnknownSize(dataStore.getId(),
+ datasetLister.listByDataStoreWithUnknownSize(dataStore.getId(), chunkSize,
DATASET_FETCH_OPTIONS_FILE_DATASETS);
return SimpleDataSetHelper.filterAndTranslate(dataSets);
}
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ServiceForDataStoreServerLogger.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ServiceForDataStoreServerLogger.java
index 177f144b56a..802d3ee6ed2 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ServiceForDataStoreServerLogger.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ServiceForDataStoreServerLogger.java
@@ -417,9 +417,9 @@ public class ServiceForDataStoreServerLogger extends AbstractServerLogger implem
}
@Override
- public List<SimpleDataSetInformationDTO> listPhysicalDataSetsWithUnknownSize(String sessionToken, String dataStoreCode)
+ public List<SimpleDataSetInformationDTO> listPhysicalDataSetsWithUnknownSize(String sessionToken, String dataStoreCode, int chunkSize)
{
- logAccess(Level.DEBUG, sessionToken, "listPhysicalDataSetsWithUnknownSize", "DATA_STORE(%s)", dataStoreCode);
+ logAccess(Level.DEBUG, sessionToken, "listPhysicalDataSetsWithUnknownSize", "DATA_STORE(%s) CHUNK_SIZE(%s)", dataStoreCode, chunkSize);
return null;
}
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/datasetlister/DatasetLister.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/datasetlister/DatasetLister.java
index d1142e75100..932a25ecfca 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/datasetlister/DatasetLister.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/datasetlister/DatasetLister.java
@@ -491,13 +491,10 @@ public class DatasetLister extends AbstractLister implements IDatasetLister
}
@Override
- public List<AbstractExternalData> listByDataStoreWithUnknownSize(long dataStoreID, EnumSet<DataSetFetchOption> datasetFetchOptions)
+ public List<AbstractExternalData> listByDataStoreWithUnknownSize(long dataStoreID, int limit, EnumSet<DataSetFetchOption> datasetFetchOptions)
{
checkFetchOptions(datasetFetchOptions);
- return orderByDate(enrichDatasets(
- handleDegenerateRegistrationTimestamp(
- query.getDatasetsByDataStoreIdWithUnknownSize(dataStoreID),
- dataStoreID), datasetFetchOptions));
+ return orderByDate(enrichDatasets(query.getDatasetsByDataStoreIdWithUnknownSize(dataStoreID, limit), datasetFetchOptions));
}
private Iterable<DatasetRecord> handleDegenerateRegistrationTimestamp(List<DatasetRecord> list,
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/datasetlister/IDatasetLister.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/datasetlister/IDatasetLister.java
index 6ca0063c3e9..e06e5601963 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/datasetlister/IDatasetLister.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/datasetlister/IDatasetLister.java
@@ -128,7 +128,7 @@ public interface IDatasetLister
/**
* Lists physical datasets with unknown size of the specified data store.
*/
- List<AbstractExternalData> listByDataStoreWithUnknownSize(long dataStoreID, EnumSet<DataSetFetchOption> datasetFetchOptions);
+ List<AbstractExternalData> listByDataStoreWithUnknownSize(long dataStoreID, int limit, EnumSet<DataSetFetchOption> datasetFetchOptions);
/**
* Lists {@link DataSetShareId}s of all data sets (even those in trash) in specified data store.
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/datasetlister/IDatasetListingQuery.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/datasetlister/IDatasetListingQuery.java
index 5bafe710d77..885dec3c801 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/datasetlister/IDatasetListingQuery.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/datasetlister/IDatasetListingQuery.java
@@ -245,8 +245,8 @@ public interface IDatasetListingQuery extends BaseQuery, IPropertyListingQuery
@Select(sql = SELECT_ALL_EXTERNAL_DATAS
+ " where data.dast_id = ?{1} and is_placeholder = false and size is null"
- + " order by registration_timestamp", fetchSize = FETCH_SIZE)
- public List<DatasetRecord> getDatasetsByDataStoreIdWithUnknownSize(long dataStoreID);
+ + " order by registration_timestamp limit ?{2}", fetchSize = FETCH_SIZE)
+ public List<DatasetRecord> getDatasetsByDataStoreIdWithUnknownSize(long dataStoreID, int limit);
// NOTE: we list ALL data sets (even those in trash) using data_all table here
@Select(sql = "SELECT code, share_id FROM data_all LEFT OUTER JOIN external_data "
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/IServiceForDataStoreServer.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/IServiceForDataStoreServer.java
index 95461422b5e..f5661abb241 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/IServiceForDataStoreServer.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/IServiceForDataStoreServer.java
@@ -437,7 +437,7 @@ public interface IServiceForDataStoreServer extends IServer, ISessionProvider
* Returns informations about physical data sets with unknown size that belong to the specified data store server.
*/
@Transactional(readOnly = true)
- public List<SimpleDataSetInformationDTO> listPhysicalDataSetsWithUnknownSize(String sessionToken, String dataStoreCode);
+ public List<SimpleDataSetInformationDTO> listPhysicalDataSetsWithUnknownSize(String sessionToken, String dataStoreCode, int chunkSize);
/**
* Updates sizes of the specified physical data sets (map key: data set code, map value: data set size).
diff --git a/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/ServiceForDataStoreServerTest.java b/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/ServiceForDataStoreServerTest.java
index d330f608143..22d86700e7a 100644
--- a/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/ServiceForDataStoreServerTest.java
+++ b/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/ServiceForDataStoreServerTest.java
@@ -41,12 +41,13 @@ public class ServiceForDataStoreServerTest extends SystemTestCase
public void testListPhysicalDataSetsWithUnknownSize()
{
String sessionToken = authenticateAs("test");
- List<SimpleDataSetInformationDTO> dataSetsWithUnknownSize = etlService.listPhysicalDataSetsWithUnknownSize(sessionToken, "STANDARD");
+ List<SimpleDataSetInformationDTO> dataSetsWithUnknownSize = etlService.listPhysicalDataSetsWithUnknownSize(sessionToken, "STANDARD", 3);
sortByCode(dataSetsWithUnknownSize);
- Assert.assertEquals(20, dataSetsWithUnknownSize.size());
+ Assert.assertEquals(3, dataSetsWithUnknownSize.size());
Assert.assertEquals("20081105092159188-3", dataSetsWithUnknownSize.get(0).getDataSetCode());
- Assert.assertEquals("VALIDATIONS_PARENT-28", dataSetsWithUnknownSize.get(dataSetsWithUnknownSize.size() - 1).getDataSetCode());
+ Assert.assertEquals("20081105092259000-8", dataSetsWithUnknownSize.get(1).getDataSetCode());
+ Assert.assertEquals("20081105092259000-9", dataSetsWithUnknownSize.get(2).getDataSetCode());
}
@Test(dependsOnMethods = "testListPhysicalDataSetsWithUnknownSize")
@@ -59,7 +60,7 @@ public class ServiceForDataStoreServerTest extends SystemTestCase
etlService.updatePhysicalDataSetsSize(sessionToken, sizeMap);
- List<SimpleDataSetInformationDTO> dataSetsWithUnknownSize = etlService.listPhysicalDataSetsWithUnknownSize(sessionToken, "STANDARD");
+ List<SimpleDataSetInformationDTO> dataSetsWithUnknownSize = etlService.listPhysicalDataSetsWithUnknownSize(sessionToken, "STANDARD", 100);
List<AbstractExternalData> updatedDataSets = etlService.listDataSetsByCode(sessionToken, Arrays.asList("20081105092159188-3"));
sortByCode(dataSetsWithUnknownSize);
--
GitLab