diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/path/IPathsInfoDAO.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/path/IPathsInfoDAO.java
index 9dd2a9c1ff3be3efd138369f28dd10baab96b9b5..3886f15f9d6005bce6a803fb53b10a83960c5555 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/path/IPathsInfoDAO.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/path/IPathsInfoDAO.java
@@ -20,11 +20,12 @@ import java.util.Collection;
import java.util.Date;
import java.util.List;
-
import net.lemnik.eodsql.Select;
import net.lemnik.eodsql.TransactionQuery;
import net.lemnik.eodsql.Update;
+import ch.systemsx.cisd.openbis.generic.server.dataaccess.db.StringArrayMapper;
+
/**
* Data Access Object for feeding and updating pathinfo database.
*
@@ -42,13 +43,13 @@ public interface IPathsInfoDAO extends TransactionQuery
+ "size_in_bytes, is_directory, last_modified) values (?{1}, ?{2}, ?{3}, ?{4}, ?{5}, ?{6}, ?{7}) returning id")
public long createDataSetFile(long dataSetId, Long parentId, String relativePath,
String fileName, long sizeInBytes, boolean directory, Date lastModifiedDate);
-
+
@Select("select registration_timestamp from last_feeding_event")
public Date getRegistrationTimestampOfLastFeedingEvent();
-
+
@Update("delete from last_feeding_event")
public void deleteLastFeedingEvent();
-
+
@Update("insert into last_feeding_event (registration_timestamp) values (?{1})")
public void createLastFeedingEvent(Date registrationTimestamp);
@@ -57,13 +58,18 @@ public interface IPathsInfoDAO extends TransactionQuery
+ "(?{1.dataSetId}, ?{1.parentId}, ?{1.relativePath}, ?{1.fileName}, ?{1.sizeInBytes}, "
+ "?{1.checksumCRC32}, ?{1.directory}, ?{1.lastModifiedDate})", batchUpdate = true)
public void createDataSetFiles(Collection<PathEntryDTO> filePaths);
-
+
@Select("select f.id, d.code as data_set_code, relative_path " +
- "from data_set_files as f join data_sets as d on f.dase_id = d.id " +
- "where checksum_crc32 is null and is_directory = 'F'")
+ "from data_set_files as f join data_sets as d on f.dase_id = d.id " +
+ "where checksum_crc32 is null and is_directory = 'F'")
public List<PathEntryDTO> listDataSetFilesWithUnkownChecksum();
-
+
@Update("update data_set_files set checksum_crc32 = ?{2} where id = ?{1}")
public void updateChecksum(long id, int checksum);
-
+
+ @Select(sql = "select d.code as data_set_code, size_in_bytes " +
+ "from data_set_files as f join data_sets as d on f.dase_id = d.id " +
+ "where d.code = any(?{1}) and parent_id is null", parameterBindings = { StringArrayMapper.class })
+ public List<PathEntryDTO> listDataSetsSize(String[] dataSetCodes);
+
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/path/PathEntryDTO.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/path/PathEntryDTO.java
index 87cf46950dfaea6a614998f6a36be6155bf0343b..61c2f1095d0002818daf62721ed6f3149e0540eb 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/path/PathEntryDTO.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/path/PathEntryDTO.java
@@ -212,7 +212,7 @@ public class PathEntryDTO
this.fileName = fileName;
}
- public long getSizeInBytes()
+ public Long getSizeInBytes()
{
return sizeInBytes;
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.java
new file mode 100644
index 0000000000000000000000000000000000000000..ef8c02d24fa8d98926bca4c14370d6ffbe287405
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright 2014 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.plugins;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+import net.lemnik.eodsql.QueryTool;
+
+import org.apache.log4j.Logger;
+
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.maintenance.IMaintenanceTask;
+import ch.systemsx.cisd.etlserver.path.IPathsInfoDAO;
+import ch.systemsx.cisd.etlserver.path.PathEntryDTO;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.utils.PathInfoDataSourceProvider;
+import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
+
+/**
+ * @author pkupczyk
+ */
+public class FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask implements IMaintenanceTask
+{
+
+ private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
+ FillUnknownDataSetSizeInOpenbisDBFromPathInfoDBMaintenanceTask.class);
+
+ private IEncapsulatedOpenBISService service;
+
+ private IPathsInfoDAO dao;
+
+ @Override
+ public void setUp(String pluginName, Properties properties)
+ {
+ service = ServiceProvider.getOpenBISService();
+ dao = createDAO();
+ }
+
+ @Override
+ public void execute()
+ {
+ operationLog.info("Start filling.");
+
+ List<SimpleDataSetInformationDTO> dataSets = service.listPhysicalDataSetsWithUnknownSize();
+ Set<String> codes = new HashSet<String>();
+
+ for (SimpleDataSetInformationDTO dataSet : dataSets)
+ {
+ codes.add(dataSet.getDataSetCode());
+ }
+
+ operationLog.info("Found " + codes.size() + " dataset(s) with unknown size in openbis database.");
+
+ if (codes.size() > 0)
+ {
+ List<PathEntryDTO> pathInfoEntries = dao.listDataSetsSize(codes.toArray(new String[codes.size()]));
+ Map<String, Long> sizeMap = new HashMap<String, Long>();
+
+ for (PathEntryDTO pathInfoEntry : pathInfoEntries)
+ {
+ if (pathInfoEntry.getSizeInBytes() != null)
+ {
+ sizeMap.put(pathInfoEntry.getDataSetCode(), pathInfoEntry.getSizeInBytes());
+ }
+ }
+
+ operationLog.info("Found sizes for " + sizeMap.size() + " dataset(s) in pathinfo database.");
+
+ service.updatePhysicalDataSetsSize(sizeMap);
+
+ }
+ operationLog.info("Filling finished.");
+ }
+
+ private static IPathsInfoDAO createDAO()
+ {
+ return QueryTool.getQuery(PathInfoDataSourceProvider.getDataSource(), IPathsInfoDAO.class);
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/EncapsulatedOpenBISService.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/EncapsulatedOpenBISService.java
index 6f30257d28ec33496786f0f42b2f822986672b15..17353cce39b0a36a47b90f46a3ad38e42d839558 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/EncapsulatedOpenBISService.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/EncapsulatedOpenBISService.java
@@ -664,6 +664,21 @@ public final class EncapsulatedOpenBISService implements IEncapsulatedOpenBISSer
return injectDefaultShareIdIfMissing(dataSets);
}
+ @Override
+ public List<SimpleDataSetInformationDTO> listPhysicalDataSetsWithUnknownSize() throws UserFailureException
+ {
+ List<SimpleDataSetInformationDTO> dataSets =
+ service.listPhysicalDataSetsWithUnknownSize(session.getSessionToken(),
+ session.getDataStoreCode());
+ return injectDefaultShareIdIfMissing(dataSets);
+ }
+
+ @Override
+ public void updatePhysicalDataSetsSize(Map<String, Long> sizeMap) throws UserFailureException
+ {
+ service.updatePhysicalDataSetsSize(session.getSessionToken(), sizeMap);
+ }
+
private List<SimpleDataSetInformationDTO> injectDefaultShareIdIfMissing(
List<SimpleDataSetInformationDTO> dataSets)
{
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IEncapsulatedOpenBISService.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IEncapsulatedOpenBISService.java
index 793fef241868a225a5237b35204bd5749eccd152..db800139a673648dbc3bc15dc28eccc40c4468c2 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IEncapsulatedOpenBISService.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IEncapsulatedOpenBISService.java
@@ -280,6 +280,18 @@ public interface IEncapsulatedOpenBISService extends IEncapsulatedBasicOpenBISSe
public List<SimpleDataSetInformationDTO> listOldestPhysicalDataSets(Date youngerThan,
int chunkSize) throws UserFailureException;
+ /**
+ * Returns informations about physical data sets with unknown size that belong to the calling data store server.
+ */
+ @ManagedAuthentication
+ public List<SimpleDataSetInformationDTO> listPhysicalDataSetsWithUnknownSize() throws UserFailureException;
+
+ /**
+ * Updates sizes of the specified physical data sets (map key: data set code, map value: data set size).
+ */
+ @ManagedAuthentication
+ public void updatePhysicalDataSetsSize(Map<String, Long> sizeMap) throws UserFailureException;
+
/** @see IServiceForDataStoreServer#listDataSets(String, String, TrackingDataSetCriteria) */
@ManagedAuthentication
public List<AbstractExternalData> listNewerDataSets(TrackingDataSetCriteria criteria)
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ServiceForDataStoreServer.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ServiceForDataStoreServer.java
index dec4a9b597d9520ac6ec5b8f48bcbb62e27ca59d..68fd8ed8c6ab87bd19de96c574ca6c3ebb4a47be 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ServiceForDataStoreServer.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ServiceForDataStoreServer.java
@@ -1290,6 +1290,27 @@ public class ServiceForDataStoreServer extends AbstractCommonServer<IServiceForD
return SimpleDataSetHelper.filterAndTranslate(dataSets);
}
+ @Override
+ public List<SimpleDataSetInformationDTO> listPhysicalDataSetsWithUnknownSize(String sessionToken, String dataStoreCode)
+ {
+ final Session session = getSession(sessionToken);
+ final DataStorePE dataStore = loadDataStore(session, dataStoreCode);
+ final IDatasetLister datasetLister = businessObjectFactory.createDatasetLister(session);
+ final List<AbstractExternalData> dataSets =
+ datasetLister.listByDataStoreWithUnknownSize(dataStore.getId(),
+ DATASET_FETCH_OPTIONS_FILE_DATASETS);
+ return SimpleDataSetHelper.filterAndTranslate(dataSets);
+ }
+
+ @Override
+ public void updatePhysicalDataSetsSize(String sessionToken, Map<String, Long> sizeMap)
+ {
+ assert sessionToken != null : "Unspecified session token.";
+ final Session session = getSession(sessionToken);
+ final IDataBO dataBO = businessObjectFactory.createDataBO(session);
+ dataBO.updateSizes(sizeMap);
+ }
+
@Override
@RolesAllowed(RoleWithHierarchy.SPACE_ETL_SERVER)
public List<AbstractExternalData> listAvailableDataSets(String sessionToken,
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ServiceForDataStoreServerLogger.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ServiceForDataStoreServerLogger.java
index d5041d48e16d34b3d8c35c8976fe2ce7b08daa26..177f144b56a2e336b960bcf7c9d70bb378332e77 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ServiceForDataStoreServerLogger.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ServiceForDataStoreServerLogger.java
@@ -17,6 +17,7 @@
package ch.systemsx.cisd.openbis.generic.server;
import java.util.Collection;
+import java.util.Collections;
import java.util.Date;
import java.util.EnumSet;
import java.util.List;
@@ -415,6 +416,20 @@ public class ServiceForDataStoreServerLogger extends AbstractServerLogger implem
return null;
}
+ @Override
+ public List<SimpleDataSetInformationDTO> listPhysicalDataSetsWithUnknownSize(String sessionToken, String dataStoreCode)
+ {
+ logAccess(Level.DEBUG, sessionToken, "listPhysicalDataSetsWithUnknownSize", "DATA_STORE(%s)", dataStoreCode);
+ return null;
+ }
+
+ @Override
+ public void updatePhysicalDataSetsSize(String sessionToken, Map<String, Long> sizeMap)
+ {
+ logAccess(Level.DEBUG, sessionToken, "updatePhysicalDataSetsSize", "CODES(%s)",
+ sizeMap != null ? CollectionUtils.abbreviate(sizeMap.keySet(), 10) : Collections.emptySet());
+ }
+
@Override
public List<AbstractExternalData> listAvailableDataSets(String sessionToken,
String dataStoreCode,
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/DataBO.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/DataBO.java
index c54ac5e329fac8d81e80623de12e8740bad6350d..5fa4905d6a72b772f180519dbba13b5787a676a1 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/DataBO.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/DataBO.java
@@ -22,6 +22,7 @@ import java.util.Collection;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import org.springframework.dao.DataAccessException;
@@ -161,7 +162,7 @@ public class DataBO extends AbstractDataSetBusinessObject implements IDataBO
public void loadDataByTechId(TechId datasetId)
{
String[] connections =
- { PROPERTY_TYPES, DATA_SET_TYPE };
+ { PROPERTY_TYPES, DATA_SET_TYPE };
data = getDataDAO().tryGetByTechId(datasetId, connections);
if (data == null)
{
@@ -769,6 +770,12 @@ public class DataBO extends AbstractDataSetBusinessObject implements IDataBO
getDataDAO().updateDataSetStatuses(dataSetCodes, newStatus, newPresentInArchive);
}
+ @Override
+ public void updateSizes(Map<String, Long> sizeMap)
+ {
+ getDataDAO().updateSizes(sizeMap);
+ }
+
@Override
public boolean compareAndSetDataSetStatus(DataSetArchivingStatus oldStatus,
DataSetArchivingStatus newStatus, boolean newPresentInArchive)
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/IDataBO.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/IDataBO.java
index 7d1469156c6a58df555d249a898c729c85ebe1fd..69136de8cf1ad79a2c8eae4876f7688476355c29 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/IDataBO.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/IDataBO.java
@@ -17,6 +17,7 @@
package ch.systemsx.cisd.openbis.generic.server.business.bo;
import java.util.List;
+import java.util.Map;
import ch.systemsx.cisd.common.exceptions.UserFailureException;
import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.id.dataset.IDataSetId;
@@ -38,37 +39,32 @@ public interface IDataBO extends IEntityBusinessObject
{
/**
- * Returns a data set found by the given id or null if it does not exist. Does not change the
- * state of this object, especially the result of {@link #getData()}.
+ * Returns a data set found by the given id or null if it does not exist. Does not change the state of this object, especially the result of
+ * {@link #getData()}.
*/
DataPE tryFindByDataSetId(final IDataSetId dataSetId);
/**
- * Returns the data item which has been created by
- * {@link #define(NewExternalData, SamplePE, SourceType)} or null.
+ * Returns the data item which has been created by {@link #define(NewExternalData, SamplePE, SourceType)} or null.
*/
public DataPE tryGetData();
/**
- * Returns the data item which has been created by
- * {@link #define(NewExternalData, SamplePE, SourceType)}.
+ * Returns the data item which has been created by {@link #define(NewExternalData, SamplePE, SourceType)}.
*/
public DataPE getData();
/**
* Defines a new external data item directly connected to a sample.
* <p>
- * After invocation of this method {@link IExperimentBO#save()} should be invoked to store the
- * new external data item in the Data Access Layer.
+ * After invocation of this method {@link IExperimentBO#save()} should be invoked to store the new external data item in the Data Access Layer.
*/
public void define(NewExternalData data, SamplePE sample, SourceType sourceType);
/**
- * Defines a new external data item not directly connected to a sample but with mandatory
- * connection with an experiment.
+ * Defines a new external data item not directly connected to a sample but with mandatory connection with an experiment.
* <p>
- * After invocation of this method {@link IExperimentBO#save()} should be invoked to store the
- * new external data item in the Data Access Layer.
+ * After invocation of this method {@link IExperimentBO#save()} should be invoked to store the new external data item in the Data Access Layer.
*/
public void define(NewExternalData data, ExperimentPE experiment, SourceType sourceType);
@@ -86,18 +82,22 @@ public interface IDataBO extends IEntityBusinessObject
boolean newPresentInArchive) throws UserFailureException;
/**
- * Set the status for the loaded data set to the given new status value if the current status
- * equals an expected value.
+ * Updates sizes of given data sets (map key: data set code, map value: data set size).
*
- * @return true if the update is successful, false if the current status is different than
- * <code>oldStatus</code>.
+ * @throws UserFailureException if a data set does not exist or size couldn't be set.
+ */
+ public void updateSizes(Map<String, Long> sizeMap);
+
+ /**
+ * Set the status for the loaded data set to the given new status value if the current status equals an expected value.
+ *
+ * @return true if the update is successful, false if the current status is different than <code>oldStatus</code>.
*/
public boolean compareAndSetDataSetStatus(DataSetArchivingStatus oldStatus,
DataSetArchivingStatus newStatus, boolean newPresentInArchive);
/**
- * Adds chosen properties to given data set. If given property has been already defined, the
- * value is not updated.
+ * Adds chosen properties to given data set. If given property has been already defined, the value is not updated.
*/
public void addPropertiesToDataSet(String dataSetCode, List<NewProperty> properties);
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/datasetlister/DatasetLister.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/datasetlister/DatasetLister.java
index 78a093436753d3bc89c945d93984ad8c310d2f5e..d1142e75100aff60825333ab75f4f61e00c124fb 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/datasetlister/DatasetLister.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/datasetlister/DatasetLister.java
@@ -91,8 +91,8 @@ import ch.systemsx.cisd.openbis.generic.shared.translator.DataStoreTranslator;
* @author Tomasz Pylak
*/
@Friend(toClasses =
- { DatasetRecord.class, DatasetRelationRecord.class, DataStoreRecord.class,
- DatasetCodeWithShareIdRecord.class, IDatasetListingQuery.class })
+{ DatasetRecord.class, DatasetRelationRecord.class, DataStoreRecord.class,
+ DatasetCodeWithShareIdRecord.class, IDatasetListingQuery.class })
public class DatasetLister extends AbstractLister implements IDatasetLister
{
public static final EnumSet<DataSetFetchOption> SUPPORTED_DATASET_FETCH_OPTIONS = EnumSet.of(
@@ -490,6 +490,16 @@ public class DatasetLister extends AbstractLister implements IDatasetLister
dataStoreID), datasetFetchOptions));
}
+ @Override
+ public List<AbstractExternalData> listByDataStoreWithUnknownSize(long dataStoreID, EnumSet<DataSetFetchOption> datasetFetchOptions)
+ {
+ checkFetchOptions(datasetFetchOptions);
+ return orderByDate(enrichDatasets(
+ handleDegenerateRegistrationTimestamp(
+ query.getDatasetsByDataStoreIdWithUnknownSize(dataStoreID),
+ dataStoreID), datasetFetchOptions));
+ }
+
private Iterable<DatasetRecord> handleDegenerateRegistrationTimestamp(List<DatasetRecord> list,
long dataStoreID)
{
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/datasetlister/IDatasetLister.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/datasetlister/IDatasetLister.java
index 1a58ce649a9e33fb39baff09970a9b34e654808e..6ca0063c3e985b3454ea6399a459a0d80e890045 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/datasetlister/IDatasetLister.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/datasetlister/IDatasetLister.java
@@ -26,8 +26,8 @@ import java.util.Set;
import ch.systemsx.cisd.openbis.generic.server.business.bo.common.GenericEntityPropertyRecord;
import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.DataSetFetchOption;
import ch.systemsx.cisd.openbis.generic.shared.basic.TechId;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ArchiverDataSetCriteria;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ArchiverDataSetCriteria;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IDatasetLocationNode;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Sample;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.TrackingDataSetCriteria;
@@ -42,15 +42,15 @@ public interface IDatasetLister
{
/**
* @return datasets connected to the experiment with the specified id
- * @param showOnlyDirectlyConnected whether to return only directly connected datasets, or also
- * all descendants in dataset parent-child relationship hierarchy
+ * @param showOnlyDirectlyConnected whether to return only directly connected datasets, or also all descendants in dataset parent-child
+ * relationship hierarchy
*/
List<AbstractExternalData> listByExperimentTechId(TechId experimentId, boolean showOnlyDirectlyConnected);
/**
* @return datasets connected to the sample with the specified id
- * @param showOnlyDirectlyConnected whether to return only directly connected datasets, or also
- * all descendants in dataset parent-child relationship hierarchy
+ * @param showOnlyDirectlyConnected whether to return only directly connected datasets, or also all descendants in dataset parent-child
+ * relationship hierarchy
*/
List<AbstractExternalData> listBySampleTechId(TechId sampleId, boolean showOnlyDirectlyConnected);
@@ -67,21 +67,20 @@ public interface IDatasetLister
List<AbstractExternalData> listByMetaprojectId(Long metaprojectId);
/**
- * Returns a map with all parent data set IDs of specified data set IDs. The keys of the map are
- * IDs from the argument. A value of the map contains at least one element.
+ * Returns a map with all parent data set IDs of specified data set IDs. The keys of the map are IDs from the argument. A value of the map
+ * contains at least one element.
*/
Map<Long, Set<Long>> listParentIds(Collection<Long> dataSetIDs);
/**
- * Returns a map with all child data set IDs of specified data set IDs. The keys of the map are
- * IDs from the argument. A value of the map contains at least one element.
+ * Returns a map with all child data set IDs of specified data set IDs. The keys of the map are IDs from the argument. A value of the map contains
+ * at least one element.
*/
Map<Long, Set<Long>> listChildrenIds(Collection<Long> dataSetIDs);
/**
- * Returns a map with all data sets of specified samples. The sample arguments are the key into
- * the returned map. The returned data sets contains all derived data sets (children, grand
- * children, etc.).
+ * Returns a map with all data sets of specified samples. The sample arguments are the key into the returned map. The returned data sets contains
+ * all derived data sets (children, grand children, etc.).
*/
Map<Sample, List<AbstractExternalData>> listAllDataSetsFor(List<Sample> samples);
@@ -92,8 +91,7 @@ public interface IDatasetLister
/**
* @param datasetCodes Codes of datasets.
- * @param datasetFetchOptions The options of what datasets to fetch.
- * Lists all data sets with specified codes.
+ * @param datasetFetchOptions The options of what datasets to fetch. Lists all data sets with specified codes.
*/
List<AbstractExternalData> listByDatasetCode(Collection<String> datasetCodes,
EnumSet<DataSetFetchOption> datasetFetchOptions);
@@ -120,14 +118,18 @@ public interface IDatasetLister
EnumSet<DataSetFetchOption> datasetFetchOptions);
/**
- * Lists the oldest <var>limit</var> physical datasets younger than <var>youngerThan</var> of
- * the specified data store.
+ * Lists the oldest <var>limit</var> physical datasets younger than <var>youngerThan</var> of the specified data store.
*
* @param datasetFetchOptions The options of what datasets to fetch.
*/
public List<AbstractExternalData> listByDataStore(long dataStoreID, Date youngerThan, int limit,
EnumSet<DataSetFetchOption> datasetFetchOptions);
+ /**
+ * Lists physical datasets with unknown size of the specified data store.
+ */
+ List<AbstractExternalData> listByDataStoreWithUnknownSize(long dataStoreID, EnumSet<DataSetFetchOption> datasetFetchOptions);
+
/**
* Lists {@link DataSetShareId}s of all data sets (even those in trash) in specified data store.
*/
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/datasetlister/IDatasetListingQuery.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/datasetlister/IDatasetListingQuery.java
index 53973c37e481848a45154615bc357216e2e84b76..fdbde2d471a7f0b93e9e08dd7028753a55392ff1 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/datasetlister/IDatasetListingQuery.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/datasetlister/IDatasetListingQuery.java
@@ -39,17 +39,16 @@ import ch.systemsx.cisd.openbis.generic.server.dataaccess.db.LongSetMapper;
import ch.systemsx.cisd.openbis.generic.server.dataaccess.db.StringArrayMapper;
/**
- * A {@link TransactionQuery} interface for obtaining large sets of dataset-related entities from
- * the database.
+ * A {@link TransactionQuery} interface for obtaining large sets of dataset-related entities from the database.
* <p>
- * This interface is intended to be used only in this package. The <code>public</code> modifier is
- * needed for creating a dynamic proxy by the EOD SQL library.
+ * This interface is intended to be used only in this package. The <code>public</code> modifier is needed for creating a dynamic proxy by the EOD SQL
+ * library.
*
* @author Tomasz Pylak
*/
@Private
@Friend(toClasses =
- { DataStoreRecord.class })
+{ DataStoreRecord.class })
public interface IDatasetListingQuery extends BaseQuery, IPropertyListingQuery
{
public static final int FETCH_SIZE = 1000;
@@ -100,7 +99,7 @@ public interface IDatasetListingQuery extends BaseQuery, IPropertyListingQuery
* Returns the directly connected datasets for the given sample ids.
*/
@Select(sql = SELECT_ALL + " WHERE data.samp_id = any(?{1})", parameterBindings =
- { LongSetMapper.class }, fetchSize = FETCH_SIZE)
+ { LongSetMapper.class }, fetchSize = FETCH_SIZE)
public DataIterator<DatasetRecord> getDatasetsForSamples(LongSet sampleIds);
/**
@@ -110,8 +109,8 @@ public interface IDatasetListingQuery extends BaseQuery, IPropertyListingQuery
public DataIterator<DatasetRecord> getNewDataSets(long lastSeenDatasetId);
/**
- * Returns datasets that are newer than dataset with given id (<var>lastSeenDatasetId</var>) and
- * are directly connected with samples of sample type with given <var>sampleTypeId</var>.
+ * Returns datasets that are newer than dataset with given id (<var>lastSeenDatasetId</var>) and are directly connected with samples of sample
+ * type with given <var>sampleTypeId</var>.
*/
@Select(sql = SELECT_ALL
+ " WHERE data.id > ?{2} AND data.samp_id IN (SELECT id FROM samples s WHERE s.saty_id=?{1})", fetchSize = FETCH_SIZE)
@@ -119,8 +118,7 @@ public interface IDatasetListingQuery extends BaseQuery, IPropertyListingQuery
long lastSeenDatasetId);
/**
- * Returns datasets from store with given id that have status equal 'AVAILABLE' and were
- * modified before given date.
+ * Returns datasets from store with given id that have status equal 'AVAILABLE' and were modified before given date.
*/
@Select(sql = SELECT_ALL_EXTERNAL_DATAS
+ " WHERE data.dast_id = ?{1} AND external_data.status = 'AVAILABLE' "
@@ -129,8 +127,7 @@ public interface IDatasetListingQuery extends BaseQuery, IPropertyListingQuery
Date lastModificationDate, boolean presentInArchive);
/**
- * Like {@link #getAvailableExtDatasRegisteredBefore(long, Date, boolean)} with additional
- * condition for data set type id.
+ * Like {@link #getAvailableExtDatasRegisteredBefore(long, Date, boolean)} with additional condition for data set type id.
*/
@Select(sql = SELECT_ALL_EXTERNAL_DATAS
+ " WHERE data.dast_id = ?{1} AND external_data.status = 'AVAILABLE' "
@@ -150,15 +147,15 @@ public interface IDatasetListingQuery extends BaseQuery, IPropertyListingQuery
* Returns ids of datasets directly connected to samples with given ids.
*/
@Select(sql = "select id from data where data.samp_id = any(?{1})", parameterBindings =
- { LongSetMapper.class }, fetchSize = FETCH_SIZE)
+ { LongSetMapper.class }, fetchSize = FETCH_SIZE)
public DataIterator<Long> getDatasetIdsForSamples(LongSet sampleIds);
@Select(sql = "select * from data_set_relationships where data_id_child = any(?{1})", parameterBindings =
- { LongSetMapper.class }, fetchSize = FETCH_SIZE)
+ { LongSetMapper.class }, fetchSize = FETCH_SIZE)
public DataIterator<DatasetRelationRecord> listParentDataSetIds(LongSet ids);
@Select(sql = "select * from data_set_relationships where data_id_parent = any(?{1})", parameterBindings =
- { LongSetMapper.class }, fetchSize = FETCH_SIZE)
+ { LongSetMapper.class }, fetchSize = FETCH_SIZE)
public DataIterator<DatasetRelationRecord> listChildrenDataSetIds(LongSet ids);
/**
@@ -166,7 +163,7 @@ public interface IDatasetListingQuery extends BaseQuery, IPropertyListingQuery
*/
@Select(sql = SELECT_ALL
+ " WHERE data.id IN (SELECT data_id_child FROM data_set_relationships r WHERE r.data_id_parent = any(?{1}))", parameterBindings =
- { LongSetMapper.class }, fetchSize = FETCH_SIZE)
+ { LongSetMapper.class }, fetchSize = FETCH_SIZE)
public DataIterator<DatasetRecord> getChildDatasetsForParents(LongSet parentDatasetIds);
/**
@@ -219,11 +216,11 @@ public interface IDatasetListingQuery extends BaseQuery, IPropertyListingQuery
* Returns the datasets for the given <var>entityIds</var>.
*/
@Select(sql = SELECT_ALL + " where data.id = any(?{1})", parameterBindings =
- { LongSetMapper.class }, fetchSize = FETCH_SIZE)
+ { LongSetMapper.class }, fetchSize = FETCH_SIZE)
public DataIterator<DatasetRecord> getDatasets(LongSet entityIds);
@Select(sql = SELECT_ALL + " where data.code = any(?{1})", parameterBindings =
- { StringArrayMapper.class }, fetchSize = FETCH_SIZE)
+ { StringArrayMapper.class }, fetchSize = FETCH_SIZE)
public DataIterator<DatasetRecord> getDatasets(String[] datasetCodes);
@Select(sql = SELECT_ALL_EXTERNAL_DATAS
@@ -246,6 +243,11 @@ public interface IDatasetListingQuery extends BaseQuery, IPropertyListingQuery
+ " order by registration_timestamp", fetchSize = FETCH_SIZE)
public List<DatasetRecord> getDatasetsByDataStoreId(long dataStoreId, Date at);
+ @Select(sql = SELECT_ALL_EXTERNAL_DATAS
+ + " where data.dast_id = ?{1} and is_placeholder = false and size is null"
+ + " order by registration_timestamp", fetchSize = FETCH_SIZE)
+ public List<DatasetRecord> getDatasetsByDataStoreIdWithUnknownSize(long dataStoreID);
+
// NOTE: we list ALL data sets (even those in trash) using data_all table here
@Select(sql = "SELECT code, share_id FROM data_all LEFT OUTER JOIN external_data "
+ "ON data_all.id = external_data.data_id WHERE data_all.dast_id = ?{1}", fetchSize = FETCH_SIZE)
@@ -256,11 +258,11 @@ public interface IDatasetListingQuery extends BaseQuery, IPropertyListingQuery
* Returns the children dataset ids of the specified datasets.
*/
@Select(sql = "select data_id_child from data_set_relationships where data_id_parent = any(?{1})", parameterBindings =
- { LongSetMapper.class }, fetchSize = FETCH_SIZE)
+ { LongSetMapper.class }, fetchSize = FETCH_SIZE)
public DataIterator<Long> getDatasetChildrenIds(LongSet sampleId);
@Select(sql = "select id from data where ctnr_id = any(?{1})", parameterBindings =
- { LongSetMapper.class }, fetchSize = FETCH_SIZE)
+ { LongSetMapper.class }, fetchSize = FETCH_SIZE)
public DataIterator<Long> getContainedDataSetIds(LongSet containerIDs);
@Select(sql = "select code from data where ctnr_id = (select id from data where code = ?{1})", fetchSize = FETCH_SIZE)
@@ -280,13 +282,12 @@ public interface IDatasetListingQuery extends BaseQuery, IPropertyListingQuery
+ " JOIN data_set_type_property_types etpt ON pr.dstpt_id=etpt.id"
+ " LEFT OUTER JOIN scripts sc ON etpt.script_id = sc.id"
+ " WHERE pr.value is not null AND pr.ds_id = any(?{1})", parameterBindings =
- { LongSetMapper.class }, fetchSize = FETCH_SIZE)
+ { LongSetMapper.class }, fetchSize = FETCH_SIZE)
public DataIterator<GenericEntityPropertyRecord> getEntityPropertyGenericValues(
LongSet entityIds);
/**
- * Returns property values for specified type code of all datasets specified by
- * <var>entityIds</var>.
+ * Returns property values for specified type code of all datasets specified by <var>entityIds</var>.
*
* @param entityIds The set of sample ids to get the property values for.
* @param propertyTypeCode type code f properties we want to fetch
@@ -296,13 +297,12 @@ public interface IDatasetListingQuery extends BaseQuery, IPropertyListingQuery
+ " JOIN data_set_type_property_types etpt ON pr.dstpt_id=etpt.id"
+ " JOIN property_types pt ON etpt.prty_id=pt.id"
+ " WHERE pr.value is not null AND pr.ds_id = any(?{1}) AND pt.code = ?{2}", parameterBindings =
- { LongSetMapper.class, TypeMapper.class }, fetchSize = FETCH_SIZE)
+ { LongSetMapper.class, TypeMapper.class }, fetchSize = FETCH_SIZE)
public DataIterator<GenericEntityPropertyRecord> getEntityPropertyGenericValues(
LongSet entityIds, String propertyTypeCode);
/**
- * Returns all controlled vocabulary property values of all datasets specified by
- * <var>entityIds</var>.
+ * Returns all controlled vocabulary property values of all datasets specified by <var>entityIds</var>.
*
* @param entityIds The set of sample ids to get the property values for.
*/
@@ -311,7 +311,7 @@ public interface IDatasetListingQuery extends BaseQuery, IPropertyListingQuery
+ " JOIN data_set_type_property_types etpt ON pr.dstpt_id=etpt.id"
+ " JOIN controlled_vocabulary_terms cvte ON pr.cvte_id=cvte.id"
+ " WHERE pr.cvte_id is not null AND pr.ds_id = any(?{1})", parameterBindings =
- { LongSetMapper.class }, fetchSize = FETCH_SIZE)
+ { LongSetMapper.class }, fetchSize = FETCH_SIZE)
public DataIterator<VocabularyTermRecord> getEntityPropertyVocabularyTermValues(
LongSet entityIds);
@@ -325,7 +325,7 @@ public interface IDatasetListingQuery extends BaseQuery, IPropertyListingQuery
+ " JOIN data_set_type_property_types etpt ON pr.dstpt_id=etpt.id"
+ " JOIN materials m ON pr.mate_prop_id=m.id "
+ " WHERE pr.mate_prop_id is not null AND pr.ds_id = any(?{1})", parameterBindings =
- { LongSetMapper.class }, fetchSize = FETCH_SIZE)
+ { LongSetMapper.class }, fetchSize = FETCH_SIZE)
public DataIterator<MaterialEntityPropertyRecord> getEntityPropertyMaterialValues(
LongSet entityIds);
@@ -342,7 +342,7 @@ public interface IDatasetListingQuery extends BaseQuery, IPropertyListingQuery
+ " m.private as is_private, m.creation_date as creation_date, ma.data_id as entity_id "
+ " from metaprojects m, metaproject_assignments ma, persons p "
+ " where ma.data_id = any(?{1}) and m.owner = ?{2} and m.id = ma.mepr_id and m.owner = p.id", parameterBindings =
- { LongSetMapper.class }, fetchSize = FETCH_SIZE)
+ { LongSetMapper.class }, fetchSize = FETCH_SIZE)
public List<MetaProjectWithEntityId> getMetaprojects(LongSet entityIds, Long userId);
}
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/IDataDAO.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/IDataDAO.java
index cce2e548b360364aeb8f44531d50ca73adedea21..beaf94c402363dc3d3efa5ef10089e489860c392 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/IDataDAO.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/IDataDAO.java
@@ -18,6 +18,7 @@ package ch.systemsx.cisd.openbis.generic.server.dataaccess;
import java.util.Collection;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import org.springframework.dao.DataAccessException;
@@ -48,8 +49,7 @@ public interface IDataDAO extends IGenericDAO<DataPE>
public boolean hasDataSet(final SamplePE sample) throws DataAccessException;
/**
- * List the {@link DataPE} related to given <var>entities</var> of specified
- * <var>entityKind</var>.
+ * List the {@link DataPE} related to given <var>entities</var> of specified <var>entityKind</var>.
*
* @returns list of {@link DataPE}s that are related to given {@link IEntityInformationHolder}.
*/
@@ -64,8 +64,7 @@ public interface IDataDAO extends IGenericDAO<DataPE>
public List<DataPE> listDataSets(final SamplePE sample) throws DataAccessException;
/**
- * List the {@link DataPE} for given <var>sample</var>. The datasets are fetched without the
- * additional relationships or properties.
+ * List the {@link DataPE} for given <var>sample</var>. The datasets are fetched without the additional relationships or properties.
*
* @returns list of {@link DataPE}s that are related to given {@link SamplePE}.
*/
@@ -118,6 +117,11 @@ public interface IDataDAO extends IGenericDAO<DataPE>
public void updateDataSetStatuses(List<String> dataSetCodes, DataSetArchivingStatus status,
boolean newPresentInArchive);
+ /**
+ * Updates sizes of given data sets (map key: data set code, map value: data set size).
+ */
+ public void updateSizes(Map<String, Long> sizeMap);
+
/**
* Persists the specified data set.
*/
@@ -159,8 +163,7 @@ public interface IDataDAO extends IGenericDAO<DataPE>
public List<TechId> listContainedDataSets(final Collection<TechId> containerIds);
/**
- * Returs ids of contained data sets, and of datasets contained in those datasets etc. Also
- * includes the input ids.
+ * Returs ids of contained data sets, and of datasets contained in those datasets etc. Also includes the input ids.
*/
public List<TechId> listContainedDataSetsRecursively(final Collection<TechId> containersIds);
@@ -171,17 +174,14 @@ public interface IDataDAO extends IGenericDAO<DataPE>
throws DataAccessException;
/**
- * Confirms a storage for the specified data set. It confirms the storage even if the data set
- * is in the trash.
+ * Confirms a storage for the specified data set. It confirms the storage even if the data set is in the trash.
*
- * @return Returns true if the data sets exists and the storage confirmed value has been
- * changed.
+ * @return Returns true if the data sets exists and the storage confirmed value has been changed.
*/
boolean confirmStorage(String dataSetCode);
/**
- * Checks whether a data set with the specified code exists. It takes into consideration also
- * data sets that are in the trash.
+ * Checks whether a data set with the specified code exists. It takes into consideration also data sets that are in the trash.
*
* @return Returns true if the data set exists.
*/
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/DataDAO.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/DataDAO.java
index b646075da824efb101f8c3193c64c511c916c40f..6954a7c2c1e3a5edfba60edb60b38100a68978cc 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/DataDAO.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/DataDAO.java
@@ -24,6 +24,7 @@ import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
@@ -549,6 +550,35 @@ final class DataDAO extends AbstractGenericEntityWithPropertiesDAO<DataPE> imple
}
}
+ @Override
+ public void updateSizes(final Map<String, Long> sizeMap)
+ {
+ final HibernateTemplate hibernateTemplate = getHibernateTemplate();
+ hibernateTemplate.execute(new HibernateCallback()
+ {
+ @Override
+ public final Object doInHibernate(final Session session)
+ throws HibernateException, SQLException
+ {
+ for (Map.Entry<String, Long> sizeEntry : sizeMap.entrySet())
+ {
+ // data sets consisting out of empty folders have a size of 0,
+ // but we want the size of a data set to be strictly positive
+ long positiveSize = Math.max(1, sizeEntry.getValue());
+
+ session.createQuery(
+ "UPDATE " + EXTERNAL_DATA_TABLE_NAME
+ + " SET size = :size WHERE code = :code")
+ .setParameter("size", positiveSize)
+ .setParameter("code", sizeEntry.getKey())
+ .executeUpdate();
+ }
+ return null;
+ }
+ });
+ hibernateTemplate.flush();
+ }
+
@Override
public void updateDataSetStatuses(final List<String> dataSetCodes,
final DataSetArchivingStatus status, final boolean presentInArchive)
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/IServiceForDataStoreServer.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/IServiceForDataStoreServer.java
index 5e4febeb2b6e6d3cb697ce2f5fc947a7a82ee274..95461422b5eaac1f10abad65cb87ee5f5af4997a 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/IServiceForDataStoreServer.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/IServiceForDataStoreServer.java
@@ -433,6 +433,19 @@ public interface IServiceForDataStoreServer extends IServer, ISessionProvider
public List<SimpleDataSetInformationDTO> listOldestPhysicalDataSets(final String sessionToken,
String dataStore, Date youngerThan, int limit) throws UserFailureException;
+ /**
+ * Returns informations about physical data sets with unknown size that belong to the specified data store server.
+ */
+ @Transactional(readOnly = true)
+ public List<SimpleDataSetInformationDTO> listPhysicalDataSetsWithUnknownSize(String sessionToken, String dataStoreCode);
+
+ /**
+ * Updates sizes of the specified physical data sets (map key: data set code, map value: data set size).
+ */
+ @Transactional
+ @DatabaseUpdateModification(value = ObjectKind.DATA_SET)
+ public void updatePhysicalDataSetsSize(String sessionToken, Map<String, Long> sizeMap);
+
/**
* List data sets deleted after specified date.
*/