Skip to content
Snippets Groups Projects
Commit 498cc5da authored by felmer's avatar felmer
Browse files

LMS-2854 Utility methods established which read data in chunks for criteria... 

LMS-2854 Utility methods established which read data in chunks for criteria with IN restrictions. Using these methods in many places where Restrictions.in() has been used. Note: Not all places could be refactored using the new utility methods.

SVN: 24750
parent d5dfe5d7
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 233 additions and 103 deletions
openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/DAOUtils.java
+ 54
− 0
View file @ 498cc5da
......@@ -16,8 +16,15 @@
package ch.systemsx.cisd.openbis.generic.server.dataaccess.db;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.hibernate.Criteria;
import org.hibernate.criterion.DetachedCriteria;
import org.hibernate.criterion.Projections;
import org.hibernate.criterion.Restrictions;
import org.springframework.orm.hibernate3.HibernateTemplate;
/**
* Utility routines for DAOs.
......@@ -49,4 +56,51 @@ final class DAOUtils
return count;
}
private static final int BATCH_SIZE = 1000;
/**
* Lists entities of type <code>T</code> by specified collection of identifiers. A Hibernate
* Criteria query for specified class is created with an
* {@link Restrictions#in(String, Collection)} restriction on the collection of identifiers.
* This is a helper method which reads the data in chunks because otherwise an exception will be
* thrown by PosgreSQL if more than 2<sup>15</sup>-1 identfiers.
*/
static <T> List<T> listByCollection(HibernateTemplate hibernateTemplate,
final Class<?> entityClass, String columnName, Collection<?> identifiers)
{
return listByCollection(hibernateTemplate, new IDetachedCriteriaFactory()
{
public DetachedCriteria createCriteria()
{
return DetachedCriteria.forClass(entityClass);
}
}, columnName, identifiers);
}
/**
* Lists entities of type <code>T</code> by specified collection of identifiers. A Hibernate
* Criteria query is created by the specified factory and an
* {@link Restrictions#in(String, Collection)} restriction on the collection of identifiers is
* added. This is a helper method which reads the data in chunks because otherwise an exception
* will be thrown by PosgreSQL if more than 2<sup>15</sup>-1 identfiers.
*/
@SuppressWarnings("unchecked")
static <T> List<T> listByCollection(HibernateTemplate hibernateTemplate,
IDetachedCriteriaFactory factory, String columnName, Collection<?> identifiers)
{
List<?> parameters = new ArrayList<Object>(identifiers);
List<T> result = new ArrayList<T>();
for (int i = 0, n = parameters.size(); i < n; i += BATCH_SIZE)
{
DetachedCriteria criteria = factory.createCriteria();
List<?> subList = parameters.subList(i, Math.min(n, i + BATCH_SIZE));
if (subList.isEmpty() == false)
{
criteria.add(Restrictions.in(columnName, subList));
result.addAll(hibernateTemplate.findByCriteria(criteria));
}
}
return result;
}
}
openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/DataDAO.java
+ 77
− 44
View file @ 498cc5da
......@@ -263,13 +263,18 @@ final class DataDAO extends AbstractGenericEntityWithPropertiesDAO<DataPE> imple
return Collections.emptyList();
}
final Criterion codeIn = Restrictions.in("code", dataSetCodes);
final DetachedCriteria criteria = DetachedCriteria.forClass(DeletedDataPE.class);
criteria.add(codeIn);
criteria.setFetchMode("dataStore", FetchMode.SELECT);
criteria.setResultTransformer(DetachedCriteria.DISTINCT_ROOT_ENTITY);
final List<DeletedDataPE> list = cast(getHibernateTemplate().findByCriteria(criteria));
final List<DeletedDataPE> list =
DAOUtils.listByCollection(getHibernateTemplate(), new IDetachedCriteriaFactory()
{
public DetachedCriteria createCriteria()
{
final DetachedCriteria criteria =
DetachedCriteria.forClass(DeletedDataPE.class);
criteria.setFetchMode("dataStore", FetchMode.SELECT);
criteria.setResultTransformer(DetachedCriteria.DISTINCT_ROOT_ENTITY);
return criteria;
}
}, "code", dataSetCodes);
if (operationLog.isDebugEnabled())
{
......@@ -325,32 +330,39 @@ final class DataDAO extends AbstractGenericEntityWithPropertiesDAO<DataPE> imple
}
private List<DataPE> primFindFullDataSetsByCode(String identifierColumn,
Collection<?> identifiers, boolean withPropertyTypes, boolean lockForUpdate)
Collection<?> identifiers, final boolean withPropertyTypes, final boolean lockForUpdate)
{
if (identifiers.size() == 0)
{
return Collections.emptyList();
}
final Criterion codeIn = Restrictions.in(identifierColumn, identifiers);
final DetachedCriteria criteria = DetachedCriteria.forClass(ENTITY_CLASS);
criteria.add(codeIn);
criteria.setFetchMode("dataSetType", FetchMode.SELECT);
criteria.setFetchMode("dataStore", FetchMode.SELECT);
criteria.setFetchMode("experimentInternal", FetchMode.SELECT);
criteria.setFetchMode("sampleInternal", FetchMode.SELECT);
criteria.setFetchMode("fileFormat", FetchMode.SELECT);
if (withPropertyTypes)
{
criteria.setFetchMode("dataSetType.dataSetTypePropertyTypesInternal", FetchMode.JOIN);
}
criteria.setResultTransformer(DetachedCriteria.DISTINCT_ROOT_ENTITY);
if (lockForUpdate)
{
criteria.setLockMode(LockMode.UPGRADE);
}
final List<DataPE> list = cast(getHibernateTemplate().findByCriteria(criteria));
final List<DataPE> list =
DAOUtils.listByCollection(getHibernateTemplate(), new IDetachedCriteriaFactory()
{
public DetachedCriteria createCriteria()
{
final DetachedCriteria criteria =
DetachedCriteria.forClass(ENTITY_CLASS);
criteria.setFetchMode("dataSetType", FetchMode.SELECT);
criteria.setFetchMode("dataStore", FetchMode.SELECT);
criteria.setFetchMode("experimentInternal", FetchMode.SELECT);
criteria.setFetchMode("sampleInternal", FetchMode.SELECT);
criteria.setFetchMode("fileFormat", FetchMode.SELECT);
if (withPropertyTypes)
{
criteria.setFetchMode(
"dataSetType.dataSetTypePropertyTypesInternal",
FetchMode.JOIN);
}
criteria.setResultTransformer(DetachedCriteria.DISTINCT_ROOT_ENTITY);
if (lockForUpdate)
{
criteria.setLockMode(LockMode.UPGRADE);
}
return criteria;
}
}, identifierColumn, identifiers);
if (operationLog.isDebugEnabled())
{
......@@ -964,9 +976,8 @@ final class DataDAO extends AbstractGenericEntityWithPropertiesDAO<DataPE> imple
{
return new ArrayList<DataPE>();
}
final DetachedCriteria criteria = DetachedCriteria.forClass(DataPE.class);
criteria.add(Restrictions.in("code", values));
final List<DataPE> list = cast(getHibernateTemplate().findByCriteria(criteria));
final List<DataPE> list =
DAOUtils.listByCollection(getHibernateTemplate(), DataPE.class, "code", values);
if (operationLog.isDebugEnabled())
{
operationLog.debug(String.format("%d data set(s) have been found.", list.size()));
......@@ -1007,11 +1018,19 @@ final class DataDAO extends AbstractGenericEntityWithPropertiesDAO<DataPE> imple
public List<TechId> listDataSetIdsBySampleIds(final Collection<TechId> samples)
{
final DetachedCriteria criteria = DetachedCriteria.forClass(DataPE.class);
final List<Long> longIds = TechId.asLongs(samples);
criteria.setProjection(Projections.id());
criteria.add(Restrictions.in("sampleInternal.id", longIds));
final List<Long> results = cast(getHibernateTemplate().findByCriteria(criteria));
final List<Long> results =
DAOUtils.listByCollection(getHibernateTemplate(), new IDetachedCriteriaFactory()
{
public DetachedCriteria createCriteria()
{
final DetachedCriteria criteria =
DetachedCriteria.forClass(DataPE.class);
criteria.setProjection(Projections.id());
return criteria;
}
}, "sampleInternal.id", longIds);
if (operationLog.isDebugEnabled())
{
operationLog
......@@ -1022,11 +1041,18 @@ final class DataDAO extends AbstractGenericEntityWithPropertiesDAO<DataPE> imple
public List<TechId> listDataSetIdsByExperimentIds(final Collection<TechId> experiments)
{
final DetachedCriteria criteria = DetachedCriteria.forClass(DataPE.class);
final List<Long> longIds = TechId.asLongs(experiments);
criteria.setProjection(Projections.id());
criteria.add(Restrictions.in("experimentInternal.id", longIds));
final List<Long> results = cast(getHibernateTemplate().findByCriteria(criteria));
final List<Long> results =
DAOUtils.listByCollection(getHibernateTemplate(), new IDetachedCriteriaFactory()
{
public DetachedCriteria createCriteria()
{
final DetachedCriteria criteria =
DetachedCriteria.forClass(DataPE.class);
criteria.setProjection(Projections.id());
return criteria;
}
}, "experimentInternal.id", longIds);
if (operationLog.isDebugEnabled())
{
operationLog.info(String.format("found %s data sets for given experiments",
......@@ -1052,12 +1078,19 @@ final class DataDAO extends AbstractGenericEntityWithPropertiesDAO<DataPE> imple
public void execute(List<Long> batchIds)
{
final DetachedCriteria criteria = DetachedCriteria.forClass(DataPE.class);
criteria.setProjection(Projections.id());
criteria.add(Restrictions.in("containerInternal.id", batchIds));
final List<Long> batchResults =
cast(getHibernateTemplate().findByCriteria(criteria));
totalResults.addAll(batchResults);
List<Long> result =
DAOUtils.listByCollection(getHibernateTemplate(),
new IDetachedCriteriaFactory()
{
public DetachedCriteria createCriteria()
{
final DetachedCriteria criteria =
DetachedCriteria.forClass(DataPE.class);
criteria.setProjection(Projections.id());
return criteria;
}
}, "containerInternal.id", batchIds);
totalResults.addAll(result);
}
public List<Long> getAllEntities()
......
openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/DeletionDAO.java
+ 34
− 37
View file @ 498cc5da
......@@ -17,12 +17,10 @@
package ch.systemsx.cisd.openbis.generic.server.dataaccess.db;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.apache.log4j.Logger;
import org.hibernate.Criteria;
import org.hibernate.HibernateException;
import org.hibernate.SQLQuery;
import org.hibernate.Session;
......@@ -231,34 +229,49 @@ final class DeletionDAO extends AbstractGenericEntityDAO<DeletionPE> implements
{
return Collections.emptyList();
}
final DetachedCriteria criteria =
DetachedCriteria.forClass(EntityKind.DATA_SET.getDeletedEntityClass());
final List<Long> longIds = TechId.asLongs(deletionIds);
criteria.setProjection(Projections.property("code"));
criteria.add(Restrictions.in(DELETION_ID, longIds));
final List<String> results = cast(getHibernateTemplate().findByCriteria(criteria));
final List<String> results =
DAOUtils.listByCollection(getHibernateTemplate(), new IDetachedCriteriaFactory()
{
public DetachedCriteria createCriteria()
{
final DetachedCriteria criteria =
DetachedCriteria.forClass(EntityKind.DATA_SET
.getDeletedEntityClass());
criteria.setProjection(Projections.property("code"));
return criteria;
}
}, DELETION_ID, longIds);
operationLog.info(String.format("found %s trashed %s(s)", results.size(),
EntityKind.DATA_SET.name()));
return results;
}
private List<TechId> findTrashedEntityIds(final List<TechId> deletionIds,
final EntityKind entityKind, Criterion... additionalCriteria)
final EntityKind entityKind, final Criterion... additionalCriteria)
{
if (deletionIds.isEmpty())
{
return Collections.emptyList();
}
final DetachedCriteria criteria =
DetachedCriteria.forClass(entityKind.getDeletedEntityClass());
final List<Long> longIds = TechId.asLongs(deletionIds);
criteria.setProjection(Projections.id());
criteria.add(Restrictions.in(DELETION_ID, longIds));
for (Criterion criterion : additionalCriteria)
{
criteria.add(criterion);
}
final List<Long> results = cast(getHibernateTemplate().findByCriteria(criteria));
final List<Long> results =
DAOUtils.listByCollection(getHibernateTemplate(), new IDetachedCriteriaFactory()
{
public DetachedCriteria createCriteria()
{
final DetachedCriteria criteria =
DetachedCriteria.forClass(entityKind.getDeletedEntityClass());
criteria.setProjection(Projections.id());
for (Criterion criterion : additionalCriteria)
{
criteria.add(criterion);
}
return criteria;
}
}, DELETION_ID, longIds);
operationLog
.info(String.format("found %s trashed %s(s)", results.size(), entityKind.name()));
return transformNumbers2TechIdList(results);
......@@ -402,11 +415,8 @@ final class DeletionDAO extends AbstractGenericEntityDAO<DeletionPE> implements
{
return Collections.emptyList();
}
final Criteria criteria = getSession().createCriteria(DeletionPE.class);
criteria.add(Restrictions.in("id", ids));
@SuppressWarnings("unchecked")
List<DeletionPE> result = criteria.list();
List<DeletionPE> result =
DAOUtils.listByCollection(getHibernateTemplate(), DeletionPE.class, "id", ids);
if (operationLog.isDebugEnabled())
{
operationLog.debug(String.format("%s deletions has been found", result.size()));
......@@ -451,21 +461,8 @@ final class DeletionDAO extends AbstractGenericEntityDAO<DeletionPE> implements
}
HibernateTemplate hibernateTemplate = getHibernateTemplate();
List<Long> ids = TechId.asLongs(entityIds);
int chunkSize = 30000;
List<IDeletablePE> result = new ArrayList<IDeletablePE>();
for (int i = 0, n = ids.size(); i < n; i += chunkSize)
{
DetachedCriteria criteria =
DetachedCriteria.forClass(entityKind.getDeletedEntityClass());
List<Long> subList = ids.subList(i, Math.min(n, i + chunkSize));
if (subList.isEmpty() == false)
{
criteria.add(Restrictions.in(ID, subList));
List<IDeletablePE> list = cast(hibernateTemplate.findByCriteria(criteria));
result.addAll(list);
}
}
return result;
return DAOUtils.listByCollection(hibernateTemplate, entityKind.getDeletedEntityClass(), ID,
ids);
}
public List<TechId> listDeletedEntitiesForType(EntityKind entityKind, TechId entityTypeId)
......
openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/ExperimentDAO.java
+ 13
− 8
View file @ 498cc5da
......@@ -142,11 +142,17 @@ public class ExperimentDAO extends AbstractGenericEntityWithPropertiesDAO<Experi
{
return new ArrayList<ExperimentPE>();
}
DetachedCriteria criteria = DetachedCriteria.forClass(getEntityClass());
criteria.add(Restrictions.in("id", experimentIDs));
criteria.setFetchMode("experimentProperties", FetchMode.JOIN);
criteria.setResultTransformer(Criteria.DISTINCT_ROOT_ENTITY);
final List<ExperimentPE> list = cast(getHibernateTemplate().findByCriteria(criteria));
final List<ExperimentPE> list =
DAOUtils.listByCollection(getHibernateTemplate(), new IDetachedCriteriaFactory()
{
public DetachedCriteria createCriteria()
{
DetachedCriteria criteria = DetachedCriteria.forClass(getEntityClass());
criteria.setFetchMode("experimentProperties", FetchMode.JOIN);
criteria.setResultTransformer(Criteria.DISTINCT_ROOT_ENTITY);
return criteria;
}
}, "id", experimentIDs);
if (operationLog.isDebugEnabled())
{
operationLog.debug(String.format("%d experiments have been found for specified IDs.",
......@@ -279,9 +285,8 @@ public class ExperimentDAO extends AbstractGenericEntityWithPropertiesDAO<Experi
{
return new ArrayList<ExperimentPE>();
}
final DetachedCriteria criteria = DetachedCriteria.forClass(ExperimentPE.class);
criteria.add(Restrictions.in(idName, ids));
final List<ExperimentPE> list = cast(getHibernateTemplate().findByCriteria(criteria));
final List<ExperimentPE> list =
DAOUtils.listByCollection(getHibernateTemplate(), ExperimentPE.class, idName, ids);
if (operationLog.isDebugEnabled())
{
operationLog.debug(String.format("%d experiment(s) have been found.", list.size()));
......
openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/IDetachedCriteriaFactory.java 0 → 100644
+ 29
− 0
View file @ 498cc5da
/*
* Copyright 2012 ETH Zuerich, CISD
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ch.systemsx.cisd.openbis.generic.server.dataaccess.db;
import org.hibernate.criterion.DetachedCriteria;
/**
* Factory of {@link DetachedCriteria} instances.
*
* @author Franz-Josef Elmer
*/
public interface IDetachedCriteriaFactory
{
public DetachedCriteria createCriteria();
}
openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/MaterialDAO.java
+ 2
− 3
View file @ 498cc5da
......@@ -199,9 +199,8 @@ public class MaterialDAO extends AbstractGenericEntityWithPropertiesDAO<Material
{
return Collections.emptyList();
}
Criteria criteria = getSession().createCriteria(ENTITY_CLASS);
criteria.add(Restrictions.in("id", ids));
final List<MaterialPE> list = cast(criteria.list());
final List<MaterialPE> list =
DAOUtils.listByCollection(getHibernateTemplate(), ENTITY_CLASS, "id", ids);
if (operationLog.isDebugEnabled())
{
operationLog.debug(String.format("%d materials have been found for ids: %s.",
......
openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/SampleDAO.java
+ 24
− 11
View file @ 498cc5da
......@@ -380,9 +380,8 @@ public class SampleDAO extends AbstractGenericEntityWithPropertiesDAO<SamplePE>
{
return new ArrayList<SamplePE>();
}
final DetachedCriteria criteria = DetachedCriteria.forClass(SamplePE.class);
criteria.add(Restrictions.in(idName, values));
final List<SamplePE> list = cast(getHibernateTemplate().findByCriteria(criteria));
final List<SamplePE> list =
DAOUtils.listByCollection(getHibernateTemplate(), SamplePE.class, idName, values);
if (operationLog.isDebugEnabled())
{
operationLog.debug(String.format("%d sample(s) have been found.", list.size()));
......@@ -479,11 +478,18 @@ public class SampleDAO extends AbstractGenericEntityWithPropertiesDAO<SamplePE>
public List<TechId> listSampleIdsByContainerIds(final Collection<TechId> containers)
{
final DetachedCriteria criteria = DetachedCriteria.forClass(SamplePE.class);
final List<Long> longIds = TechId.asLongs(containers);
criteria.setProjection(Projections.id());
criteria.add(Restrictions.in("container.id", longIds));
final List<Long> results = cast(getHibernateTemplate().findByCriteria(criteria));
final List<Long> results =
DAOUtils.listByCollection(getHibernateTemplate(), new IDetachedCriteriaFactory()
{
public DetachedCriteria createCriteria()
{
final DetachedCriteria criteria =
DetachedCriteria.forClass(SamplePE.class);
criteria.setProjection(Projections.id());
return criteria;
}
}, "container.id", longIds);
if (operationLog.isDebugEnabled())
{
operationLog.info(String.format("found %s sample components for given containers",
......@@ -494,11 +500,18 @@ public class SampleDAO extends AbstractGenericEntityWithPropertiesDAO<SamplePE>
public List<TechId> listSampleIdsByExperimentIds(final Collection<TechId> experiments)
{
final DetachedCriteria criteria = DetachedCriteria.forClass(SamplePE.class);
final List<Long> longIds = TechId.asLongs(experiments);
criteria.setProjection(Projections.id());
criteria.add(Restrictions.in("experimentInternal.id", longIds));
final List<Long> results = cast(getHibernateTemplate().findByCriteria(criteria));
final List<Long> results =
DAOUtils.listByCollection(getHibernateTemplate(), new IDetachedCriteriaFactory()
{
public DetachedCriteria createCriteria()
{
final DetachedCriteria criteria =
DetachedCriteria.forClass(SamplePE.class);
criteria.setProjection(Projections.id());
return criteria;
}
}, "experimentInternal.id", longIds);
if (operationLog.isDebugEnabled())
{
operationLog.info(String.format("found %s samples for given experiments",
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment