diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/DAOUtils.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/DAOUtils.java index 190fa365b658cf6bf4b59c2c7d7a32f6bb816f2a..5c0128cb78dabe68f0dc10156b1fec189b40a973 100644 --- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/DAOUtils.java +++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/DAOUtils.java @@ -16,8 +16,15 @@ package ch.systemsx.cisd.openbis.generic.server.dataaccess.db; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + import org.hibernate.Criteria; +import org.hibernate.criterion.DetachedCriteria; import org.hibernate.criterion.Projections; +import org.hibernate.criterion.Restrictions; +import org.springframework.orm.hibernate3.HibernateTemplate; /** * Utility routines for DAOs. @@ -49,4 +56,51 @@ final class DAOUtils return count; } + private static final int BATCH_SIZE = 1000; + + /** + * Lists entities of type <code>T</code> by specified collection of identifiers. A Hibernate + * Criteria query for specified class is created with an + * {@link Restrictions#in(String, Collection)} restriction on the collection of identifiers. + * This is a helper method which reads the data in chunks because otherwise an exception will be + * thrown by PosgreSQL if more than 2<sup>15</sup>-1 identfiers. + */ + static <T> List<T> listByCollection(HibernateTemplate hibernateTemplate, + final Class<?> entityClass, String columnName, Collection<?> identifiers) + { + return listByCollection(hibernateTemplate, new IDetachedCriteriaFactory() + { + public DetachedCriteria createCriteria() + { + return DetachedCriteria.forClass(entityClass); + } + }, columnName, identifiers); + } + + /** + * Lists entities of type <code>T</code> by specified collection of identifiers. A Hibernate + * Criteria query is created by the specified factory and an + * {@link Restrictions#in(String, Collection)} restriction on the collection of identifiers is + * added. This is a helper method which reads the data in chunks because otherwise an exception + * will be thrown by PosgreSQL if more than 2<sup>15</sup>-1 identfiers. + */ + @SuppressWarnings("unchecked") + static <T> List<T> listByCollection(HibernateTemplate hibernateTemplate, + IDetachedCriteriaFactory factory, String columnName, Collection<?> identifiers) + { + List<?> parameters = new ArrayList<Object>(identifiers); + List<T> result = new ArrayList<T>(); + for (int i = 0, n = parameters.size(); i < n; i += BATCH_SIZE) + { + DetachedCriteria criteria = factory.createCriteria(); + List<?> subList = parameters.subList(i, Math.min(n, i + BATCH_SIZE)); + if (subList.isEmpty() == false) + { + criteria.add(Restrictions.in(columnName, subList)); + result.addAll(hibernateTemplate.findByCriteria(criteria)); + } + } + return result; + } + } diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/DataDAO.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/DataDAO.java index 71dc6e8273285241381ae639868e5f1bc8a2f14e..e50182a71412f461ffb28457a09d04724a2964db 100644 --- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/DataDAO.java +++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/DataDAO.java @@ -263,13 +263,18 @@ final class DataDAO extends AbstractGenericEntityWithPropertiesDAO<DataPE> imple return Collections.emptyList(); } - final Criterion codeIn = Restrictions.in("code", dataSetCodes); - - final DetachedCriteria criteria = DetachedCriteria.forClass(DeletedDataPE.class); - criteria.add(codeIn); - criteria.setFetchMode("dataStore", FetchMode.SELECT); - criteria.setResultTransformer(DetachedCriteria.DISTINCT_ROOT_ENTITY); - final List<DeletedDataPE> list = cast(getHibernateTemplate().findByCriteria(criteria)); + final List<DeletedDataPE> list = + DAOUtils.listByCollection(getHibernateTemplate(), new IDetachedCriteriaFactory() + { + public DetachedCriteria createCriteria() + { + final DetachedCriteria criteria = + DetachedCriteria.forClass(DeletedDataPE.class); + criteria.setFetchMode("dataStore", FetchMode.SELECT); + criteria.setResultTransformer(DetachedCriteria.DISTINCT_ROOT_ENTITY); + return criteria; + } + }, "code", dataSetCodes); if (operationLog.isDebugEnabled()) { @@ -325,32 +330,39 @@ final class DataDAO extends AbstractGenericEntityWithPropertiesDAO<DataPE> imple } private List<DataPE> primFindFullDataSetsByCode(String identifierColumn, - Collection<?> identifiers, boolean withPropertyTypes, boolean lockForUpdate) + Collection<?> identifiers, final boolean withPropertyTypes, final boolean lockForUpdate) { if (identifiers.size() == 0) { return Collections.emptyList(); } - final Criterion codeIn = Restrictions.in(identifierColumn, identifiers); - - final DetachedCriteria criteria = DetachedCriteria.forClass(ENTITY_CLASS); - criteria.add(codeIn); - criteria.setFetchMode("dataSetType", FetchMode.SELECT); - criteria.setFetchMode("dataStore", FetchMode.SELECT); - criteria.setFetchMode("experimentInternal", FetchMode.SELECT); - criteria.setFetchMode("sampleInternal", FetchMode.SELECT); - criteria.setFetchMode("fileFormat", FetchMode.SELECT); - if (withPropertyTypes) - { - criteria.setFetchMode("dataSetType.dataSetTypePropertyTypesInternal", FetchMode.JOIN); - } - criteria.setResultTransformer(DetachedCriteria.DISTINCT_ROOT_ENTITY); - if (lockForUpdate) - { - criteria.setLockMode(LockMode.UPGRADE); - } - final List<DataPE> list = cast(getHibernateTemplate().findByCriteria(criteria)); + final List<DataPE> list = + DAOUtils.listByCollection(getHibernateTemplate(), new IDetachedCriteriaFactory() + { + public DetachedCriteria createCriteria() + { + final DetachedCriteria criteria = + DetachedCriteria.forClass(ENTITY_CLASS); + criteria.setFetchMode("dataSetType", FetchMode.SELECT); + criteria.setFetchMode("dataStore", FetchMode.SELECT); + criteria.setFetchMode("experimentInternal", FetchMode.SELECT); + criteria.setFetchMode("sampleInternal", FetchMode.SELECT); + criteria.setFetchMode("fileFormat", FetchMode.SELECT); + if (withPropertyTypes) + { + criteria.setFetchMode( + "dataSetType.dataSetTypePropertyTypesInternal", + FetchMode.JOIN); + } + criteria.setResultTransformer(DetachedCriteria.DISTINCT_ROOT_ENTITY); + if (lockForUpdate) + { + criteria.setLockMode(LockMode.UPGRADE); + } + return criteria; + } + }, identifierColumn, identifiers); if (operationLog.isDebugEnabled()) { @@ -964,9 +976,8 @@ final class DataDAO extends AbstractGenericEntityWithPropertiesDAO<DataPE> imple { return new ArrayList<DataPE>(); } - final DetachedCriteria criteria = DetachedCriteria.forClass(DataPE.class); - criteria.add(Restrictions.in("code", values)); - final List<DataPE> list = cast(getHibernateTemplate().findByCriteria(criteria)); + final List<DataPE> list = + DAOUtils.listByCollection(getHibernateTemplate(), DataPE.class, "code", values); if (operationLog.isDebugEnabled()) { operationLog.debug(String.format("%d data set(s) have been found.", list.size())); @@ -1007,11 +1018,19 @@ final class DataDAO extends AbstractGenericEntityWithPropertiesDAO<DataPE> imple public List<TechId> listDataSetIdsBySampleIds(final Collection<TechId> samples) { - final DetachedCriteria criteria = DetachedCriteria.forClass(DataPE.class); final List<Long> longIds = TechId.asLongs(samples); - criteria.setProjection(Projections.id()); - criteria.add(Restrictions.in("sampleInternal.id", longIds)); - final List<Long> results = cast(getHibernateTemplate().findByCriteria(criteria)); + final List<Long> results = + DAOUtils.listByCollection(getHibernateTemplate(), new IDetachedCriteriaFactory() + { + public DetachedCriteria createCriteria() + { + final DetachedCriteria criteria = + DetachedCriteria.forClass(DataPE.class); + criteria.setProjection(Projections.id()); + return criteria; + } + }, "sampleInternal.id", longIds); + if (operationLog.isDebugEnabled()) { operationLog @@ -1022,11 +1041,18 @@ final class DataDAO extends AbstractGenericEntityWithPropertiesDAO<DataPE> imple public List<TechId> listDataSetIdsByExperimentIds(final Collection<TechId> experiments) { - final DetachedCriteria criteria = DetachedCriteria.forClass(DataPE.class); final List<Long> longIds = TechId.asLongs(experiments); - criteria.setProjection(Projections.id()); - criteria.add(Restrictions.in("experimentInternal.id", longIds)); - final List<Long> results = cast(getHibernateTemplate().findByCriteria(criteria)); + final List<Long> results = + DAOUtils.listByCollection(getHibernateTemplate(), new IDetachedCriteriaFactory() + { + public DetachedCriteria createCriteria() + { + final DetachedCriteria criteria = + DetachedCriteria.forClass(DataPE.class); + criteria.setProjection(Projections.id()); + return criteria; + } + }, "experimentInternal.id", longIds); if (operationLog.isDebugEnabled()) { operationLog.info(String.format("found %s data sets for given experiments", @@ -1052,12 +1078,19 @@ final class DataDAO extends AbstractGenericEntityWithPropertiesDAO<DataPE> imple public void execute(List<Long> batchIds) { - final DetachedCriteria criteria = DetachedCriteria.forClass(DataPE.class); - criteria.setProjection(Projections.id()); - criteria.add(Restrictions.in("containerInternal.id", batchIds)); - final List<Long> batchResults = - cast(getHibernateTemplate().findByCriteria(criteria)); - totalResults.addAll(batchResults); + List<Long> result = + DAOUtils.listByCollection(getHibernateTemplate(), + new IDetachedCriteriaFactory() + { + public DetachedCriteria createCriteria() + { + final DetachedCriteria criteria = + DetachedCriteria.forClass(DataPE.class); + criteria.setProjection(Projections.id()); + return criteria; + } + }, "containerInternal.id", batchIds); + totalResults.addAll(result); } public List<Long> getAllEntities() diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/DeletionDAO.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/DeletionDAO.java index 347cb2ee48dff9d485ab44b01ce8d8fae082cab1..f823d4950d6350ed09f3b62dc6006fcb605b6c44 100644 --- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/DeletionDAO.java +++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/DeletionDAO.java @@ -17,12 +17,10 @@ package ch.systemsx.cisd.openbis.generic.server.dataaccess.db; import java.sql.SQLException; -import java.util.ArrayList; import java.util.Collections; import java.util.List; import org.apache.log4j.Logger; -import org.hibernate.Criteria; import org.hibernate.HibernateException; import org.hibernate.SQLQuery; import org.hibernate.Session; @@ -231,34 +229,49 @@ final class DeletionDAO extends AbstractGenericEntityDAO<DeletionPE> implements { return Collections.emptyList(); } - final DetachedCriteria criteria = - DetachedCriteria.forClass(EntityKind.DATA_SET.getDeletedEntityClass()); final List<Long> longIds = TechId.asLongs(deletionIds); - criteria.setProjection(Projections.property("code")); - criteria.add(Restrictions.in(DELETION_ID, longIds)); - final List<String> results = cast(getHibernateTemplate().findByCriteria(criteria)); + final List<String> results = + DAOUtils.listByCollection(getHibernateTemplate(), new IDetachedCriteriaFactory() + { + public DetachedCriteria createCriteria() + { + final DetachedCriteria criteria = + DetachedCriteria.forClass(EntityKind.DATA_SET + .getDeletedEntityClass()); + criteria.setProjection(Projections.property("code")); + return criteria; + } + }, DELETION_ID, longIds); + operationLog.info(String.format("found %s trashed %s(s)", results.size(), EntityKind.DATA_SET.name())); return results; } private List<TechId> findTrashedEntityIds(final List<TechId> deletionIds, - final EntityKind entityKind, Criterion... additionalCriteria) + final EntityKind entityKind, final Criterion... additionalCriteria) { if (deletionIds.isEmpty()) { return Collections.emptyList(); } - final DetachedCriteria criteria = - DetachedCriteria.forClass(entityKind.getDeletedEntityClass()); final List<Long> longIds = TechId.asLongs(deletionIds); - criteria.setProjection(Projections.id()); - criteria.add(Restrictions.in(DELETION_ID, longIds)); - for (Criterion criterion : additionalCriteria) - { - criteria.add(criterion); - } - final List<Long> results = cast(getHibernateTemplate().findByCriteria(criteria)); + final List<Long> results = + DAOUtils.listByCollection(getHibernateTemplate(), new IDetachedCriteriaFactory() + { + public DetachedCriteria createCriteria() + { + final DetachedCriteria criteria = + DetachedCriteria.forClass(entityKind.getDeletedEntityClass()); + criteria.setProjection(Projections.id()); + for (Criterion criterion : additionalCriteria) + { + criteria.add(criterion); + } + return criteria; + } + }, DELETION_ID, longIds); + operationLog .info(String.format("found %s trashed %s(s)", results.size(), entityKind.name())); return transformNumbers2TechIdList(results); @@ -402,11 +415,8 @@ final class DeletionDAO extends AbstractGenericEntityDAO<DeletionPE> implements { return Collections.emptyList(); } - final Criteria criteria = getSession().createCriteria(DeletionPE.class); - criteria.add(Restrictions.in("id", ids)); - - @SuppressWarnings("unchecked") - List<DeletionPE> result = criteria.list(); + List<DeletionPE> result = + DAOUtils.listByCollection(getHibernateTemplate(), DeletionPE.class, "id", ids); if (operationLog.isDebugEnabled()) { operationLog.debug(String.format("%s deletions has been found", result.size())); @@ -451,21 +461,8 @@ final class DeletionDAO extends AbstractGenericEntityDAO<DeletionPE> implements } HibernateTemplate hibernateTemplate = getHibernateTemplate(); List<Long> ids = TechId.asLongs(entityIds); - int chunkSize = 30000; - List<IDeletablePE> result = new ArrayList<IDeletablePE>(); - for (int i = 0, n = ids.size(); i < n; i += chunkSize) - { - DetachedCriteria criteria = - DetachedCriteria.forClass(entityKind.getDeletedEntityClass()); - List<Long> subList = ids.subList(i, Math.min(n, i + chunkSize)); - if (subList.isEmpty() == false) - { - criteria.add(Restrictions.in(ID, subList)); - List<IDeletablePE> list = cast(hibernateTemplate.findByCriteria(criteria)); - result.addAll(list); - } - } - return result; + return DAOUtils.listByCollection(hibernateTemplate, entityKind.getDeletedEntityClass(), ID, + ids); } public List<TechId> listDeletedEntitiesForType(EntityKind entityKind, TechId entityTypeId) diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/ExperimentDAO.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/ExperimentDAO.java index a085be34aeb409ffa03e7910a6dd595748d5897c..1bc90ff24d618fdcff389a3ba37160f88dbe15ca 100644 --- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/ExperimentDAO.java +++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/ExperimentDAO.java @@ -142,11 +142,17 @@ public class ExperimentDAO extends AbstractGenericEntityWithPropertiesDAO<Experi { return new ArrayList<ExperimentPE>(); } - DetachedCriteria criteria = DetachedCriteria.forClass(getEntityClass()); - criteria.add(Restrictions.in("id", experimentIDs)); - criteria.setFetchMode("experimentProperties", FetchMode.JOIN); - criteria.setResultTransformer(Criteria.DISTINCT_ROOT_ENTITY); - final List<ExperimentPE> list = cast(getHibernateTemplate().findByCriteria(criteria)); + final List<ExperimentPE> list = + DAOUtils.listByCollection(getHibernateTemplate(), new IDetachedCriteriaFactory() + { + public DetachedCriteria createCriteria() + { + DetachedCriteria criteria = DetachedCriteria.forClass(getEntityClass()); + criteria.setFetchMode("experimentProperties", FetchMode.JOIN); + criteria.setResultTransformer(Criteria.DISTINCT_ROOT_ENTITY); + return criteria; + } + }, "id", experimentIDs); if (operationLog.isDebugEnabled()) { operationLog.debug(String.format("%d experiments have been found for specified IDs.", @@ -279,9 +285,8 @@ public class ExperimentDAO extends AbstractGenericEntityWithPropertiesDAO<Experi { return new ArrayList<ExperimentPE>(); } - final DetachedCriteria criteria = DetachedCriteria.forClass(ExperimentPE.class); - criteria.add(Restrictions.in(idName, ids)); - final List<ExperimentPE> list = cast(getHibernateTemplate().findByCriteria(criteria)); + final List<ExperimentPE> list = + DAOUtils.listByCollection(getHibernateTemplate(), ExperimentPE.class, idName, ids); if (operationLog.isDebugEnabled()) { operationLog.debug(String.format("%d experiment(s) have been found.", list.size())); diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/IDetachedCriteriaFactory.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/IDetachedCriteriaFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..4113bd977baec4f44108dec4396ae0fce16e7e85 --- /dev/null +++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/IDetachedCriteriaFactory.java @@ -0,0 +1,29 @@ +/* + * Copyright 2012 ETH Zuerich, CISD + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.systemsx.cisd.openbis.generic.server.dataaccess.db; + +import org.hibernate.criterion.DetachedCriteria; + +/** + * Factory of {@link DetachedCriteria} instances. + * + * @author Franz-Josef Elmer + */ +public interface IDetachedCriteriaFactory +{ + public DetachedCriteria createCriteria(); +} diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/MaterialDAO.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/MaterialDAO.java index 9daedb5e34fe77f321a4ca90954eb8634bed501c..822204cb3bdfd655cef06a2490a385e633c5bc5c 100644 --- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/MaterialDAO.java +++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/MaterialDAO.java @@ -199,9 +199,8 @@ public class MaterialDAO extends AbstractGenericEntityWithPropertiesDAO<Material { return Collections.emptyList(); } - Criteria criteria = getSession().createCriteria(ENTITY_CLASS); - criteria.add(Restrictions.in("id", ids)); - final List<MaterialPE> list = cast(criteria.list()); + final List<MaterialPE> list = + DAOUtils.listByCollection(getHibernateTemplate(), ENTITY_CLASS, "id", ids); if (operationLog.isDebugEnabled()) { operationLog.debug(String.format("%d materials have been found for ids: %s.", diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/SampleDAO.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/SampleDAO.java index cb9c14cb41ad387a1369ba367f68300e6803e0fc..efd7f42afadcc136fc6b850c7a186c0e6d9e0cbe 100644 --- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/SampleDAO.java +++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/SampleDAO.java @@ -380,9 +380,8 @@ public class SampleDAO extends AbstractGenericEntityWithPropertiesDAO<SamplePE> { return new ArrayList<SamplePE>(); } - final DetachedCriteria criteria = DetachedCriteria.forClass(SamplePE.class); - criteria.add(Restrictions.in(idName, values)); - final List<SamplePE> list = cast(getHibernateTemplate().findByCriteria(criteria)); + final List<SamplePE> list = + DAOUtils.listByCollection(getHibernateTemplate(), SamplePE.class, idName, values); if (operationLog.isDebugEnabled()) { operationLog.debug(String.format("%d sample(s) have been found.", list.size())); @@ -479,11 +478,18 @@ public class SampleDAO extends AbstractGenericEntityWithPropertiesDAO<SamplePE> public List<TechId> listSampleIdsByContainerIds(final Collection<TechId> containers) { - final DetachedCriteria criteria = DetachedCriteria.forClass(SamplePE.class); final List<Long> longIds = TechId.asLongs(containers); - criteria.setProjection(Projections.id()); - criteria.add(Restrictions.in("container.id", longIds)); - final List<Long> results = cast(getHibernateTemplate().findByCriteria(criteria)); + final List<Long> results = + DAOUtils.listByCollection(getHibernateTemplate(), new IDetachedCriteriaFactory() + { + public DetachedCriteria createCriteria() + { + final DetachedCriteria criteria = + DetachedCriteria.forClass(SamplePE.class); + criteria.setProjection(Projections.id()); + return criteria; + } + }, "container.id", longIds); if (operationLog.isDebugEnabled()) { operationLog.info(String.format("found %s sample components for given containers", @@ -494,11 +500,18 @@ public class SampleDAO extends AbstractGenericEntityWithPropertiesDAO<SamplePE> public List<TechId> listSampleIdsByExperimentIds(final Collection<TechId> experiments) { - final DetachedCriteria criteria = DetachedCriteria.forClass(SamplePE.class); final List<Long> longIds = TechId.asLongs(experiments); - criteria.setProjection(Projections.id()); - criteria.add(Restrictions.in("experimentInternal.id", longIds)); - final List<Long> results = cast(getHibernateTemplate().findByCriteria(criteria)); + final List<Long> results = + DAOUtils.listByCollection(getHibernateTemplate(), new IDetachedCriteriaFactory() + { + public DetachedCriteria createCriteria() + { + final DetachedCriteria criteria = + DetachedCriteria.forClass(SamplePE.class); + criteria.setProjection(Projections.id()); + return criteria; + } + }, "experimentInternal.id", longIds); if (operationLog.isDebugEnabled()) { operationLog.info(String.format("found %s samples for given experiments",