From b2ee03558430fe21f4ef03c1dcf4de66f79415d5 Mon Sep 17 00:00:00 2001
From: juanf <juanf>
Date: Thu, 6 Oct 2016 08:07:57 +0000
Subject: [PATCH] SSDM-4192 : Bugfix, Batch InQuery class to avoid postgres
driver limit with test.
SVN: 37145
---
.../generic/server/dataaccess/db/DataDAO.java | 18 ++++-----
.../generic/server/dataaccess/db/InQuery.java | 37 +++++++++++++++++++
.../systemtest/asapi/v3/UpdateSampleTest.java | 20 +++++++++-
3 files changed, 65 insertions(+), 10 deletions(-)
create mode 100644 openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/InQuery.java
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/DataDAO.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/DataDAO.java
index 1a342831760..43caf1f12bd 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/DataDAO.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/DataDAO.java
@@ -21,7 +21,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
-import java.util.Deque;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
@@ -147,9 +146,10 @@ final class DataDAO extends AbstractGenericEntityWithPropertiesDAO<DataPE> imple
@Override
public Object doInHibernate(Session session) throws HibernateException
{
- SQLQuery query = session.createSQLQuery("select distinct samp_id from data where samp_id in (:sampleIds)");
- query.setParameterList("sampleIds", sampleIds);
- List<Number> list = query.list();
+
+ InQuery inQuery = new InQuery<Long, Number>();
+ List<Number> list =
+ inQuery.withBatch(session, "select distinct samp_id from data where samp_id in (:sampleIds)", "sampleIds", sampleIds);
Set<Long> ids = new HashSet<Long>();
for (Number item : list)
{
@@ -1330,11 +1330,11 @@ final class DataDAO extends AbstractGenericEntityWithPropertiesDAO<DataPE> imple
flushWithSqlExceptionHandling(getHibernateTemplate());
scheduleDynamicPropertiesEvaluation(dataSets);
-
+
// if session is not cleared registration of many samples slows down after each batch
hibernateTemplate.clear();
}
-
+
@Override
public final void validateAndSaveUpdatedEntity(DataPE entity) throws DataAccessException
{
@@ -1344,13 +1344,13 @@ final class DataDAO extends AbstractGenericEntityWithPropertiesDAO<DataPE> imple
}
@Override
- protected void scheduleDynamicPropertiesEvaluation(List<DataPE> dataSets)
+ protected void scheduleDynamicPropertiesEvaluation(List<DataPE> dataSets)
{
List<DataPE> toUpdate = new ArrayList<DataPE>();
addAllDataSetsAndComponentsRecursively(toUpdate, dataSets);
super.scheduleDynamicPropertiesEvaluation(toUpdate);
}
-
+
private void addAllDataSetsAndComponentsRecursively(List<DataPE> resultDataSets, List<DataPE> dataSets)
{
for (DataPE dataSet : dataSets)
@@ -1359,7 +1359,7 @@ final class DataDAO extends AbstractGenericEntityWithPropertiesDAO<DataPE> imple
addAllDataSetsAndComponentsRecursively(resultDataSets, dataSet.getContainedDataSets());
}
}
-
+
@Override
public List<TechId> listDataSetIdsBySampleIds(final Collection<TechId> samples)
{
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/InQuery.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/InQuery.java
new file mode 100644
index 00000000000..0c587b01bbe
--- /dev/null
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/InQuery.java
@@ -0,0 +1,37 @@
+package ch.systemsx.cisd.openbis.generic.server.dataaccess.db;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.hibernate.SQLQuery;
+import org.hibernate.Session;
+
+public class InQuery<I, O>
+{
+ private static final int POSTGRES_DRIVER_MAX_ARGS = 32767; // Uses a signed 2 bytes integer
+
+ public List<O> withBatch(Session session, String inQuery, String inParameter, List<I> inArguments)
+ {
+ List<O> result = new ArrayList<O>(inArguments.size());
+
+ int fromIndex = 0;
+ while (fromIndex < inArguments.size())
+ {
+ int toIndex = fromIndex + POSTGRES_DRIVER_MAX_ARGS;
+ if (toIndex > inArguments.size())
+ {
+ toIndex = inArguments.size();
+ }
+
+ List<I> partialInArguments = inArguments.subList(fromIndex, toIndex);
+
+ SQLQuery query = session.createSQLQuery(inQuery);
+ query.setParameterList(inParameter, partialInArguments);
+ List<O> partialResult = query.list();
+ result.addAll(partialResult);
+ fromIndex = toIndex;
+ }
+
+ return result;
+ }
+}
\ No newline at end of file
diff --git a/openbis/sourceTest/java/ch/ethz/sis/openbis/systemtest/asapi/v3/UpdateSampleTest.java b/openbis/sourceTest/java/ch/ethz/sis/openbis/systemtest/asapi/v3/UpdateSampleTest.java
index 2472b332853..80aac39c4ba 100644
--- a/openbis/sourceTest/java/ch/ethz/sis/openbis/systemtest/asapi/v3/UpdateSampleTest.java
+++ b/openbis/sourceTest/java/ch/ethz/sis/openbis/systemtest/asapi/v3/UpdateSampleTest.java
@@ -52,7 +52,6 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.tag.id.TagCode;
import ch.ethz.sis.openbis.systemtest.asapi.v3.index.ReindexingState;
import ch.systemsx.cisd.common.action.IDelegatedAction;
import ch.systemsx.cisd.common.test.AssertionUtil;
-
import junit.framework.Assert;
/**
@@ -61,6 +60,25 @@ import junit.framework.Assert;
public class UpdateSampleTest extends AbstractSampleTest
{
+ @Test
+ public void testUpdateBiggerThanPostgresDriverArgumentsLimitWithIndexCheck()
+ {
+ String sessionToken = v3api.login(TEST_USER, PASSWORD);
+ ReindexingState state = new ReindexingState();
+ List<SampleUpdate> updates = new ArrayList<SampleUpdate>();
+ for (int i = 0; i < 40000; i++)
+ {
+ SampleUpdate update = new SampleUpdate();
+ update.setSampleId(new SamplePermId("200811050945092-976"));
+ update.setProperty("OFFSET", "50");
+ updates.add(update);
+ }
+
+ v3api.updateSamples(sessionToken, updates);
+
+ assertSamplesReindexed(state, "200811050945092-976");
+ }
+
@Test
public void testUpdateWithIndexCheck()
{
--
GitLab