From 1cb7a6b9502e3c91f8cfcd4265e8bdd9c51f1da5 Mon Sep 17 00:00:00 2001
From: buczekp <buczekp>
Date: Sun, 17 Oct 2010 19:07:30 +0000
Subject: [PATCH] [LMS-1840] made sample batch update scale linearly
SVN: 18327
---
.../server/batch/BatchOperationExecutor.java | 23 ++++++++++++++---
.../server/batch/SampleBatchRegistration.java | 2 +-
.../server/batch/SampleBatchUpdate.java | 2 +-
.../server/business/bo/ISampleTable.java | 13 ++++++----
.../server/business/bo/SampleTable.java | 25 ++++++++-----------
.../server/dataaccess/db/SampleDAO.java | 5 ++--
6 files changed, 43 insertions(+), 27 deletions(-)
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/batch/BatchOperationExecutor.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/batch/BatchOperationExecutor.java
index 53883fafe4e..852d29a98d5 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/batch/BatchOperationExecutor.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/batch/BatchOperationExecutor.java
@@ -14,8 +14,8 @@ import ch.systemsx.cisd.common.logging.LogFactory;
*/
public class BatchOperationExecutor
{
- private static final Logger operationLog =
- LogFactory.getLogger(LogCategory.OPERATION, BatchOperationExecutor.class);
+ private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
+ BatchOperationExecutor.class);
private static final int DEFAULT_BATCH_SIZE = 1000;
@@ -31,15 +31,30 @@ public class BatchOperationExecutor
final List<S> allEntities = strategy.getAllEntities();
int maxIndex = allEntities.size();
+ operationLog.info(getMemoryUsageMessage());
// Loop over the list, one block at a time
for (int startIndex = 0, endIndex = Math.min(startIndex + batchSize, maxIndex); startIndex < maxIndex; startIndex =
endIndex, endIndex = Math.min(startIndex + batchSize, maxIndex))
{
final List<S> batch = allEntities.subList(startIndex, endIndex);
strategy.execute(batch);
- operationLog.info(String.format("%s %s progress: %d/%d", strategy.getEntityName(),
- strategy.getOperationName(), endIndex, maxIndex));
+ operationLog.info(String.format("%s %s progress: %d-%d/%d", strategy.getEntityName(),
+ strategy.getOperationName(), startIndex, endIndex, maxIndex));
+ if (operationLog.isDebugEnabled())
+ {
+ operationLog.debug(getMemoryUsageMessage());
+ }
}
}
+ private static String getMemoryUsageMessage()
+ {
+ Runtime runtime = Runtime.getRuntime();
+ long mb = 1024l * 1024l;
+ long totalMemory = runtime.totalMemory() / mb;
+ long freeMemory = runtime.freeMemory() / mb;
+ long maxMemory = runtime.maxMemory() / mb;
+ return "MEMORY (in MB): free:" + freeMemory + " total:" + totalMemory + " max:" + maxMemory;
+ }
+
}
\ No newline at end of file
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/batch/SampleBatchRegistration.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/batch/SampleBatchRegistration.java
index c385c0106ef..5e1d825ca90 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/batch/SampleBatchRegistration.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/batch/SampleBatchRegistration.java
@@ -40,7 +40,7 @@ public class SampleBatchRegistration implements IBatchOperation<NewSample>
public void execute(List<NewSample> batch)
{
- businessTable.set(batch);
+ businessTable.prepareForRegistration(batch);
businessTable.save();
}
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/batch/SampleBatchUpdate.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/batch/SampleBatchUpdate.java
index b5c0bcd0288..e2195e4f835 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/batch/SampleBatchUpdate.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/batch/SampleBatchUpdate.java
@@ -40,7 +40,7 @@ public class SampleBatchUpdate implements IBatchOperation<SampleBatchUpdatesDTO>
public void execute(List<SampleBatchUpdatesDTO> updates)
{
- businessTable.update(updates);
+ businessTable.prepareForUpdate(updates);
businessTable.save();
}
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ISampleTable.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ISampleTable.java
index a8e3e912a7d..5aff88420cc 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ISampleTable.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ISampleTable.java
@@ -45,16 +45,19 @@ public interface ISampleTable
List<SamplePE> getSamples();
/**
- * Sets given samples to be handled by this table.
+ * Prepares given samples for registration and stores them in this table.
*/
- public void set(List<NewSample> newSamples) throws UserFailureException;
+ public void prepareForRegistration(List<NewSample> newSamples) throws UserFailureException;
/**
- * Writes changed are added data to the Data Access Layers.
+ * Prepares given samples for update and stores them in this table.
*/
- public void save() throws UserFailureException;
+ public void prepareForUpdate(List<SampleBatchUpdatesDTO> updates) throws UserFailureException;
- void update(List<SampleBatchUpdatesDTO> updates);
+ /**
+ * Writes changed or added data to the Data Access Layers.
+ */
+ public void save() throws UserFailureException;
/**
* Deletes samples for specified reason.
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/SampleTable.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/SampleTable.java
index 94a0f15b1c4..2be1156625d 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/SampleTable.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/SampleTable.java
@@ -83,8 +83,8 @@ public final class SampleTable extends AbstractSampleBusinessObject implements I
{
foundSamples =
filterSamplesByExperiment(foundSamples, criteria.getProjectIdentifier(),
- experimentPropertyCode, localExperimentIdentifier
- .getPropertyValue());
+ experimentPropertyCode,
+ localExperimentIdentifier.getPropertyValue());
} else
{
ExperimentIdentifier ident =
@@ -161,7 +161,7 @@ public final class SampleTable extends AbstractSampleBusinessObject implements I
return samples;
}
- public void set(List<NewSample> newSamples) throws UserFailureException
+ public void prepareForRegistration(List<NewSample> newSamples) throws UserFailureException
{
onlyNewSamples = true;
samples = new ArrayList<SamplePE>();
@@ -216,7 +216,7 @@ public final class SampleTable extends AbstractSampleBusinessObject implements I
}
}
- public SamplePE prepareBatchUpdate(SampleBatchUpdatesDTO updates,
+ private SamplePE prepareBatchUpdate(SampleBatchUpdatesDTO updates,
Map<SampleOwnerIdentifier, SampleOwner> sampleOwnerCache,
Map<String, ExperimentPE> experimentCache)
{
@@ -239,13 +239,13 @@ public final class SampleTable extends AbstractSampleBusinessObject implements I
}
if (details.isParentUpdateRequested())
{
- setGeneratedFrom(updates.getSampleIdentifier(), sample, updates
- .getParentIdentifierOrNull());
+ setGeneratedFrom(updates.getSampleIdentifier(), sample,
+ updates.getParentIdentifierOrNull());
}
if (details.isContainerUpdateRequested())
{
- setContainer(updates.getSampleIdentifier(), sample, updates
- .getContainerIdentifierOrNull());
+ setContainer(updates.getSampleIdentifier(), sample,
+ updates.getContainerIdentifierOrNull());
}
return sample;
}
@@ -260,15 +260,12 @@ public final class SampleTable extends AbstractSampleBusinessObject implements I
properties, registrator, propertiesToUpdate));
}
- public void update(List<SampleBatchUpdatesDTO> updates)
+ public void prepareForUpdate(List<SampleBatchUpdatesDTO> updates)
{
assert updates != null : "Unspecified samples.";
- setBatchUpdateMode(true);
- if (samples == null)
- {
- samples = new ArrayList<SamplePE>();
- }
+ setBatchUpdateMode(true);
+ samples = new ArrayList<SamplePE>();
final Map<SampleOwnerIdentifier, SampleOwner> sampleOwnerCache =
new HashMap<SampleOwnerIdentifier, SampleOwner>();
final Map<String, ExperimentPE> experimentCache = new HashMap<String, ExperimentPE>();
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/SampleDAO.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/SampleDAO.java
index aaaac501b87..0d91e911425 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/SampleDAO.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/SampleDAO.java
@@ -302,10 +302,11 @@ public class SampleDAO extends AbstractGenericEntityDAO<SamplePE> implements ISa
operationLog.info(String.format("ADD: %d samples.", samples.size()));
}
- // TODO 2010-06-16, Piotr Buczek: is memory usage increasing without clear of session?
-
// need to deal with exception thrown by trigger checking code uniqueness
flushWithSqlExceptionHandling(getHibernateTemplate());
+
+ // if session is not cleared registration of many samples slows down after each batch
+ hibernateTemplate.clear();
}
public final void updateSample(final SamplePE sample) throws DataAccessException
--
GitLab