From 0d77fd303e5ab1a2aabca9953375489ef76252f7 Mon Sep 17 00:00:00 2001
From: buczekp <buczekp>
Date: Thu, 29 Apr 2010 12:02:33 +0000
Subject: [PATCH] [LMS-1507] improved speed of updating archiving statuses
SVN: 15679
---
.../ch/systemsx/cisd/etlserver/ETLDaemon.java | 6 +-
.../server/EncapsulatedOpenBISService.java | 22 +--
.../server/plugins/demo/DemoArchiver.java | 6 +-
.../AbstractArchiverProcessingPlugin.java | 25 ++--
.../shared/IEncapsulatedOpenBISService.java | 6 +-
.../QueueingDataSetStatusUpdaterService.java | 53 ++++----
...tatus.java => DataSetCodesWithStatus.java} | 15 ++-
.../openbis/generic/server/ETLService.java | 6 +-
.../generic/server/ETLServiceLogger.java | 6 +-
.../server/business/bo/ExternalDataBO.java | 6 +-
.../server/business/bo/ExternalDataTable.java | 38 +++---
.../server/business/bo/IExternalDataBO.java | 6 +-
.../server/dataaccess/IExternalDataDAO.java | 8 +-
.../server/dataaccess/db/ExternalDataDAO.java | 126 ++++++++++++++++--
.../generic/shared/IETLLIMSService.java | 17 +--
.../business/bo/ExternalDataBOTest.java | 13 +-
.../business/bo/ExternalDataTableTest.java | 2 +-
17 files changed, 239 insertions(+), 122 deletions(-)
rename datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/dto/{DataSetCodeWithStatus.java => DataSetCodesWithStatus.java} (74%)
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/ETLDaemon.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/ETLDaemon.java
index 5e5304dff65..8cee248982b 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/ETLDaemon.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/ETLDaemon.java
@@ -64,7 +64,7 @@ import ch.systemsx.cisd.openbis.dss.BuildAndEnvironmentInfo;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
import ch.systemsx.cisd.openbis.dss.generic.shared.QueueingDataSetStatusUpdaterService;
import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
-import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetCodeWithStatus;
+import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetCodesWithStatus;
import ch.systemsx.cisd.openbis.dss.generic.shared.utils.PropertyParametersUtil;
import ch.systemsx.cisd.openbis.generic.shared.IServer;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DatabaseInstance;
@@ -122,7 +122,7 @@ public final class ETLDaemon
public static void listUpdaterQueue()
{
- final List<DataSetCodeWithStatus> items =
+ final List<DataSetCodesWithStatus> items =
QueueingDataSetStatusUpdaterService.listItems(updaterQueueFile);
if (items.isEmpty())
{
@@ -130,7 +130,7 @@ public final class ETLDaemon
} else
{
System.out.println("Found " + items.size() + " items in updater:");
- for (final DataSetCodeWithStatus item : items)
+ for (final DataSetCodesWithStatus item : items)
{
System.out.println(item);
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/EncapsulatedOpenBISService.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/EncapsulatedOpenBISService.java
index 5c5eb985b4f..90bfb4102c6 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/EncapsulatedOpenBISService.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/EncapsulatedOpenBISService.java
@@ -308,7 +308,8 @@ public final class EncapsulatedOpenBISService implements IEncapsulatedOpenBISSer
}
}
- synchronized public ExperimentType getExperimentType(String experimentTypeCode) throws UserFailureException
+ synchronized public ExperimentType getExperimentType(String experimentTypeCode)
+ throws UserFailureException
{
checkSessionToken();
try
@@ -408,7 +409,8 @@ public final class EncapsulatedOpenBISService implements IEncapsulatedOpenBISSer
}
}
- synchronized public void updateSample(SampleUpdatesDTO sampleUpdate) throws UserFailureException
+ synchronized public void updateSample(SampleUpdatesDTO sampleUpdate)
+ throws UserFailureException
{
assert sampleUpdate != null : "Unspecified sample.";
@@ -474,31 +476,31 @@ public final class EncapsulatedOpenBISService implements IEncapsulatedOpenBISSer
service.addPropertiesToDataSet(sessionToken, properties, code, space);
}
- synchronized public final void updateDataSetStatus(String code, DataSetArchivingStatus newStatus)
- throws UserFailureException
+ synchronized public final void updateDataSetStatuses(List<String> codes,
+ DataSetArchivingStatus newStatus) throws UserFailureException
{
- assert code != null : "missing data set code";
+ assert codes != null : "missing data set codes";
assert newStatus != null : "missing status";
checkSessionToken();
try
{
- primUpdateDataSetStatus(code, newStatus);
+ primUpdateDataSetStatuses(codes, newStatus);
} catch (final InvalidSessionException ex)
{
authenticate();
- primUpdateDataSetStatus(code, newStatus);
+ primUpdateDataSetStatuses(codes, newStatus);
}
if (operationLog.isInfoEnabled())
{
- operationLog.info("Updated in openBIS: data set " + code + ", status=" + newStatus);
+ operationLog.info("Updated in openBIS: data sets " + codes + ", status=" + newStatus);
}
}
- private void primUpdateDataSetStatus(String code, DataSetArchivingStatus newStatus)
+ private void primUpdateDataSetStatuses(List<String> codes, DataSetArchivingStatus newStatus)
{
- service.updateDataSetStatus(sessionToken, code, newStatus);
+ service.updateDataSetStatuses(sessionToken, codes, newStatus);
}
synchronized public final IEntityProperty[] getPropertiesOfTopSampleRegisteredFor(
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/demo/DemoArchiver.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/demo/DemoArchiver.java
index 509af5bba2a..4e6b846d4ce 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/demo/DemoArchiver.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/demo/DemoArchiver.java
@@ -21,7 +21,6 @@ import java.util.Properties;
import ch.systemsx.cisd.common.exceptions.UserFailureException;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.AbstractArchiverProcessingPlugin;
-import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.HighWaterMarkChecker;
import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
/**
@@ -33,8 +32,9 @@ public class DemoArchiver extends AbstractArchiverProcessingPlugin
public DemoArchiver(Properties properties, File storeRoot)
{
- super(properties, storeRoot, new HighWaterMarkChecker(storeRoot), new HighWaterMarkChecker(
- storeRoot));
+ super(properties, storeRoot, null, null);
+ // NOTE using HighWaterMarkChecker before archiving every dataset degrades performance
+ // super(properties, storeRoot, new HighWaterMarkChecker(storeRoot), new HighWaterMarkChecker(storeRoot);
}
@Override
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java
index 0b93ede47f1..5002ecaec9a 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java
@@ -17,6 +17,7 @@
package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard;
import java.io.File;
+import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
@@ -30,7 +31,7 @@ import ch.systemsx.cisd.openbis.dss.generic.server.ProcessDatasetsCommand;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.IArchiverTask;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.ProcessingStatus;
import ch.systemsx.cisd.openbis.dss.generic.shared.QueueingDataSetStatusUpdaterService;
-import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetCodeWithStatus;
+import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetCodesWithStatus;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetArchivingStatus;
import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
@@ -66,8 +67,7 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
public ProcessingStatus archive(List<DatasetDescription> datasets)
{
- operationLog
- .info("Archiving of the following datasets has been requested: " + datasets);
+ operationLog.info("Archiving of the following datasets has been requested: " + datasets);
return handleDatasets(datasets, DataSetArchivingStatus.ARCHIVED,
DataSetArchivingStatus.AVAILABLE, new IDatasetDescriptionHandler()
{
@@ -97,8 +97,7 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
public ProcessingStatus unarchive(List<DatasetDescription> datasets)
{
- operationLog.info("Unarchiving of the following datasets has been requested: "
- + datasets);
+ operationLog.info("Unarchiving of the following datasets has been requested: " + datasets);
return handleDatasets(datasets, DataSetArchivingStatus.AVAILABLE,
DataSetArchivingStatus.ARCHIVED, new IDatasetDescriptionHandler()
{
@@ -131,17 +130,27 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
IDatasetDescriptionHandler handler)
{
final ProcessingStatus result = new ProcessingStatus();
+ List<String> successful = new ArrayList<String>();
+ List<String> failed = new ArrayList<String>();
for (DatasetDescription dataset : datasets)
{
Status status = handler.handle(dataset);
- DataSetArchivingStatus newStatus = status.isError() ? failure : success;
- QueueingDataSetStatusUpdaterService.update(new DataSetCodeWithStatus(dataset
- .getDatasetCode(), newStatus));
+ List<String> codes = status.isError() ? failed : successful;
+ codes.add(dataset.getDatasetCode());
result.addDatasetStatus(dataset, status);
}
+ asyncUpdateStatuses(successful, success);
+ asyncUpdateStatuses(failed, failure);
return result;
}
+ private static void asyncUpdateStatuses(List<String> dataSetCodes,
+ DataSetArchivingStatus newStatus)
+ {
+ QueueingDataSetStatusUpdaterService.update(new DataSetCodesWithStatus(dataSetCodes,
+ newStatus));
+ }
+
private interface IDatasetDescriptionHandler
{
public Status handle(DatasetDescription dataset);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IEncapsulatedOpenBISService.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IEncapsulatedOpenBISService.java
index fdcb57eba66..f7e2ecae3bf 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IEncapsulatedOpenBISService.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IEncapsulatedOpenBISService.java
@@ -89,7 +89,7 @@ public interface IEncapsulatedOpenBISService
* Gets the experiment type with assigned property types for the specified experiment type code.
*/
public ExperimentType getExperimentType(String experimentTypeCode) throws UserFailureException;
-
+
/**
* Gets the sample type with assigned property types for the specified sample type code.
*/
@@ -208,7 +208,7 @@ public interface IEncapsulatedOpenBISService
/** See {@link IETLLIMSService#unarchiveDatasets(String, List)} */
public void unarchiveDataSets(List<String> dataSetCodes) throws UserFailureException;
- /** See {@link IETLLIMSService#updateDataSetStatus(String, String, DataSetArchivingStatus)} */
- public void updateDataSetStatus(String code, DataSetArchivingStatus newStatus)
+ /** See {@link IETLLIMSService#updateDataSetStatuses(String, List, DataSetArchivingStatus)} */
+ public void updateDataSetStatuses(List<String> codes, DataSetArchivingStatus newStatus)
throws UserFailureException;
}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/QueueingDataSetStatusUpdaterService.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/QueueingDataSetStatusUpdaterService.java
index 1d419b2b579..5ead3ce28c7 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/QueueingDataSetStatusUpdaterService.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/QueueingDataSetStatusUpdaterService.java
@@ -33,12 +33,12 @@ import ch.systemsx.cisd.common.exceptions.UserFailureException;
import ch.systemsx.cisd.common.filesystem.ICloseable;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
-import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetCodeWithStatus;
+import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetCodesWithStatus;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetArchivingStatus;
/**
* A service for updating data set status in openBIS. It provides a method
- * {@link #update(DataSetCodeWithStatus)} that queues updates using a separate thread to actually
+ * {@link #update(DataSetCodesWithStatus)} that queues updates using a separate thread to actually
* perform update.
* <p>
* Note that the service needs to be started via {@link #start(File, TimingParameters)}.
@@ -62,7 +62,7 @@ public class QueueingDataSetStatusUpdaterService
@Private
final static String UPDATER_PREFIX = ".UPDATER_";
- private static IExtendedBlockingQueue<DataSetCodeWithStatus> queue = null;
+ private static IExtendedBlockingQueue<DataSetCodesWithStatus> queue = null;
private static ICloseable queueCloseableOrNull = null;
@@ -91,7 +91,7 @@ public class QueueingDataSetStatusUpdaterService
*/
public static synchronized final void start(final File queueFile, TimingParameters parameters)
{
- final PersistentExtendedBlockingQueueDecorator<DataSetCodeWithStatus> persistentQueue =
+ final PersistentExtendedBlockingQueueDecorator<DataSetCodesWithStatus> persistentQueue =
ExtendedBlockingQueueFactory.createPersistRecordBased(queueFile,
INITIAL_RECORD_SIZE);
queue = persistentQueue;
@@ -105,10 +105,10 @@ public class QueueingDataSetStatusUpdaterService
{
while (true)
{
- final DataSetCodeWithStatus dataSet = queue.peekWait();
+ final DataSetCodesWithStatus dataSets = queue.peekWait();
try
{
- updater.updateDataSetStatus(dataSet.getDataSetCode(), dataSet
+ updater.updateDataSetStatuses(dataSets.getDataSetCodes(), dataSets
.getStatus());
// Note: this is the only consumer of this queue.
queue.take();
@@ -121,16 +121,16 @@ public class QueueingDataSetStatusUpdaterService
// If connection with openBIS fails it is not possible
// the same problem will occur for other updates in the queue,
// so we just retry after increasing time.
- notifyUpdateFailure(dataSet, ex);
+ notifyUpdateFailure(dataSets, ex);
Sleeper.sleepAndIncreaseSleepTime();
} catch (UserFailureException ex)
{
// OpenBIS failure occurred - the problem may be connected with
// certain data set so move this item to the end of the queue and
// try to update other data sets before retrying.
- notifyUpdateFailure(dataSet, ex);
+ notifyUpdateFailure(dataSets, ex);
Sleeper.sleepAndIncreaseSleepTime();
- queue.add(dataSet);
+ queue.add(dataSets);
queue.remove();
}
}
@@ -143,10 +143,10 @@ public class QueueingDataSetStatusUpdaterService
}
}
- private void notifyUpdateFailure(final DataSetCodeWithStatus dataSet, Exception ex)
+ private void notifyUpdateFailure(final DataSetCodesWithStatus dataSets, Exception ex)
{
- notificationLog.error("Update of data set " + dataSet.getDataSetCode()
- + " status to '" + dataSet.getStatus()
+ notificationLog.error("Update of data sets " + dataSets.getDataSetCodes()
+ + " status to '" + dataSets.getStatus()
+ "' has failed.\nRetry will occur not sooner than "
+ Sleeper.getCurrentSleepTime() + ".", ex);
}
@@ -159,23 +159,27 @@ public class QueueingDataSetStatusUpdaterService
{
return new IDataSetStatusUpdater()
{
- public void updateDataSetStatus(String dataSetCode,
+ public void updateDataSetStatuses(List<String> dataSetCodes,
DataSetArchivingStatus newStatus)
{
- ServiceProvider.getOpenBISService().updateDataSetStatus(dataSetCode, newStatus);
- operationLog
- .info("Data Set " + dataSetCode + " changed status to " + newStatus);
+ ServiceProvider.getOpenBISService().updateDataSetStatuses(dataSetCodes,
+ newStatus);
+ operationLog.info("Data Sets " + dataSetCodes + " changed status to "
+ + newStatus);
}
};
}
/**
- * Schedules update of given data set. If operation fails the updating thread will exit.
+ * Schedules update of given data sets.
*/
- public static void update(DataSetCodeWithStatus dataSet)
+ public static void update(DataSetCodesWithStatus dataSets)
{
- queue.add(dataSet);
+ if (dataSets.getDataSetCodes().isEmpty() == false)
+ {
+ queue.add(dataSets);
+ }
}
private static final void close()
@@ -242,9 +246,9 @@ public class QueueingDataSetStatusUpdaterService
/**
* Returns the list of currently queued up items.
*/
- public static final List<DataSetCodeWithStatus> listItems(File queueFile)
+ public static final List<DataSetCodesWithStatus> listItems(File queueFile)
{
- return RecordBasedQueuePersister.list(DataSetCodeWithStatus.class, queueFile);
+ return RecordBasedQueuePersister.list(DataSetCodesWithStatus.class, queueFile);
}
private QueueingDataSetStatusUpdaterService()
@@ -304,12 +308,13 @@ public class QueueingDataSetStatusUpdaterService
public interface IDataSetStatusUpdater
{
/**
- * Updates status of data set with given code.
+ * Updates status of data sets with given codes.
*
- * @param dataSetCode code of data set to be updated
+ * @param dataSetCodes codes of data sets to be updated
* @param newStatus status to be set
*/
- public void updateDataSetStatus(String dataSetCode, DataSetArchivingStatus newStatus);
+ public void updateDataSetStatuses(List<String> dataSetCodes,
+ DataSetArchivingStatus newStatus);
}
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/dto/DataSetCodeWithStatus.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/dto/DataSetCodesWithStatus.java
similarity index 74%
rename from datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/dto/DataSetCodeWithStatus.java
rename to datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/dto/DataSetCodesWithStatus.java
index 26e6b72d386..5a72c8ea761 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/dto/DataSetCodeWithStatus.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/dto/DataSetCodesWithStatus.java
@@ -17,29 +17,30 @@
package ch.systemsx.cisd.openbis.dss.generic.shared.dto;
import java.io.Serializable;
+import java.util.List;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetArchivingStatus;
/**
* @author Piotr Buczek
*/
-public class DataSetCodeWithStatus implements Serializable
+public class DataSetCodesWithStatus implements Serializable
{
private static final long serialVersionUID = 1L;
- private String dataSetCode;
+ private List<String> dataSetCodes;
private DataSetArchivingStatus status;
- public DataSetCodeWithStatus(String dataSetCode, DataSetArchivingStatus status)
+ public DataSetCodesWithStatus(List<String> dataSetCodes, DataSetArchivingStatus status)
{
- this.dataSetCode = dataSetCode;
+ this.dataSetCodes = dataSetCodes;
this.status = status;
}
- public String getDataSetCode()
+ public List<String> getDataSetCodes()
{
- return dataSetCode;
+ return dataSetCodes;
}
public DataSetArchivingStatus getStatus()
@@ -50,7 +51,7 @@ public class DataSetCodeWithStatus implements Serializable
@Override
public String toString()
{
- return dataSetCode + " - " + getStatus();
+ return dataSetCodes + " - " + getStatus();
}
}
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ETLService.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ETLService.java
index a3f43f8d136..a3d24afdab1 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ETLService.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ETLService.java
@@ -438,7 +438,7 @@ public class ETLService extends AbstractCommonServer<IETLService> implements IET
throws UserFailureException
{
final Session session = getSession(sessionToken);
-
+
IExternalDataTable dataSetTable = businessObjectFactory.createExternalDataTable(session);
dataSetTable.loadByExperimentTechId(experimentID);
return ExternalDataTranslator.translate(dataSetTable.getExternalData(), "", "");
@@ -589,13 +589,13 @@ public class ETLService extends AbstractCommonServer<IETLService> implements IET
externalDataBO.addPropertiesToDataSet(dataSetCode, properties);
}
- public void updateDataSetStatus(String sessionToken, String dataSetCode,
+ public void updateDataSetStatuses(String sessionToken, List<String> dataSetCodes,
DataSetArchivingStatus newStatus) throws UserFailureException
{
assert sessionToken != null : "Unspecified session token.";
final Session session = getSession(sessionToken);
final IExternalDataBO externalDataBO = businessObjectFactory.createExternalDataBO(session);
- externalDataBO.updateStatus(dataSetCode, newStatus);
+ externalDataBO.updateStatuses(dataSetCodes, newStatus);
}
public ExternalData tryGetDataSet(String sessionToken, String dataSetCode)
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ETLServiceLogger.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ETLServiceLogger.java
index 6e12684d691..775e7da6781 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ETLServiceLogger.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ETLServiceLogger.java
@@ -252,11 +252,11 @@ public class ETLServiceLogger extends AbstractServerLogger implements IETLServic
properties.size());
}
- public void updateDataSetStatus(String sessionToken, String dataSetCode,
+ public void updateDataSetStatuses(String sessionToken, List<String> dataSetCodes,
DataSetArchivingStatus newStatus) throws UserFailureException
{
- logTracking(sessionToken, "updateDataSetStatus", "DATA_SET_CODE(%s) STATUS(%s)",
- dataSetCode, newStatus);
+ logTracking(sessionToken, "updateDataSetStatus", "DATA_SET_CODES(%s) STATUS(%s)",
+ dataSetCodes, newStatus);
}
}
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ExternalDataBO.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ExternalDataBO.java
index 94cb5b5dfcf..f4ae211db37 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ExternalDataBO.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ExternalDataBO.java
@@ -102,7 +102,7 @@ public class ExternalDataBO extends AbstractExternalDataBusinessObject implement
loadByCode(dataSetCode, true, false);
}
- public void loadByCode(String dataSetCode, boolean withPropertyTypes, boolean lockForUpdate)
+ private void loadByCode(String dataSetCode, boolean withPropertyTypes, boolean lockForUpdate)
{
externalData =
getExternalDataDAO().tryToFindFullDataSetByCode(dataSetCode, withPropertyTypes,
@@ -662,9 +662,9 @@ public class ExternalDataBO extends AbstractExternalDataBusinessObject implement
return result;
}
- public void updateStatus(String dataSetCode, DataSetArchivingStatus newStatus)
+ public void updateStatuses(List<String> dataSetCodes, DataSetArchivingStatus newStatus)
{
- getExternalDataDAO().updateDataSetStatus(dataSetCode, newStatus);
+ getExternalDataDAO().updateDataSetStatuses(dataSetCodes, newStatus);
}
}
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ExternalDataTable.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ExternalDataTable.java
index 05260ac6020..7601abf7deb 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ExternalDataTable.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ExternalDataTable.java
@@ -163,7 +163,7 @@ public final class ExternalDataTable extends AbstractExternalDataBusinessObject
IExternalDataDAO externalDataDAO = getExternalDataDAO();
externalData = new ArrayList<ExternalDataPE>();
- externalData.addAll(externalDataDAO.tryToFindFullDataSetByCodes(dataSetCodes,
+ externalData.addAll(externalDataDAO.tryToFindFullDataSetsByCodes(dataSetCodes,
withProperties, lockForUpdate));
}
@@ -434,24 +434,21 @@ public final class ExternalDataTable extends AbstractExternalDataBusinessObject
return service.createReportFromDatasets(sessionToken, datastoreServiceKey, locations);
}
- private List<DatasetDescription> loadAvailableDatasetDescriptions(List<String> datasetCodes)
+ private List<DatasetDescription> loadAvailableDatasetDescriptions(List<String> dataSetCodes)
{
IExternalDataDAO externalDataDAO = getExternalDataDAO();
List<DatasetDescription> result = new ArrayList<DatasetDescription>();
List<String> notAvailableDatasets = new ArrayList<String>();
- for (String datasetCode : datasetCodes)
+ List<ExternalDataPE> dataSets =
+ externalDataDAO.tryToFindFullDataSetsByCodes(dataSetCodes, false, false);
+ for (ExternalDataPE dataSet : dataSets)
{
- ExternalDataPE dataSet =
- externalDataDAO.tryToFindFullDataSetByCode(datasetCode, false, false);
- if (dataSet != null)
+ if (dataSet.getStatus().isAvailable())
{
- if (dataSet.getStatus().isAvailable())
- {
- result.add(createDatasetDescription(dataSet));
- } else
- {
- notAvailableDatasets.add(datasetCode);
- }
+ result.add(createDatasetDescription(dataSet));
+ } else
+ {
+ notAvailableDatasets.add(dataSet.getCode());
}
}
throwUnavailableOperationExceptionIfNecessary(notAvailableDatasets);
@@ -547,7 +544,7 @@ public final class ExternalDataTable extends AbstractExternalDataBusinessObject
private int filterByStatusAndUpdate(Map<DataStorePE, List<ExternalDataPE>> datasetsByStore,
DataSetArchivingStatus oldStatus, DataSetArchivingStatus newStatus)
{
- int counter = 0;
+ List<String> codesToUpdate = new ArrayList<String>();
IExternalDataDAO externalDataDAO = getExternalDataDAO();
for (List<ExternalDataPE> dataSets : datasetsByStore.values())
{
@@ -560,14 +557,17 @@ public final class ExternalDataTable extends AbstractExternalDataBusinessObject
iterator.remove();
} else
{
- dataSet.setStatus(newStatus);
- externalDataDAO.validate(dataSet);
- counter++;
+ codesToUpdate.add(dataSet.getCode());
}
}
}
- externalDataDAO.flush();
- return counter;
+ // WORKAROUND In order not to load data set properties at the end of transaction
+ // for Hibernate Search indexing, we don't make change to PE's loaded by Hibernate,
+ // but perform a 'bulk update' operation. Such an operation is quicker and Hibernate
+ // Search doesn't spot the change. The drawback is that the loaded objects are
+ // not updated with a new status.
+ externalDataDAO.updateDataSetStatuses(codesToUpdate, newStatus);
+ return codesToUpdate.size();
}
private interface IArchivingAction
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/IExternalDataBO.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/IExternalDataBO.java
index 28bb7046803..c7440658fa6 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/IExternalDataBO.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/IExternalDataBO.java
@@ -68,11 +68,11 @@ public interface IExternalDataBO extends IEntityBusinessObject
public void update(DataSetUpdatesDTO updates);
/**
- * Updates status of given data set.
+ * Updates status of given data sets.
*
- * @throws UserFailureException if data set does not exist or status couldn't be set.
+ * @throws UserFailureException if a data set does not exist or status couldn't be set.
*/
- public void updateStatus(String dataSetCode, DataSetArchivingStatus newStatus)
+ public void updateStatuses(List<String> dataSetCodes, DataSetArchivingStatus newStatus)
throws UserFailureException;
/**
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/IExternalDataDAO.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/IExternalDataDAO.java
index 00b4cf18765..7e419c56bbd 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/IExternalDataDAO.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/IExternalDataDAO.java
@@ -81,15 +81,15 @@ public interface IExternalDataDAO extends IGenericDAO<ExternalDataPE>
boolean lockForUpdate);
/**
- * Tries to get the full data set for the specified code with optional locking.
+ * Tries to get the full data sets for the specified codes with optional locking.
*/
- public List<ExternalDataPE> tryToFindFullDataSetByCodes(Collection<String> dataSetCodes,
+ public List<ExternalDataPE> tryToFindFullDataSetsByCodes(Collection<String> dataSetCodes,
boolean withPropertyTypes, boolean lockForUpdate);
/**
- * Sets status of dataset with given code.
+ * Sets status of datasets with given codes.
*/
- public void updateDataSetStatus(String dataSetCodes, DataSetArchivingStatus status);
+ public void updateDataSetStatuses(List<String> dataSetCodes, DataSetArchivingStatus status);
/**
* Persists the specified data set.
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/ExternalDataDAO.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/ExternalDataDAO.java
index e6e07ce9b5f..6f845885d8d 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/ExternalDataDAO.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/ExternalDataDAO.java
@@ -16,15 +16,19 @@
package ch.systemsx.cisd.openbis.generic.server.dataaccess.db;
+import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
+import org.apache.commons.lang.builder.ReflectionToStringBuilder;
import org.apache.log4j.Logger;
import org.hibernate.FetchMode;
+import org.hibernate.HibernateException;
import org.hibernate.LockMode;
import org.hibernate.Session;
import org.hibernate.SessionFactory;
@@ -49,6 +53,8 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.CodeConverter;
import ch.systemsx.cisd.openbis.generic.shared.dto.DataPE;
import ch.systemsx.cisd.openbis.generic.shared.dto.DataStorePE;
import ch.systemsx.cisd.openbis.generic.shared.dto.DatabaseInstancePE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
+import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescriptionPE;
import ch.systemsx.cisd.openbis.generic.shared.dto.ExperimentPE;
import ch.systemsx.cisd.openbis.generic.shared.dto.ExternalDataPE;
import ch.systemsx.cisd.openbis.generic.shared.dto.SamplePE;
@@ -203,11 +209,86 @@ final class ExternalDataDAO extends AbstractGenericEntityDAO<ExternalDataPE> imp
return entity;
}
- public List<ExternalDataPE> tryToFindFullDataSetByCodes(Collection<String> dataSetCodes,
+ public List<DatasetDescription> tryToFindDatasetDescriptionsByCodes(List<String> dataSetCodes)
+ {
+ assert dataSetCodes != null : "Unspecified data set code";
+
+ // final Criterion codeEq = Restrictions.eq("code", mangledCode);
+
+ // TODO
+ // Hibernate bug (HHH-2676) - can't take a lock in this kind of query.
+ String queryString =
+ "select new ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription(d.code, d.location, d.sampleInternal.code, "
+ + "d.experimentInternal.projectInternal.group.code, "
+ + "d.experimentInternal.projectInternal.code, "
+ + "d.experimentInternal.code, "
+ + "d.dataSetType.mainDataSetPath, d.dataSetType.mainDataSetPattern, "
+ + "d.experimentInternal.projectInternal.group.databaseInstance.code) "
+ + "from " + TABLE_NAME + " d where d.code IN (:codes)";
+
+ Session session = getHibernateTemplate().getSessionFactory().getCurrentSession();
+ final List<DatasetDescription> list =
+ cast(session.createQuery(queryString).setParameterList("codes", dataSetCodes)
+ .list());
+
+ System.err.println(list.size());
+ for (DatasetDescription desc : list)
+ {
+ System.err.println(ReflectionToStringBuilder.toString(desc));
+ }
+ // if (operationLog.isDebugEnabled())
+ // {
+ // operationLog.debug(String.format("External data '%s' found for data set code '%s'.",
+ // entity, dataSetCode));
+ // }
+ return list;
+ }
+
+ public List<DatasetDescriptionPE> tryToFindDatasetDescriptionsByCodes(List<String> dataSetCodes)
+ {
+ assert dataSetCodes != null : "Unspecified data set code";
+
+ // final Criterion codeEq = Restrictions.eq("code", mangledCode);
+
+ // TODO
+ // Hibernate bug (HHH-2676) - can't take a lock in this kind of query.
+ String queryString =
+ "select new DatasetDescriptionPE(d.code, d.location, d.sampleInternal.code, "
+ + "d.experimentInternal.projectInternal.group.code, "
+ + "d.experimentInternal.projectInternal.code, "
+ + "d.experimentInternal.code, "
+ + "d.dataSetType.mainDataSetPath, d.dataSetType.mainDataSetPattern, "
+ + "d.experimentInternal.projectInternal.group.databaseInstance.code) "
+ + "from " + TABLE_NAME + " d where d.code IN (:codes)";
+
+ Session session = getHibernateTemplate().getSessionFactory().getCurrentSession();
+ final List<DatasetDescriptionPE> list =
+ cast(session.createQuery(queryString).setParameterList("codes", dataSetCodes)
+ .list());
+
+ System.err.println(list.size());
+ for (DatasetDescriptionPE desc : list)
+ {
+ System.err.println(ReflectionToStringBuilder.toString(desc));
+ }
+ // if (operationLog.isDebugEnabled())
+ // {
+ // operationLog.debug(String.format("External data '%s' found for data set code '%s'.",
+ // entity, dataSetCode));
+ // }
+ return list;
+ }
+
+ public List<ExternalDataPE> tryToFindFullDataSetsByCodes(Collection<String> dataSetCodes,
boolean withPropertyTypes, boolean lockForUpdate)
{
assert dataSetCodes != null : "Unspecified collection";
+ if (dataSetCodes.size() == 0)
+ {
+ return Collections.emptyList();
+ }
+
final Criterion codeIn = Restrictions.in("code", dataSetCodes);
final DetachedCriteria criteria = DetachedCriteria.forClass(ENTITY_CLASS);
@@ -271,28 +352,45 @@ final class ExternalDataDAO extends AbstractGenericEntityDAO<ExternalDataPE> imp
return entity;
}
- public void updateDataSetStatus(String dataSetCode, final DataSetArchivingStatus status)
+ public void updateDataSetStatuses(final List<String> dataSetCodes,
+ final DataSetArchivingStatus status)
{
- assert dataSetCode != null : "Unspecified data set code";
+ assert dataSetCodes != null : "Unspecified data set codes";
assert status != null : "Unspecified code";
- final String mangledCode = CodeConverter.tryToDatabase(dataSetCode);
+ if (dataSetCodes.size() == 0)
+ {
+ return;
+ }
final HibernateTemplate hibernateTemplate = getHibernateTemplate();
- // NOTE: 'VERSIONED' makes modification time modified too
- final int updatedRows =
- hibernateTemplate.bulkUpdate("UPDATE VERSIONED " + TABLE_NAME
- + " SET status = ? WHERE code = ? ", toArray(status, mangledCode));
+ int updatedRows = (Integer) hibernateTemplate.execute(new HibernateCallback()
+ {
+
+ //
+ // HibernateCallback
+ //
+
+ public final Object doInHibernate(final Session session) throws HibernateException,
+ SQLException
+ {
+ // NOTE: 'VERSIONED' makes modification time modified too
+ return session.createQuery(
+ "UPDATE VERSIONED " + TABLE_NAME
+ + " SET status = :status WHERE code IN (:codes) ")
+ .setParameter("status", status).setParameterList("codes", dataSetCodes)
+ .executeUpdate();
+ }
+ });
hibernateTemplate.flush();
-
- if (updatedRows == 0)
+ if (updatedRows != dataSetCodes.size())
{
- throw UserFailureException.fromTemplate("Update of dataset's %s status to %s failed.",
- dataSetCode, status);
+ throw UserFailureException.fromTemplate("Update of %s data set statuses to %s failed.",
+ dataSetCodes.size(), status);
} else if (operationLog.isInfoEnabled())
{
- operationLog.info(String.format("UPDATE: external data '%s' status %s.", dataSetCode,
- status));
+ operationLog.info(String.format("UPDATED: %s data set statuses to '%s'.", dataSetCodes
+ .size(), status));
}
}
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/IETLLIMSService.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/IETLLIMSService.java
index 1b12e1e98e0..034ebd44fcd 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/IETLLIMSService.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/IETLLIMSService.java
@@ -114,20 +114,21 @@ public interface IETLLIMSService extends IServer, ISessionProvider
throws UserFailureException;
/**
- * Returns the ExperimentType together with assigned property types for specified experiment type code.
+ * Returns the ExperimentType together with assigned property types for specified experiment
+ * type code.
*/
@Transactional(readOnly = true)
@RolesAllowed(RoleSet.ETL_SERVER)
public ExperimentType getExperimentType(String sessionToken, String experimentTypeCode)
throws UserFailureException;
-
+
/**
* Returns the SampleType together with assigned property types for specified sample type code.
*/
@Transactional(readOnly = true)
@RolesAllowed(RoleSet.ETL_SERVER)
public SampleType getSampleType(String sessionToken, String sampleTypeCode)
- throws UserFailureException;
+ throws UserFailureException;
/**
* Returns the data set type together with assigned property types for specified data set type
@@ -149,7 +150,7 @@ public interface IETLLIMSService extends IServer, ISessionProvider
final String sessionToken,
@AuthorizationGuard(guardClass = ExperimentTechIdPredicate.class) final TechId experimentID)
throws UserFailureException;
-
+
/**
* For given sample {@link TechId} returns the corresponding list of {@link ExternalData}.
*
@@ -219,7 +220,7 @@ public interface IETLLIMSService extends IServer, ISessionProvider
@DatabaseUpdateModification(value = ObjectKind.SAMPLE)
public void updateSample(String sessionToken,
@AuthorizationGuard(guardClass = SampleUpdatesPredicate.class) SampleUpdatesDTO updates);
-
+
/**
* Registers the specified data connected to a sample.
*
@@ -341,13 +342,13 @@ public interface IETLLIMSService extends IServer, ISessionProvider
throws UserFailureException;
/**
- * Updates status of given data set.
+ * Updates status of given data sets.
*/
@Transactional
@RolesAllowed(RoleSet.ETL_SERVER)
@DatabaseUpdateModification(value = ObjectKind.DATA_SET)
- public void updateDataSetStatus(String sessionToken,
- @AuthorizationGuard(guardClass = DataSetCodePredicate.class) String dataSetCode,
+ public void updateDataSetStatuses(String sessionToken,
+ @AuthorizationGuard(guardClass = DataSetCodePredicate.class) List<String> dataSetCodes,
final DataSetArchivingStatus newStatus) throws UserFailureException;
/**
diff --git a/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ExternalDataBOTest.java b/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ExternalDataBOTest.java
index f565024e9f1..6c00697d1fa 100644
--- a/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ExternalDataBOTest.java
+++ b/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ExternalDataBOTest.java
@@ -19,9 +19,11 @@ package ch.systemsx.cisd.openbis.generic.server.business.bo;
import static ch.systemsx.cisd.openbis.generic.server.business.ManagerTestTool.EXAMPLE_SESSION;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
+import java.util.List;
import org.hamcrest.BaseMatcher;
import org.hamcrest.Description;
@@ -286,21 +288,20 @@ public class ExternalDataBOTest extends AbstractBOTest
}
@Test
- public void testUpdateStatus()
+ public void testUpdateStatuses()
{
- final SamplePE sample = new SamplePE();
- sample.setCode(SAMPLE_IDENTIFIER.getSampleCode());
- final ExternalDataPE dataSet = createDataSet(sample, null);
+ final List<String> codes = Arrays.asList(new String[]
+ { "CODE-1", "CODE-2" });
context.checking(new Expectations()
{
{
- one(externalDataDAO).updateDataSetStatus(dataSet.getCode(),
+ one(externalDataDAO).updateDataSetStatuses(codes,
DataSetArchivingStatus.ARCHIVED);
}
});
IExternalDataBO dataBO = createExternalDataBO();
- dataBO.updateStatus(dataSet.getCode(), DataSetArchivingStatus.ARCHIVED);
+ dataBO.updateStatuses(codes, DataSetArchivingStatus.ARCHIVED);
// TODO DAO test
context.assertIsSatisfied();
}
diff --git a/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ExternalDataTableTest.java b/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ExternalDataTableTest.java
index ac8be65ecf6..732285f6f44 100644
--- a/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ExternalDataTableTest.java
+++ b/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ExternalDataTableTest.java
@@ -234,7 +234,7 @@ public final class ExternalDataTableTest extends AbstractBOTest
context.checking(new Expectations()
{
{
- one(externalDataDAO).tryToFindFullDataSetByCodes(
+ one(externalDataDAO).tryToFindFullDataSetsByCodes(
Code.extractCodes(Arrays.asList(searched)), withProperties,
lockForUpdate);
will(returnValue(Arrays.asList(results)));
--
GitLab