From c769b1d1d76de819dc62ab53b9c855d67bd999db Mon Sep 17 00:00:00 2001
From: felmer <franz-josef.elmer@id.ethz.ch>
Date: Mon, 26 Nov 2018 14:01:26 +0100
Subject: [PATCH] SSDM-7430: Data set registration: logging improved and time
out increased
---
.../synchronizer/EntitySynchronizer.java | 1 -
.../AttachmentSynchronizationTaskExecutor.java | 1 -
.../DataSetRegistrationIngestionService.java | 16 ++++++++++++----
.../DataSetRegistrationTaskExecutor.java | 5 +++--
.../harvester/synchronizer/util/V3Utils.java | 6 ++----
5 files changed, 17 insertions(+), 12 deletions(-)
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
index 9e8d3e6725c..67cc8d4a33e 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
@@ -517,7 +517,6 @@ public class EntitySynchronizer
ParallelizedExecutionPreferences preferences = config.getParallelizedExecutionPrefs();
String asUrl = config.getDataSourceOpenbisURL();
String dssUrl = config.getDataSourceDSSURL();
- monitor.log("AS URL: " + asUrl + ", DSS URL: " + dssUrl);
V3Utils v3Utils = V3Utils.create(asUrl, dssUrl);
monitor.log("Services for accessing data source established");
ParallelizedExecutor.process(attachmentHoldersToProcess, new AttachmentSynchronizationTaskExecutor(synchronizationSummary,
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/parallelizedExecutor/AttachmentSynchronizationTaskExecutor.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/parallelizedExecutor/AttachmentSynchronizationTaskExecutor.java
index 1a521dd72e7..55cdf712852 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/parallelizedExecutor/AttachmentSynchronizationTaskExecutor.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/parallelizedExecutor/AttachmentSynchronizationTaskExecutor.java
@@ -97,7 +97,6 @@ public final class AttachmentSynchronizationTaskExecutor implements ITaskExecuto
TechId techId = null;
ICommonServer commonServer = ServiceFinderUtils.getCommonServer(ServiceProvider.getConfigProvider().getOpenBisServerUrl());
String localSessionToken = ServiceFinderUtils.login(commonServer, config.getHarvesterUser(), config.getHarvesterPass());
- monitor.log("Logged in: " + localSessionToken);
IAttachmentsOperationsHandler attachmentsOperationsHandler = null;
if (item.getEntityKind() == SyncEntityKind.EXPERIMENT)
{
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/parallelizedExecutor/DataSetRegistrationIngestionService.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/parallelizedExecutor/DataSetRegistrationIngestionService.java
index 4054d1e7f7a..40ad671c47f 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/parallelizedExecutor/DataSetRegistrationIngestionService.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/parallelizedExecutor/DataSetRegistrationIngestionService.java
@@ -42,6 +42,8 @@ import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.id.IDataSetFileId;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.util.DSPropertyUtils;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.util.V3Utils;
import ch.systemsx.cisd.common.io.IOUtilities;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.etlserver.registrator.api.v2.IDataSet;
import ch.systemsx.cisd.etlserver.registrator.api.v2.IDataSetRegistrationTransactionV2;
import ch.systemsx.cisd.etlserver.registrator.api.v2.IDataSetUpdatable;
@@ -59,6 +61,9 @@ import ch.systemsx.cisd.openbis.generic.shared.util.SimpleTableModelBuilder;
class DataSetRegistrationIngestionService extends IngestionService<DataSetInformation>
{
private static final long serialVersionUID = 1L;
+
+ private static final Logger operationLog =
+ LogFactory.getLogger(LogCategory.OPERATION, DataSetRegistrationIngestionService.class);
private final NewExternalData dataSet;
@@ -158,6 +163,13 @@ class DataSetRegistrationIngestionService extends IngestionService<DataSetInform
}
}
+ @Override
+ protected TableModel errorTableModel(Map<String, Object> parameters, Throwable e)
+ {
+ operationLog.error("Error occurred while registering data set " + dataSet.getCode(), e);
+ return super.errorTableModel(parameters, e);
+ }
+
private TableModel summaryTableModel(Map<String, Object> parameters, String summary)
{
SimpleTableModelBuilder builder = new SimpleTableModelBuilder(true);
@@ -230,10 +242,6 @@ class DataSetRegistrationIngestionService extends IngestionService<DataSetInform
{
DataSetFilePermId filePermId = orgFile.getPermId();
FileDetails fileDetails = fileDetailsMap.get(filePermId);
-
- // System.out.println("Downloaded " + orgFile.getPath() + " "
- // + MemorySizeFormatter.format(orgFile.getFileLength()));
-
Path path = Paths.get(dir.getAbsolutePath(), filePath);
InputStream inputStream = fileDownload.getInputStream();
OutputStream outputStream = Files.newOutputStream(path);
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/parallelizedExecutor/DataSetRegistrationTaskExecutor.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/parallelizedExecutor/DataSetRegistrationTaskExecutor.java
index 7b610a96363..7aa6f51faf0 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/parallelizedExecutor/DataSetRegistrationTaskExecutor.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/parallelizedExecutor/DataSetRegistrationTaskExecutor.java
@@ -18,7 +18,6 @@ package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchroniz
import java.io.File;
import java.util.HashMap;
-import java.util.List;
import java.util.Properties;
import org.apache.log4j.Logger;
@@ -74,7 +73,9 @@ public final class DataSetRegistrationTaskExecutor implements ITaskExecutor<Inco
{
String message = resultTable.getRows().get(0).getValues().toString();
dsRegistrationSummary.notRegisteredDataSetCodes.add(dataSet.getDataSet().getCode());
- operationLog.error(message);
+ operationLog.error("Registration for data set " + dataSet.getDataSet().getCode() + " failed: "
+ + message + ", exp: " + dataSet.getDataSet().getExperimentIdentifierOrNull()
+ + ", sample:" + dataSet.getDataSet().getSampleIdentifierOrNull());
return Status.createError(message);
}
else if (header.getTitle().startsWith("Added"))
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/util/V3Utils.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/util/V3Utils.java
index 5b6c6178963..3c5529972eb 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/util/V3Utils.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/util/V3Utils.java
@@ -41,8 +41,6 @@ import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.download.DataSetFil
import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.fetchoptions.DataSetFileFetchOptions;
import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.id.IDataSetFileId;
import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.search.DataSetFileSearchCriteria;
-import ch.systemsx.cisd.common.spring.HttpInvokerUtils;
-import ch.systemsx.cisd.common.ssl.SslCertificateHelper;
import ch.systemsx.cisd.openbis.dss.generic.server.EncapsulatedOpenBISService;
/**
@@ -52,7 +50,7 @@ import ch.systemsx.cisd.openbis.dss.generic.server.EncapsulatedOpenBISService;
*/
public class V3Utils
{
- public static final int TIMEOUT = 100000;
+ public static final long TIMEOUT = 6 * DateUtils.MILLIS_PER_HOUR;
private final IDataStoreServerApi dss;
private final IApplicationServerApi as;
@@ -62,7 +60,7 @@ public class V3Utils
return new V3Utils(asUrl, dssUrl, TIMEOUT);
}
- private V3Utils (String asUrl, String dssUrl, int timeout)
+ private V3Utils (String asUrl, String dssUrl, long timeout)
{
String timeoutInMinutes = Long.toString(timeout / DateUtils.MILLIS_PER_MINUTE);
this.as = EncapsulatedOpenBISService.createOpenBisV3Service(asUrl, timeoutInMinutes);
--
GitLab