diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/DataSetRegistrationIngestionService.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/DataSetRegistrationIngestionService.java
index 83379ae11e430a30c5623544f3ffc5c13e81e613..a15e7bc74ca2eab9f98c61383eece1d456d46cb8 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/DataSetRegistrationIngestionService.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/DataSetRegistrationIngestionService.java
@@ -52,13 +52,13 @@ import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.TableModel;
import ch.systemsx.cisd.openbis.generic.shared.dto.NewExternalData;
import ch.systemsx.cisd.openbis.generic.shared.dto.NewProperty;
+import ch.systemsx.cisd.openbis.generic.shared.util.IRowBuilder;
+import ch.systemsx.cisd.openbis.generic.shared.util.SimpleTableModelBuilder;
class DataSetRegistrationIngestionService extends IngestionService<DataSetInformation>
{
private static final long serialVersionUID = 1L;
-
- private List<String> notSyncedDataSetCodes;
-
+
private final NewExternalData dataSet;
private final String loginUser;
@@ -71,20 +71,16 @@ class DataSetRegistrationIngestionService extends IngestionService<DataSetInform
private final String harvesterTempDir;
- private final Logger log;
-
- public DataSetRegistrationIngestionService(Properties properties, File storeRoot, List<String> notSyncedDataSetCodes, NewExternalData ds,
+ public DataSetRegistrationIngestionService(Properties properties, File storeRoot, NewExternalData ds,
Logger operationLog)
{
super(properties, storeRoot);
- this.notSyncedDataSetCodes = notSyncedDataSetCodes;
this.dataSet = ds;
this.loginUser = properties.getProperty("user");
this.loginPass = properties.getProperty("pass");
this.asUrl = properties.getProperty("as-url");
this.dssUrl = properties.getProperty("dss-url");
this.harvesterTempDir = properties.getProperty("harvester-temp-dir");
- this.log = operationLog;
}
@Override
@@ -134,6 +130,7 @@ class DataSetRegistrationIngestionService extends IngestionService<DataSetInform
{
transaction.moveFile(f.getAbsolutePath(), ds);
}
+ return summaryTableModel(parameters, "Added");
}
else
{
@@ -153,8 +150,18 @@ class DataSetRegistrationIngestionService extends IngestionService<DataSetInform
{
dataSetForUpdate.setPropertyValue(propCode, "");
}
+ return summaryTableModel(parameters, "Updated");
}
- return null;
+ }
+
+ private TableModel summaryTableModel(Map<String, Object> parameters, String summary)
+ {
+ SimpleTableModelBuilder builder = new SimpleTableModelBuilder(true);
+ builder.addHeader("Parameters");
+ builder.addHeader(summary);
+ IRowBuilder row = builder.addRow();
+ row.setCell("Parameters", parameters.toString());
+ return builder.getTableModel();
}
private Set<String> extractPropertyNames(List<NewProperty> dataSetProperties)
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/DataSetRegistrationSummary.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/DataSetRegistrationSummary.java
new file mode 100644
index 0000000000000000000000000000000000000000..a39234a489be41acd7e8ed82ac79c5c3bf8d9fb8
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/DataSetRegistrationSummary.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2017 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ *
+ *
+ * @author Ganime Betul Akin
+ */
+class DataSetRegistrationSummary
+{
+ final List<String> notRegisteredDataSetCodes = Collections.synchronizedList(new ArrayList<String>());
+
+ final AtomicInteger addedDsCount = new AtomicInteger(0);
+
+ final AtomicInteger updatedDsCount = new AtomicInteger(0);
+}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
index 017d65e8c62e105986734e479e733359b2f787c6..43c5645c01f3633b3b68eb77d717c00ca960c603 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
@@ -221,15 +221,18 @@ public class EntitySynchronizer
Map<String, DataSetWithConnections> physicalDSMap =
data.filterPhysicalDataSetsByLastModificationDate(lastSyncTimestamp, dataSetsCodesToRetry);
operationLog.info("Registering data sets...");
- List<String> notRegisteredDataSets = registerPhysicalDataSets(physicalDSMap);
- operationLog.info((physicalDSMap.keySet().size() - notRegisteredDataSets.size()) + " data set(s) have been successfully registered. "
- + notRegisteredDataSets.size()
+ DataSetRegistrationSummary dsRegistrationSummary = registerPhysicalDataSets(physicalDSMap);
+ operationLog.info("Data set registration summary:\n" + dsRegistrationSummary.addedDsCount + " data set(s) were added.\n"
+ + dsRegistrationSummary.updatedDsCount
+ + " data set(s) were updated.\n"
+ + dsRegistrationSummary.notRegisteredDataSetCodes.size()
+ " data set(s) FAILED to register ");
// link physical data sets registered above to container data sets
// and set parent/child relationships
+ operationLog.info("\n");
operationLog.info("start linking/un-linking container and component data sets");
- establishDataSetRelationships(data.getDataSetsToProcess(), notRegisteredDataSets, physicalDSMap);
+ establishDataSetRelationships(data.getDataSetsToProcess(), dsRegistrationSummary.notRegisteredDataSetCodes, physicalDSMap);
// cleanup();
@@ -389,19 +392,20 @@ public class EntitySynchronizer
return false;
}
- private List<String> registerPhysicalDataSets(Map<String, DataSetWithConnections> physicalDSMap) throws IOException
+ private DataSetRegistrationSummary registerPhysicalDataSets(Map<String, DataSetWithConnections> physicalDSMap) throws IOException
{
List<DataSetWithConnections> dsList = new ArrayList<DataSetWithConnections>(physicalDSMap.values());
- List<String> notRegisteredDataSetCodes = Collections.synchronizedList(new ArrayList<String>());
+ DataSetRegistrationSummary dsRegistrationSummary = new DataSetRegistrationSummary();
// This parallelization is possible because each DS is registered without dependencies
// and the dependencies are established later on in the sync process.
- ParallelizedExecutor.process(dsList, new DataSetRegistrationTaskExecutor(notRegisteredDataSetCodes), 0.5, 10, "register data sets", 0, false);
+ ParallelizedExecutor.process(dsList, new DataSetRegistrationTaskExecutor(dsRegistrationSummary),
+ 0.5, 10, "register data sets", 0, false);
// backup the current not synced data set codes file, delete the original file
- saveNotSyncedDataSetsFile(notRegisteredDataSetCodes);
+ saveNotSyncedDataSetsFile(dsRegistrationSummary.notRegisteredDataSetCodes);
- return notRegisteredDataSetCodes;
+ return dsRegistrationSummary;
}
private void saveNotSyncedDataSetsFile(List<String> notRegisteredDataSetCodes) throws IOException
@@ -1002,12 +1006,11 @@ public class EntitySynchronizer
private final class DataSetRegistrationTaskExecutor implements ITaskExecutor<DataSetWithConnections>
{
+ private DataSetRegistrationSummary dsRegistrationSummary;
- private List<String> notRegisteredDataSetCodes;
-
- public DataSetRegistrationTaskExecutor(List<String> notRegisteredDataSetCodes)
+ public DataSetRegistrationTaskExecutor(DataSetRegistrationSummary dsRegSummary)
{
- this.notRegisteredDataSetCodes = notRegisteredDataSetCodes;
+ this.dsRegistrationSummary = dsRegSummary;
}
@Override
@@ -1018,7 +1021,7 @@ public class EntitySynchronizer
Properties props = setProperties();
DataSetRegistrationIngestionService ingestionService =
- new DataSetRegistrationIngestionService(props, storeRoot, notRegisteredDataSetCodes, dataSet.getDataSet(), operationLog);
+ new DataSetRegistrationIngestionService(props, storeRoot, dataSet.getDataSet(), operationLog);
TableModel resultTable = ingestionService.createAggregationReport(new HashMap<String, Object>(), context);
if (resultTable != null)
{
@@ -1029,10 +1032,18 @@ public class EntitySynchronizer
if (headers.get(i).getTitle().startsWith("Error"))
{
String message = resultTable.getRows().get(0).getValues().toString();
- notRegisteredDataSetCodes.add(dataSet.getDataSet().getCode());
+ dsRegistrationSummary.notRegisteredDataSetCodes.add(dataSet.getDataSet().getCode());
operationLog.error(message);
return Status.createError(message);
}
+ else if (headers.get(i).getTitle().startsWith("Added"))
+ {
+ dsRegistrationSummary.addedDsCount.getAndIncrement();
+ }
+ else if (headers.get(i).getTitle().startsWith("Updated"))
+ {
+ dsRegistrationSummary.updatedDsCount.getAndIncrement();
+ }
}
}
return Status.OK;