SSDM-4858 : Data set synchronization does not give a summary of updated data sets

SVN: 37871

datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/DataSetRegistrationIngestionService.java

@@ -52,13 +52,13 @@ import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
 import ch.systemsx.cisd.openbis.generic.shared.basic.dto.TableModel;
 import ch.systemsx.cisd.openbis.generic.shared.dto.NewExternalData;
 import ch.systemsx.cisd.openbis.generic.shared.dto.NewProperty;
+import ch.systemsx.cisd.openbis.generic.shared.util.IRowBuilder;
+import ch.systemsx.cisd.openbis.generic.shared.util.SimpleTableModelBuilder;
 
 class DataSetRegistrationIngestionService extends IngestionService<DataSetInformation>
 {
     private static final long serialVersionUID = 1L;
-
-    private List<String> notSyncedDataSetCodes;
-
+ 
     private final NewExternalData dataSet;
 
     private final String loginUser;
@@ -71,20 +71,16 @@ class DataSetRegistrationIngestionService extends IngestionService<DataSetInform
 
     private final String harvesterTempDir;
 
-    private final Logger log;
-
-    public DataSetRegistrationIngestionService(Properties properties, File storeRoot, List<String> notSyncedDataSetCodes, NewExternalData ds,
+    public DataSetRegistrationIngestionService(Properties properties, File storeRoot, NewExternalData ds,
             Logger operationLog)
     {
         super(properties, storeRoot);
-        this.notSyncedDataSetCodes = notSyncedDataSetCodes;
         this.dataSet = ds;
         this.loginUser = properties.getProperty("user");
         this.loginPass = properties.getProperty("pass");
         this.asUrl = properties.getProperty("as-url");
         this.dssUrl = properties.getProperty("dss-url");
         this.harvesterTempDir = properties.getProperty("harvester-temp-dir");
-        this.log = operationLog;
     }
 
     @Override
@@ -134,6 +130,7 @@ class DataSetRegistrationIngestionService extends IngestionService<DataSetInform
             {
                 transaction.moveFile(f.getAbsolutePath(), ds);
             }
+            return summaryTableModel(parameters, "Added");
         }
         else
         {
@@ -153,8 +150,18 @@ class DataSetRegistrationIngestionService extends IngestionService<DataSetInform
             {
                 dataSetForUpdate.setPropertyValue(propCode, "");
             }
+            return summaryTableModel(parameters, "Updated");
         }
-        return null;
+    }
+
+    private TableModel summaryTableModel(Map<String, Object> parameters, String summary)
+    {
+        SimpleTableModelBuilder builder = new SimpleTableModelBuilder(true);
+        builder.addHeader("Parameters");
+        builder.addHeader(summary);
+        IRowBuilder row = builder.addRow();
+        row.setCell("Parameters", parameters.toString());
+        return builder.getTableModel();
     }
 
     private Set<String> extractPropertyNames(List<NewProperty> dataSetProperties)

datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/DataSetRegistrationSummary.java

+/*
+ * Copyright 2017 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * 
+ *
+ * @author Ganime Betul Akin
+ */
+class DataSetRegistrationSummary
+{
+    final List<String> notRegisteredDataSetCodes = Collections.synchronizedList(new ArrayList<String>());
+
+    final AtomicInteger addedDsCount = new AtomicInteger(0);
+
+    final AtomicInteger updatedDsCount = new AtomicInteger(0);
+}

datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java

@@ -221,15 +221,18 @@ public class EntitySynchronizer
         Map<String, DataSetWithConnections> physicalDSMap =
                 data.filterPhysicalDataSetsByLastModificationDate(lastSyncTimestamp, dataSetsCodesToRetry);
         operationLog.info("Registering data sets...");
-        List<String> notRegisteredDataSets = registerPhysicalDataSets(physicalDSMap);
-        operationLog.info((physicalDSMap.keySet().size() - notRegisteredDataSets.size()) + " data set(s) have been successfully registered. "
-                + notRegisteredDataSets.size()
+        DataSetRegistrationSummary dsRegistrationSummary = registerPhysicalDataSets(physicalDSMap);
+        operationLog.info("Data set registration summary:\n" + dsRegistrationSummary.addedDsCount + " data set(s) were added.\n"
+                + dsRegistrationSummary.updatedDsCount
+                + " data set(s) were updated.\n"
+                + dsRegistrationSummary.notRegisteredDataSetCodes.size()
                 + " data set(s) FAILED to register ");
 
         // link physical data sets registered above to container data sets
         // and set parent/child relationships
+        operationLog.info("\n");
         operationLog.info("start linking/un-linking container and component data sets");
-        establishDataSetRelationships(data.getDataSetsToProcess(), notRegisteredDataSets, physicalDSMap);
+        establishDataSetRelationships(data.getDataSetsToProcess(), dsRegistrationSummary.notRegisteredDataSetCodes, physicalDSMap);
 
         // cleanup();
 
@@ -389,19 +392,20 @@ public class EntitySynchronizer
         return false;
     }
 
-    private List<String> registerPhysicalDataSets(Map<String, DataSetWithConnections> physicalDSMap) throws IOException
+    private DataSetRegistrationSummary registerPhysicalDataSets(Map<String, DataSetWithConnections> physicalDSMap) throws IOException
     {
         List<DataSetWithConnections> dsList = new ArrayList<DataSetWithConnections>(physicalDSMap.values());
-        List<String> notRegisteredDataSetCodes = Collections.synchronizedList(new ArrayList<String>());
+        DataSetRegistrationSummary dsRegistrationSummary = new DataSetRegistrationSummary();
 
         // This parallelization is possible because each DS is registered without dependencies
         // and the dependencies are established later on in the sync process.
-        ParallelizedExecutor.process(dsList, new DataSetRegistrationTaskExecutor(notRegisteredDataSetCodes), 0.5, 10, "register data sets", 0, false);
+        ParallelizedExecutor.process(dsList, new DataSetRegistrationTaskExecutor(dsRegistrationSummary),
+                0.5, 10, "register data sets", 0, false);
 
         // backup the current not synced data set codes file, delete the original file
-        saveNotSyncedDataSetsFile(notRegisteredDataSetCodes);
+        saveNotSyncedDataSetsFile(dsRegistrationSummary.notRegisteredDataSetCodes);
 
-        return notRegisteredDataSetCodes;
+        return dsRegistrationSummary;
     }
 
     private void saveNotSyncedDataSetsFile(List<String> notRegisteredDataSetCodes) throws IOException
@@ -1002,12 +1006,11 @@ public class EntitySynchronizer
 
     private final class DataSetRegistrationTaskExecutor implements ITaskExecutor<DataSetWithConnections>
     {
+        private DataSetRegistrationSummary dsRegistrationSummary;
 
-        private List<String> notRegisteredDataSetCodes;
-
-        public DataSetRegistrationTaskExecutor(List<String> notRegisteredDataSetCodes)
+        public DataSetRegistrationTaskExecutor(DataSetRegistrationSummary dsRegSummary)
         {
-            this.notRegisteredDataSetCodes = notRegisteredDataSetCodes;
+            this.dsRegistrationSummary = dsRegSummary;
         }
 
         @Override
@@ -1018,7 +1021,7 @@ public class EntitySynchronizer
             Properties props = setProperties();
 
             DataSetRegistrationIngestionService ingestionService =
-                    new DataSetRegistrationIngestionService(props, storeRoot, notRegisteredDataSetCodes, dataSet.getDataSet(), operationLog);
+                    new DataSetRegistrationIngestionService(props, storeRoot, dataSet.getDataSet(), operationLog);
             TableModel resultTable = ingestionService.createAggregationReport(new HashMap<String, Object>(), context);
             if (resultTable != null)
             {
@@ -1029,10 +1032,18 @@ public class EntitySynchronizer
                     if (headers.get(i).getTitle().startsWith("Error"))
                     {
                         String message = resultTable.getRows().get(0).getValues().toString();
-                        notRegisteredDataSetCodes.add(dataSet.getDataSet().getCode());
+                        dsRegistrationSummary.notRegisteredDataSetCodes.add(dataSet.getDataSet().getCode());
                         operationLog.error(message);
                         return Status.createError(message);
                     }
+                    else if (headers.get(i).getTitle().startsWith("Added"))
+                    {
+                        dsRegistrationSummary.addedDsCount.getAndIncrement();
+                    }
+                    else if (headers.get(i).getTitle().startsWith("Updated"))
+                    {
+                        dsRegistrationSummary.updatedDsCount.getAndIncrement();
+                    }
                 }
             }
             return Status.OK;