SSDM-4402 : Read not-synced-datasets and changes to writing the file.

SVN: 37434

datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/DataSetRegistrationTask.java

@@ -31,9 +31,11 @@ import java.text.DateFormat;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Date;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Properties;
+import java.util.Set;
 
 import javax.activation.DataHandler;
 import javax.activation.DataSource;
@@ -67,7 +69,7 @@ public class DataSetRegistrationTask<T extends DataSetInformation> implements IM
     protected static final Logger operationLog =
             LogFactory.getLogger(LogCategory.OPERATION, DataSetRegistrationTask.class);
 
-    final DateFormat formater = new SimpleDateFormat("dd-MM-yy HH-mm-ss", Locale.ENGLISH);
+    final DateFormat formatter = new SimpleDateFormat("dd-MM-yy HH-mm-ss", Locale.ENGLISH);
 
     private static final String HARVESTER_CONFIG_FILE_PROPERTY_NAME = "harvester-config-file";
 
@@ -85,10 +87,6 @@ public class DataSetRegistrationTask<T extends DataSetInformation> implements IM
 
     private IMailClient mailClient;
 
-    File lastSyncTimestampFile;
-
-    File newLastSyncTimeStampFile;
-
     private String dataStoreCode;
 
     @Override
@@ -141,34 +139,21 @@ public class DataSetRegistrationTask<T extends DataSetInformation> implements IM
                         .info("Start synchronization from data source: " + config.getDataSourceOpenbisURL() + " for user " + config.getUser());
 
                 String fileName = config.getLastSyncTimestampFileName();
-                lastSyncTimestampFile = new File(fileName);
-                newLastSyncTimeStampFile = new File(fileName + ".new");
-
-                if (lastSyncTimestampFile.exists())
-                {
-                    String timeStr = FileUtilities.loadToString(lastSyncTimestampFile).trim();
-                    try
-                    {
-                        lastSyncTimestamp = formater.parse(timeStr);
-                    } catch (ParseException e)
-                    {
-                        operationLog.error("Cannot parse value as time:" + timeStr);
-                        return;
-                    }
-                }
-                else
-                {
-                    lastSyncTimestamp = new Date(0L);
-                }
+                File lastSyncTimestampFile = new File(fileName);
+                lastSyncTimestamp = getLastSyncTimeStamp(lastSyncTimestampFile);
                 // save the current time into a temp file as last sync time
-                FileUtilities.writeToFile(newLastSyncTimeStampFile, formater.format(new Date()));
+                File newLastSyncTimeStampFile = new File(fileName + ".new");
+                FileUtilities.writeToFile(newLastSyncTimeStampFile, formatter.format(new Date()));
+
+                Set<String> notSyncedDataSetCodes = getNotSyncedDataSetCodes(config.getNotSyncedDataSetsFileName());
 
                 EntitySynchronizer synchronizer =
-                        new EntitySynchronizer(service, dataStoreCode, storeRoot, lastSyncTimestamp, context, config, operationLog);
+                        new EntitySynchronizer(service, dataStoreCode, storeRoot, lastSyncTimestamp, notSyncedDataSetCodes, context, config,
+                                operationLog);
                 synchronizer.syncronizeEntities();
 
                 operationLog.info("Saving the timestamp of sync start to file");
-                saveSyncTimestamp();
+                saveSyncTimestamp(newLastSyncTimeStampFile, lastSyncTimestampFile);
 
                 operationLog.info(this.getClass() + " finished executing.");
 
@@ -180,6 +165,32 @@ public class DataSetRegistrationTask<T extends DataSetInformation> implements IM
         }
     }
 
+    private Date getLastSyncTimeStamp(File lastSyncTimestampFile) throws ParseException
+    {
+        if (lastSyncTimestampFile.exists())
+        {
+            String timeStr = FileUtilities.loadToString(lastSyncTimestampFile).trim();
+            return formatter.parse(timeStr);
+        }
+        else
+        {
+            return new Date(0L);
+        }
+    }
+
+    private Set<String> getNotSyncedDataSetCodes(String fileName)
+    {
+        File notSyncedDataSetsFile = new File(fileName);
+        if (notSyncedDataSetsFile.exists())
+        {
+            List<String> list = FileUtilities.loadToStringList(notSyncedDataSetsFile);
+            return new LinkedHashSet<String>(list);
+        }
+        else
+        {
+            return new LinkedHashSet<String>();
+        }
+    }
     private void sendErrorEmail(SyncConfig config, String subject)
     {
         if (config.getLogFilePath() != null)
@@ -215,7 +226,7 @@ public class DataSetRegistrationTask<T extends DataSetInformation> implements IM
         }
     }
 
-    private void saveSyncTimestamp()
+    private void saveSyncTimestamp(File newLastSyncTimeStampFile, File lastSyncTimestampFile)
     {
         newLastSyncTimeStampFile.renameTo(lastSyncTimestampFile);
     }

datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/EntitySynchronizer.java

@@ -40,6 +40,7 @@ import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.xpath.XPathExpressionException;
 
 import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.log4j.Logger;
 import org.w3c.dom.Document;
@@ -143,18 +144,21 @@ public class EntitySynchronizer
 
     private final Date lastSyncTimestamp;
 
+    private final Set<String> dataSetsCodesToRetry;
+
     private final SyncConfig config;
 
     private final Logger operationLog;
 
     public EntitySynchronizer(IEncapsulatedOpenBISService service, String dataStoreCode, File storeRoot, Date lastSyncTimestamp,
-            DataSetProcessingContext context,
+            Set<String> dataSetsCodesToRetry, DataSetProcessingContext context,
             SyncConfig config, Logger operationLog)
     {
         this.service = service;
         this.dataStoreCode = dataStoreCode;
         this.storeRoot = storeRoot;
         this.lastSyncTimestamp = lastSyncTimestamp;
+        this.dataSetsCodesToRetry = dataSetsCodesToRetry;
         this.context = context;
         this.config = config;
         this.operationLog = operationLog;
@@ -340,7 +344,7 @@ public class EntitySynchronizer
         operationLog.info("entity operation result: " + operationResult);
     }
 
-    private void registerPhysicalDataSets(Map<String, DataSetWithConnections> physicalDSMap)
+    private void registerPhysicalDataSets(Map<String, DataSetWithConnections> physicalDSMap) throws IOException
     {
         List<DataSetWithConnections> dsList = new ArrayList<DataSetWithConnections>(physicalDSMap.values());
         List<String> notSyncedDataSetCodes = Collections.synchronizedList(new ArrayList<String>());
@@ -350,7 +354,12 @@ public class EntitySynchronizer
         // parallelized
         ParallelizedExecutor.process(dsList, new DataSetRegistrationTaskExecutor(notSyncedDataSetCodes), 0.5, 10, "register data sets", 0, false);
 
+        // backup the current not synced data set codes file, delete the original file
         File notSyncedDataSetsFile = new File(config.getNotSyncedDataSetsFileName());
+        File backupLastSyncTimeStampFile = new File(config.getNotSyncedDataSetsFileName() + ".bk");
+        FileUtils.copyFile(notSyncedDataSetsFile, backupLastSyncTimeStampFile);
+        FileUtilities.delete(notSyncedDataSetsFile);
+
         for (String dsCode : notSyncedDataSetCodes)
         {
             FileUtilities.appendToFile(notSyncedDataSetsFile, dsCode, true);