diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/DataSetRegistrationTask.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/DataSetRegistrationTask.java
index 5a9e34c0e75348fb59c0c619a9fc27dbbe27fafe..869b06a73306821b0a3e9fb1a39137ad6a449c8d 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/DataSetRegistrationTask.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/DataSetRegistrationTask.java
@@ -31,9 +31,11 @@ import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Properties;
+import java.util.Set;
import javax.activation.DataHandler;
import javax.activation.DataSource;
@@ -67,7 +69,7 @@ public class DataSetRegistrationTask<T extends DataSetInformation> implements IM
protected static final Logger operationLog =
LogFactory.getLogger(LogCategory.OPERATION, DataSetRegistrationTask.class);
- final DateFormat formater = new SimpleDateFormat("dd-MM-yy HH-mm-ss", Locale.ENGLISH);
+ final DateFormat formatter = new SimpleDateFormat("dd-MM-yy HH-mm-ss", Locale.ENGLISH);
private static final String HARVESTER_CONFIG_FILE_PROPERTY_NAME = "harvester-config-file";
@@ -85,10 +87,6 @@ public class DataSetRegistrationTask<T extends DataSetInformation> implements IM
private IMailClient mailClient;
- File lastSyncTimestampFile;
-
- File newLastSyncTimeStampFile;
-
private String dataStoreCode;
@Override
@@ -141,34 +139,21 @@ public class DataSetRegistrationTask<T extends DataSetInformation> implements IM
.info("Start synchronization from data source: " + config.getDataSourceOpenbisURL() + " for user " + config.getUser());
String fileName = config.getLastSyncTimestampFileName();
- lastSyncTimestampFile = new File(fileName);
- newLastSyncTimeStampFile = new File(fileName + ".new");
-
- if (lastSyncTimestampFile.exists())
- {
- String timeStr = FileUtilities.loadToString(lastSyncTimestampFile).trim();
- try
- {
- lastSyncTimestamp = formater.parse(timeStr);
- } catch (ParseException e)
- {
- operationLog.error("Cannot parse value as time:" + timeStr);
- return;
- }
- }
- else
- {
- lastSyncTimestamp = new Date(0L);
- }
+ File lastSyncTimestampFile = new File(fileName);
+ lastSyncTimestamp = getLastSyncTimeStamp(lastSyncTimestampFile);
// save the current time into a temp file as last sync time
- FileUtilities.writeToFile(newLastSyncTimeStampFile, formater.format(new Date()));
+ File newLastSyncTimeStampFile = new File(fileName + ".new");
+ FileUtilities.writeToFile(newLastSyncTimeStampFile, formatter.format(new Date()));
+
+ Set<String> notSyncedDataSetCodes = getNotSyncedDataSetCodes(config.getNotSyncedDataSetsFileName());
EntitySynchronizer synchronizer =
- new EntitySynchronizer(service, dataStoreCode, storeRoot, lastSyncTimestamp, context, config, operationLog);
+ new EntitySynchronizer(service, dataStoreCode, storeRoot, lastSyncTimestamp, notSyncedDataSetCodes, context, config,
+ operationLog);
synchronizer.syncronizeEntities();
operationLog.info("Saving the timestamp of sync start to file");
- saveSyncTimestamp();
+ saveSyncTimestamp(newLastSyncTimeStampFile, lastSyncTimestampFile);
operationLog.info(this.getClass() + " finished executing.");
@@ -180,6 +165,32 @@ public class DataSetRegistrationTask<T extends DataSetInformation> implements IM
}
}
+ private Date getLastSyncTimeStamp(File lastSyncTimestampFile) throws ParseException
+ {
+ if (lastSyncTimestampFile.exists())
+ {
+ String timeStr = FileUtilities.loadToString(lastSyncTimestampFile).trim();
+ return formatter.parse(timeStr);
+ }
+ else
+ {
+ return new Date(0L);
+ }
+ }
+
+ private Set<String> getNotSyncedDataSetCodes(String fileName)
+ {
+ File notSyncedDataSetsFile = new File(fileName);
+ if (notSyncedDataSetsFile.exists())
+ {
+ List<String> list = FileUtilities.loadToStringList(notSyncedDataSetsFile);
+ return new LinkedHashSet<String>(list);
+ }
+ else
+ {
+ return new LinkedHashSet<String>();
+ }
+ }
private void sendErrorEmail(SyncConfig config, String subject)
{
if (config.getLogFilePath() != null)
@@ -215,7 +226,7 @@ public class DataSetRegistrationTask<T extends DataSetInformation> implements IM
}
}
- private void saveSyncTimestamp()
+ private void saveSyncTimestamp(File newLastSyncTimeStampFile, File lastSyncTimestampFile)
{
newLastSyncTimeStampFile.renameTo(lastSyncTimestampFile);
}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/EntitySynchronizer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/EntitySynchronizer.java
index 4463dab5e53e0ea4fe57caf8f7d9ac6a33f714d0..b113cbb0af50ec58e5e56c2531261fb6673f3a13 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/EntitySynchronizer.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/EntitySynchronizer.java
@@ -40,6 +40,7 @@ import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPathExpressionException;
import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.log4j.Logger;
import org.w3c.dom.Document;
@@ -143,18 +144,21 @@ public class EntitySynchronizer
private final Date lastSyncTimestamp;
+ private final Set<String> dataSetsCodesToRetry;
+
private final SyncConfig config;
private final Logger operationLog;
public EntitySynchronizer(IEncapsulatedOpenBISService service, String dataStoreCode, File storeRoot, Date lastSyncTimestamp,
- DataSetProcessingContext context,
+ Set<String> dataSetsCodesToRetry, DataSetProcessingContext context,
SyncConfig config, Logger operationLog)
{
this.service = service;
this.dataStoreCode = dataStoreCode;
this.storeRoot = storeRoot;
this.lastSyncTimestamp = lastSyncTimestamp;
+ this.dataSetsCodesToRetry = dataSetsCodesToRetry;
this.context = context;
this.config = config;
this.operationLog = operationLog;
@@ -340,7 +344,7 @@ public class EntitySynchronizer
operationLog.info("entity operation result: " + operationResult);
}
- private void registerPhysicalDataSets(Map<String, DataSetWithConnections> physicalDSMap)
+ private void registerPhysicalDataSets(Map<String, DataSetWithConnections> physicalDSMap) throws IOException
{
List<DataSetWithConnections> dsList = new ArrayList<DataSetWithConnections>(physicalDSMap.values());
List<String> notSyncedDataSetCodes = Collections.synchronizedList(new ArrayList<String>());
@@ -350,7 +354,12 @@ public class EntitySynchronizer
// parallelized
ParallelizedExecutor.process(dsList, new DataSetRegistrationTaskExecutor(notSyncedDataSetCodes), 0.5, 10, "register data sets", 0, false);
+ // backup the current not synced data set codes file, delete the original file
File notSyncedDataSetsFile = new File(config.getNotSyncedDataSetsFileName());
+ File backupLastSyncTimeStampFile = new File(config.getNotSyncedDataSetsFileName() + ".bk");
+ FileUtils.copyFile(notSyncedDataSetsFile, backupLastSyncTimeStampFile);
+ FileUtilities.delete(notSyncedDataSetsFile);
+
for (String dsCode : notSyncedDataSetCodes)
{
FileUtilities.appendToFile(notSyncedDataSetsFile, dsCode, true);