diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/HarvesterMaintenanceTask.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/HarvesterMaintenanceTask.java
index d9f5b3e210f735d4bce832b4f493ce10e4029afa..3283983217a96f865406a3ada896358c1eeb8dee 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/HarvesterMaintenanceTask.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/HarvesterMaintenanceTask.java
@@ -40,6 +40,7 @@ import javax.activation.DataHandler;
import javax.activation.DataSource;
import javax.mail.util.ByteArrayDataSource;
+import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.DailyRollingFileAppender;
import org.apache.log4j.Logger;
import org.apache.log4j.PatternLayout;
@@ -49,7 +50,10 @@ import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.config.Conf
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.config.SyncConfig;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.config.SynchronizationConfigReader;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.EntitySynchronizer;
+import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.SynchronizationContext;
+import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.translator.INameTranslator;
import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
+import ch.systemsx.cisd.cifex.shared.basic.UserFailureException;
import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
@@ -105,7 +109,6 @@ public class HarvesterMaintenanceTask<T extends DataSetInformation> implements I
Timestamps(Date lastIncSyncTimestamp, Date lastFullSyncTimestamp)
{
- super();
this.lastIncSyncTimestamp = lastIncSyncTimestamp;
this.lastFullSyncTimestamp = lastFullSyncTimestamp;
}
@@ -143,80 +146,14 @@ public class HarvesterMaintenanceTask<T extends DataSetInformation> implements I
{
operationLog.info(this.getClass() + " started.");
- List<SyncConfig> configs;
try
{
- configs = SynchronizationConfigReader.readConfiguration(harvesterConfigFile);
- for (SyncConfig config : configs)
+ for (SyncConfig config : SynchronizationConfigReader.readConfiguration(harvesterConfigFile))
{
Logger logger = createLogger(config);
try
{
- logger.info("====================== " + (config.isDryRun() ? "Dry " : "")
- + "Running synchronization from data source: " + config.getDataSourceOpenbisURL()
- + " for user " + config.getUser());
-
- logger.info("verbose = " + config.isVerbose());
- String fileName = config.getLastSyncTimestampFileName();
- File lastSyncTimestampFile = new File(fileName);
- Timestamps timestamps = loadCutOffTimeStamps(lastSyncTimestampFile);
-
- Date cutOffTimestamp = timestamps.lastIncSyncTimestamp;
- boolean isFullSync = lastSyncTimestampFile.exists() == false || isTimeForFullSync(config, timestamps.lastFullSyncTimestamp);
- logger.info("Last incremental sync timestamp: " + timestamps.lastIncSyncTimestamp);
- logger.info("Last full sync timestamp: " + timestamps.lastFullSyncTimestamp);
- if (isFullSync == true)
- {
- cutOffTimestamp = new Date(0L);
- if (lastSyncTimestampFile.exists() == false)
- {
- logger.info("Performing a full initial sync");
- }
- else
- {
- logger.info("Performing a full sync as a minimum of " + config.getFullSyncInterval()
- + " day(s) have elapsed since last full sync.");
- }
- }
- else
- {
- logger.info("Performing an incremental sync");
- }
- String notSyncedEntitiesFileName = config.getNotSyncedEntitiesFileName();
- Set<String> notSyncedDataSetCodes = getNotSyncedDataSetCodes(notSyncedEntitiesFileName);
- Set<String> notSyncedAttachmentHolderCodes = getNotSyncedAttachmentHolderCodes(notSyncedEntitiesFileName);
- Set<String> blackListedDataSetCodes = getBlackListedDataSetCodes(notSyncedEntitiesFileName);
-
- Date newCutOffTimestamp = new Date();
-
- EntitySynchronizer synchronizer =
- new EntitySynchronizer(service, dataStoreCode, storeRoot, cutOffTimestamp, timestamps.lastIncSyncTimestamp,
- notSyncedDataSetCodes,
- blackListedDataSetCodes,
- notSyncedAttachmentHolderCodes,
- context, config, logger);
- Date resourceListTimestamp = synchronizer.synchronizeEntities();
- if (resourceListTimestamp.before(newCutOffTimestamp))
- {
- newCutOffTimestamp = resourceListTimestamp;
- }
- Date newLastIncSyncTimestamp = newCutOffTimestamp;
- Date newLastFullSyncTimestamp = timestamps.lastFullSyncTimestamp;
- if (isFullSync == true)
- {
- newLastFullSyncTimestamp = newCutOffTimestamp;
- }
-
- if (config.isDryRun() == false)
- {
- logger.info("Saving the timestamp of sync start to file");
- saveSyncTimestamp(lastSyncTimestampFile, newLastIncSyncTimestamp, newLastFullSyncTimestamp);
- }
- else
- {
- logger.info("Dry run finished");
- }
-
+ synchronize(config, logger);
} catch (Exception e)
{
logger.error("Sync failed: ", e);
@@ -231,6 +168,91 @@ public class HarvesterMaintenanceTask<T extends DataSetInformation> implements I
operationLog.info(this.getClass() + " finished executing.");
}
+ private void synchronize(SyncConfig config, Logger logger) throws IOException, ParseException, Exception
+ {
+ logger.info("====================== " + (config.isDryRun() ? "Dry " : "")
+ + "Running synchronization from data source: " + config.getDataSourceOpenbisURL()
+ + " for user " + config.getUser());
+ checkAlias(config);
+ logger.info("verbose = " + config.isVerbose());
+
+ String fileName = config.getLastSyncTimestampFileName();
+ File lastSyncTimestampFile = new File(fileName);
+ Timestamps timestamps = loadCutOffTimeStamps(lastSyncTimestampFile);
+
+ Date cutOffTimestamp = timestamps.lastIncSyncTimestamp;
+ boolean isFullSync = lastSyncTimestampFile.exists() == false || isTimeForFullSync(config, timestamps.lastFullSyncTimestamp);
+ logger.info("Last incremental sync timestamp: " + timestamps.lastIncSyncTimestamp);
+ logger.info("Last full sync timestamp: " + timestamps.lastFullSyncTimestamp);
+ if (isFullSync == true)
+ {
+ cutOffTimestamp = new Date(0L);
+ if (lastSyncTimestampFile.exists() == false)
+ {
+ logger.info("Performing a full initial sync");
+ }
+ else
+ {
+ logger.info("Performing a full sync as a minimum of " + config.getFullSyncInterval()
+ + " day(s) have elapsed since last full sync.");
+ }
+ }
+ else
+ {
+ logger.info("Performing an incremental sync");
+ }
+ String notSyncedEntitiesFileName = config.getNotSyncedEntitiesFileName();
+ Set<String> notSyncedDataSetCodes = getNotSyncedDataSetCodes(notSyncedEntitiesFileName);
+ Set<String> notSyncedAttachmentHolderCodes = getNotSyncedAttachmentHolderCodes(notSyncedEntitiesFileName);
+ Set<String> blackListedDataSetCodes = getBlackListedDataSetCodes(notSyncedEntitiesFileName);
+
+ Date newCutOffTimestamp = new Date();
+
+
+ SynchronizationContext syncContext = new SynchronizationContext();
+ syncContext.setService(service);
+ syncContext.setDataStoreCode(dataStoreCode);
+ syncContext.setStoreRoot(storeRoot);
+ syncContext.setLastSyncTimestamp(cutOffTimestamp);
+ syncContext.setLastIncSyncTimestamp(timestamps.lastIncSyncTimestamp);
+ syncContext.setDataSetsCodesToRetry(notSyncedDataSetCodes);
+ syncContext.setBlackListedDataSetCodes(blackListedDataSetCodes);
+ syncContext.setAttachmentHolderCodesToRetry(notSyncedAttachmentHolderCodes);
+ syncContext.setContext(context);
+ syncContext.setConfig(config);
+ syncContext.setOperationLog(logger);
+ EntitySynchronizer synchronizer = new EntitySynchronizer(syncContext);
+ Date resourceListTimestamp = synchronizer.synchronizeEntities();
+ if (resourceListTimestamp.before(newCutOffTimestamp))
+ {
+ newCutOffTimestamp = resourceListTimestamp;
+ }
+ Date newLastIncSyncTimestamp = newCutOffTimestamp;
+ Date newLastFullSyncTimestamp = timestamps.lastFullSyncTimestamp;
+ if (isFullSync == true)
+ {
+ newLastFullSyncTimestamp = newCutOffTimestamp;
+ }
+
+ if (config.isDryRun() == false)
+ {
+ logger.info("Saving the timestamp of sync start to file");
+ saveSyncTimestamp(lastSyncTimestampFile, newLastIncSyncTimestamp, newLastFullSyncTimestamp);
+ }
+ else
+ {
+ logger.info("Dry run finished");
+ }
+ }
+
+ private void checkAlias(SyncConfig config) throws UserFailureException
+ {
+ if (config.isTranslateUsingDataSourceAlias() && StringUtils.isBlank(config.getDataSourceAlias()))
+ {
+ throw new UserFailureException("Please specify a data source alias in the config file to be used in name translations.");
+ }
+ }
+
private Logger createLogger(SyncConfig config)
{
Logger logger = Logger.getLogger(LogFactory.getLoggerName(LogCategory.OPERATION, EntitySynchronizer.class)
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
index d28c9c4ddf042a7f9ea6add1bcbafda78d308e6b..f9152d3b38095a54cef8c5ac62cdb82f1d5c2462 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
@@ -33,14 +33,14 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import javax.xml.xpath.XPathExpressionException;
+
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.collections4.map.MultiKeyMap;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
-import org.apache.log4j.DailyRollingFileAppender;
import org.apache.log4j.Logger;
-import org.apache.log4j.PatternLayout;
import org.w3c.dom.Document;
import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi;
@@ -66,7 +66,6 @@ import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.SyncEntityKind
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.entitygraph.EntityGraph;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.entitygraph.IEntityRetriever;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.entitygraph.INode;
-import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.HarvesterMaintenanceTask;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.config.ParallelizedExecutionPreferences;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.config.SyncConfig;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.ResourceListParserData.Connection;
@@ -78,7 +77,6 @@ import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronize
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.ResourceListParserData.MasterData;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.ResourceListParserData.MaterialWithLastModificationDate;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.datasourceconnector.DataSourceConnector;
-import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.datasourceconnector.IDataSourceConnector;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.parallelizedExecutor.AttachmentSynchronizationSummary;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.parallelizedExecutor.AttachmentSynchronizationTaskExecutor;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.parallelizedExecutor.DataSetRegistrationTaskExecutor;
@@ -91,8 +89,6 @@ import ch.systemsx.cisd.cifex.shared.basic.UserFailureException;
import ch.systemsx.cisd.common.concurrent.ParallelizedExecutor;
import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.common.logging.Log4jSimpleLogger;
-import ch.systemsx.cisd.common.logging.LogCategory;
-import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.etlserver.registrator.api.v1.impl.ConversionUtils;
import ch.systemsx.cisd.openbis.dss.generic.shared.DataSetDirectoryProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.DataSetProcessingContext;
@@ -170,117 +166,43 @@ public class EntitySynchronizer
private final Set<String> blackListedDataSetCodes;
- public EntitySynchronizer(IEncapsulatedOpenBISService service, String dataStoreCode, File storeRoot, Date lastSyncTimestamp,
- Date lastIncSyncTimestamp, Set<String> dataSetsCodesToRetry, Set<String> blackListedDataSetCodes,
- Set<String> attachmentHolderCodesToRetry,
- DataSetProcessingContext context,
- SyncConfig config, Logger operationLog)
+ public EntitySynchronizer(SynchronizationContext synContext)
{
- this.service = service;
- this.dataStoreCode = dataStoreCode;
- this.storeRoot = storeRoot;
- this.lastSyncTimestamp = lastSyncTimestamp;
- this.lastIncSyncTimestamp = lastIncSyncTimestamp;
- this.dataSetsCodesToRetry = dataSetsCodesToRetry;
- this.attachmentHolderCodesToRetry = attachmentHolderCodesToRetry;
- this.blackListedDataSetCodes = blackListedDataSetCodes;
- this.context = context;
- this.config = config;
- this.operationLog = operationLog;
+ this.service = synContext.getService();
+ this.dataStoreCode = synContext.getDataStoreCode();
+ this.storeRoot = synContext.getStoreRoot();
+ this.lastSyncTimestamp = synContext.getLastSyncTimestamp();
+ this.lastIncSyncTimestamp = synContext.getLastIncSyncTimestamp();
+ this.dataSetsCodesToRetry = synContext.getDataSetsCodesToRetry();
+ this.attachmentHolderCodesToRetry = synContext.getAttachmentHolderCodesToRetry();
+ this.blackListedDataSetCodes = synContext.getBlackListedDataSetCodes();
+ this.context = synContext.getContext();
+ this.config = synContext.getConfig();
+ this.operationLog = synContext.getOperationLog();
}
public Date synchronizeEntities() throws Exception
{
- DataSourceConnector dataSourceConnector =
- new DataSourceConnector(config.getDataSourceURI(), config.getAuthenticationCredentials(), operationLog);
- return synchronizeEntities(dataSourceConnector);
- }
-
- private Date synchronizeEntities(IDataSourceConnector dataSourceConnector) throws Exception
- {
- // retrieve the document from the data source
- operationLog.info("\n");
- operationLog.info("Retrieving the resource list...");
- Document doc = dataSourceConnector.getResourceListAsXMLDoc(Arrays.asList(ArrayUtils.EMPTY_STRING_ARRAY));
-
- // Parse the resource list: This sends back all projects,
- // experiments, samples and data sets contained in the XML together with their last modification date to be used for filtering
- operationLog.info("\n");
- operationLog.info("Parsing the resource list xml document...");
- String dataSourceAlias = config.getDataSourceAlias();
- INameTranslator nameTranslator = null;
- if (config.isTranslateUsingDataSourceAlias() == true)
- {
- if (StringUtils.isBlank(dataSourceAlias) == true)
- {
- throw new UserFailureException("Please specify a data source alias in the config file to be used in name translations.");
- }
- nameTranslator = new PrefixBasedNameTranslator(dataSourceAlias);
- }
-
- ResourceListParser parser = ResourceListParser.create(nameTranslator, dataStoreCode);
- ResourceListParserData data = parser.parseResourceListDocument(doc);
+ Document doc = getResourceList();
+ ResourceListParserData data = parseResourceList(doc);
processDeletions(data);
-
- operationLog.info("\n");
- operationLog.info("Registering master data...");
registerMasterData(data.getMasterData());
+ MultiKeyMap<String, String> newEntities = registerEntities(data);
+ List<String> notSyncedAttachmentsHolders = registerAttachments(data, newEntities);
- AtomicEntityOperationDetailsBuilder builder = new AtomicEntityOperationDetailsBuilder();
-
- for (String spaceCode : data.getHarvesterSpaceList())
- {
- Space space = service.tryGetSpace(new SpaceIdentifier(spaceCode));
- if (space == null)
- {
- builder.space(new NewSpace(spaceCode, "Synchronized from: " + config.getDataSourceURI(), null));
- }
- }
-
- processMetaData(data, builder);
-
- if (config.isVerbose() == true)
- {
- verboseLogEntityOperations(builder.getDetails());
- }
-
- MultiKeyMap<String, String> newEntities = new MultiKeyMap<String, String>();
- if (config.isDryRun() == false)
- {
- AtomicEntityOperationResult operationResult = service.performEntityOperations(builder.getDetails());
- newEntities = getNewEntities(builder);
- operationLog.info("Entity operation result: " + operationResult);
- }
- operationLog.info("\n");
- operationLog.info("Processing attachments...");
- List<IncomingEntity<?>> attachmentHoldersToProcess =
- data.filterAttachmentHoldersByLastModificationDate(lastSyncTimestamp, attachmentHolderCodesToRetry);
+ registerDataSets(data, notSyncedAttachmentsHolders);
- if (config.isVerbose())
- {
- verboseLogProcessAttachments(attachmentHoldersToProcess, newEntities);
- }
-
- List<String> notSyncedAttachmentsHolders = new ArrayList<String>();
- if (config.isDryRun() == false)
- {
- AttachmentSynchronizationSummary syncSummary = processAttachments(attachmentHoldersToProcess);
- notSyncedAttachmentsHolders = syncSummary.notRegisteredAttachmentHolderCodes;
- operationLog.info("Attachment synchronization summary:\n" + syncSummary.addedCount + " attachment(s) were added.\n"
- + syncSummary.updatedCount + " attachment(s) were updated.\n"
- + syncSummary.deletedCount + " attachment(s) were deleted.\n"
- + (notSyncedAttachmentsHolders.isEmpty() ? ""
- : "synchronization of attachments for "
- + notSyncedAttachmentsHolders.size() + " entitities FAILED."));
- }
+ return data.getResourceListTimestamp();
+ }
+ private void registerDataSets(ResourceListParserData data, List<String> notSyncedAttachmentsHolders) throws IOException
+ {
// register physical data sets without any hierarchy
// Note that container/component and parent/child relationships are established post-reg.
// setParentDataSetsOnTheChildren(data);
Map<String, IncomingDataSet> physicalDSMap =
data.filterPhysicalDataSetsByLastModificationDate(lastSyncTimestamp, dataSetsCodesToRetry, blackListedDataSetCodes);
- operationLog.info("\n");
operationLog.info("Registering data sets...");
if (config.isVerbose())
@@ -315,26 +237,115 @@ public class EntitySynchronizer
skippedDataSets.addAll(notRegisteredDataSetCodes);
skippedDataSets.addAll(blackListedDataSetCodes);
establishDataSetRelationships(data.getDataSetsToProcess(), skippedDataSets, physicalDSMap);
+ }
- // cleanup();
+ private List<String> registerAttachments(ResourceListParserData data, MultiKeyMap<String, String> newEntities)
+ {
+ operationLog.info("Processing attachments...");
+ List<IncomingEntity<?>> attachmentHoldersToProcess =
+ data.filterAttachmentHoldersByLastModificationDate(lastSyncTimestamp, attachmentHolderCodesToRetry);
- return data.getResourceListTimestamp();
+ if (config.isVerbose())
+ {
+ verboseLogProcessAttachments(attachmentHoldersToProcess, newEntities);
+ }
+
+ List<String> notSyncedAttachmentsHolders = new ArrayList<String>();
+ if (config.isDryRun() == false)
+ {
+ AttachmentSynchronizationSummary syncSummary = processAttachments(attachmentHoldersToProcess);
+ notSyncedAttachmentsHolders = syncSummary.notRegisteredAttachmentHolderCodes;
+ operationLog.info("Attachment synchronization summary:\n" + syncSummary.addedCount + " attachment(s) were added.\n"
+ + syncSummary.updatedCount + " attachment(s) were updated.\n"
+ + syncSummary.deletedCount + " attachment(s) were deleted.\n"
+ + (notSyncedAttachmentsHolders.isEmpty() ? ""
+ : "synchronization of attachments for "
+ + notSyncedAttachmentsHolders.size() + " entitities FAILED."));
+ }
+ return notSyncedAttachmentsHolders;
+ }
+
+ private MultiKeyMap<String, String> registerEntities(ResourceListParserData data)
+ {
+ AtomicEntityOperationDetails entityOperationDetails = createEntityOperationDetails(data);
+
+ MultiKeyMap<String, String> newEntities = new MultiKeyMap<String, String>();
+ if (config.isDryRun() == false)
+ {
+ AtomicEntityOperationResult operationResult = service.performEntityOperations(entityOperationDetails);
+ newEntities = getNewEntities(entityOperationDetails);
+ operationLog.info("Entity operation result: " + operationResult);
+ }
+ return newEntities;
+ }
+
+ private AtomicEntityOperationDetails createEntityOperationDetails(ResourceListParserData data)
+ {
+ AtomicEntityOperationDetailsBuilder builder = new AtomicEntityOperationDetailsBuilder();
+
+ processSpaces(data, builder);
+ processProjects(data, builder);
+ processExperiments(data, builder);
+ processSamples(data, builder);
+ processMaterials(data, builder);
+
+ AtomicEntityOperationDetails entityOperationDetails = builder.getDetails();
+ if (config.isVerbose() == true)
+ {
+ verboseLogEntityOperations(entityOperationDetails);
+ }
+ return entityOperationDetails;
+ }
+
+ private void processSpaces(ResourceListParserData data, AtomicEntityOperationDetailsBuilder builder)
+ {
+ for (String spaceCode : data.getHarvesterSpaceList())
+ {
+ Space space = service.tryGetSpace(new SpaceIdentifier(spaceCode));
+ if (space == null)
+ {
+ builder.space(new NewSpace(spaceCode, "Synchronized from: " + config.getDataSourceURI(), null));
+ }
+ }
+ }
+
+ private ResourceListParserData parseResourceList(Document doc) throws XPathExpressionException
+ {
+ // Parse the resource list: This sends back all projects,
+ // experiments, samples and data sets contained in the XML together with their last modification date to be used for filtering
+ operationLog.info("Parsing the resource list xml document...");
+ INameTranslator nameTranslator = new PrefixBasedNameTranslator(config.getDataSourceAlias());
+
+ ResourceListParser parser = ResourceListParser.create(nameTranslator, dataStoreCode);
+ ResourceListParserData data = parser.parseResourceListDocument(doc);
+ return data;
+ }
+
+ private Document getResourceList() throws Exception
+ {
+
+ DataSourceConnector dataSourceConnector =
+ new DataSourceConnector(config.getDataSourceURI(), config.getAuthenticationCredentials(), operationLog);
+ operationLog.info("\nRetrieving the resource list...");
+ Document doc = dataSourceConnector.getResourceListAsXMLDoc(Arrays.asList(ArrayUtils.EMPTY_STRING_ARRAY));
+
+ return doc;
}
- private MultiKeyMap<String, String> getNewEntities(AtomicEntityOperationDetailsBuilder builder)
+ private MultiKeyMap<String, String> getNewEntities(AtomicEntityOperationDetails details)
{
MultiKeyMap<String, String> newEntities = new MultiKeyMap<String, String>();
- List<NewSample> sampleRegistrations = builder.getDetails().getSampleRegistrations();
+ List<NewSample> sampleRegistrations = details.getSampleRegistrations();
for (NewSample newSample : sampleRegistrations)
{
newEntities.put(SyncEntityKind.SAMPLE.getLabel(), newSample.getPermID(), newSample.getIdentifier());
}
- List<NewExperiment> experimentRegistrations = builder.getDetails().getExperimentRegistrations();
+ List<NewExperiment> experimentRegistrations = details.getExperimentRegistrations();
for (NewExperiment newExperiment : experimentRegistrations)
{
newEntities.put(SyncEntityKind.EXPERIMENT.getLabel(), newExperiment.getPermID(), newExperiment.getIdentifier());
}
- List<NewProject> projectRegistrations = builder.getDetails().getProjectRegistrations();
+ List<NewProject> projectRegistrations = details.getProjectRegistrations();
for (NewProject newProject : projectRegistrations)
{
newEntities.put(SyncEntityKind.PROJECT.getLabel(), newProject.getPermID(), newProject.getIdentifier());
@@ -723,19 +734,9 @@ public class EntitySynchronizer
FileUtils.writeStringToFile(notSyncedDataSetsFile, "");
}
- private void processMetaData(ResourceListParserData data, AtomicEntityOperationDetailsBuilder builder)
- {
- processProjects(data, builder);
-
- processExperiments(data, builder);
-
- processSamples(data, builder);
-
- processMaterials(data, builder);
- }
-
private void registerMasterData(MasterData masterData)
{
+ operationLog.info("Registering master data...");
if (config.isVerbose() == true)
{
// operationLog.info("-------The following master data will be synchronized-------");
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/SynchronizationContext.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/SynchronizationContext.java
new file mode 100644
index 0000000000000000000000000000000000000000..fddecb4ba0533efbfb3f2743d7bdfa434fe138fa
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/SynchronizationContext.java
@@ -0,0 +1,166 @@
+/*
+ * Copyright 2018 ETH Zuerich;private SIS
+ *
+ * Licensed under the Apache License;private Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing;private software
+ * distributed under the License is distributed on an "AS IS" BASIS;private final
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND;private either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer;
+
+import java.io.File;
+import java.util.Date;
+import java.util.Set;
+
+import org.apache.log4j.Logger;
+
+import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.config.SyncConfig;
+import ch.systemsx.cisd.openbis.dss.generic.shared.DataSetProcessingContext;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
+
+/**
+ * @author Franz-Josef Elmer
+ */
+public class SynchronizationContext
+{
+ private IEncapsulatedOpenBISService service;
+
+ private String dataStoreCode;
+
+ private File storeRoot;
+
+ private Date lastSyncTimestamp;
+
+ private Date lastIncSyncTimestamp;
+
+ private Set<String> dataSetsCodesToRetry;
+
+ private Set<String> blackListedDataSetCodes;
+
+ private Set<String> attachmentHolderCodesToRetry;
+
+ private DataSetProcessingContext context;
+
+ private SyncConfig config;
+
+ private Logger operationLog;
+
+ public IEncapsulatedOpenBISService getService()
+ {
+ return service;
+ }
+
+ public void setService(IEncapsulatedOpenBISService service)
+ {
+ this.service = service;
+ }
+
+ public String getDataStoreCode()
+ {
+ return dataStoreCode;
+ }
+
+ public void setDataStoreCode(String dataStoreCode)
+ {
+ this.dataStoreCode = dataStoreCode;
+ }
+
+ public File getStoreRoot()
+ {
+ return storeRoot;
+ }
+
+ public void setStoreRoot(File storeRoot)
+ {
+ this.storeRoot = storeRoot;
+ }
+
+ public Date getLastSyncTimestamp()
+ {
+ return lastSyncTimestamp;
+ }
+
+ public void setLastSyncTimestamp(Date lastSyncTimestamp)
+ {
+ this.lastSyncTimestamp = lastSyncTimestamp;
+ }
+
+ public Date getLastIncSyncTimestamp()
+ {
+ return lastIncSyncTimestamp;
+ }
+
+ public void setLastIncSyncTimestamp(Date lastIncSyncTimestamp)
+ {
+ this.lastIncSyncTimestamp = lastIncSyncTimestamp;
+ }
+
+ public Set<String> getDataSetsCodesToRetry()
+ {
+ return dataSetsCodesToRetry;
+ }
+
+ public void setDataSetsCodesToRetry(Set<String> dataSetsCodesToRetry)
+ {
+ this.dataSetsCodesToRetry = dataSetsCodesToRetry;
+ }
+
+ public Set<String> getBlackListedDataSetCodes()
+ {
+ return blackListedDataSetCodes;
+ }
+
+ public void setBlackListedDataSetCodes(Set<String> blackListedDataSetCodes)
+ {
+ this.blackListedDataSetCodes = blackListedDataSetCodes;
+ }
+
+ public Set<String> getAttachmentHolderCodesToRetry()
+ {
+ return attachmentHolderCodesToRetry;
+ }
+
+ public void setAttachmentHolderCodesToRetry(Set<String> attachmentHolderCodesToRetry)
+ {
+ this.attachmentHolderCodesToRetry = attachmentHolderCodesToRetry;
+ }
+
+ public DataSetProcessingContext getContext()
+ {
+ return context;
+ }
+
+ public void setContext(DataSetProcessingContext context)
+ {
+ this.context = context;
+ }
+
+ public SyncConfig getConfig()
+ {
+ return config;
+ }
+
+ public void setConfig(SyncConfig config)
+ {
+ this.config = config;
+ }
+
+ public Logger getOperationLog()
+ {
+ return operationLog;
+ }
+
+ public void setOperationLog(Logger operationLog)
+ {
+ this.operationLog = operationLog;
+ }
+
+}