diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/DataSetRegistrationTask.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/DataSetRegistrationTask.java index 869b06a73306821b0a3e9fb1a39137ad6a449c8d..040d230edd79ac45343e10756067b4653804623c 100644 --- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/DataSetRegistrationTask.java +++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/DataSetRegistrationTask.java @@ -53,6 +53,9 @@ import ch.systemsx.cisd.common.logging.LogFactory; import ch.systemsx.cisd.common.mail.EMailAddress; import ch.systemsx.cisd.common.mail.IMailClient; import ch.systemsx.cisd.common.maintenance.IMaintenanceTask; +import ch.systemsx.cisd.common.parser.ILine; +import ch.systemsx.cisd.common.parser.filter.ExcludeEmptyAndCommentLineFilter; +import ch.systemsx.cisd.common.parser.filter.ILineFilter; import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.PluginTaskInfoProvider; import ch.systemsx.cisd.openbis.dss.generic.shared.DataSetProcessingContext; import ch.systemsx.cisd.openbis.dss.generic.shared.IConfigProvider; @@ -145,10 +148,13 @@ public class DataSetRegistrationTask<T extends DataSetInformation> implements IM File newLastSyncTimeStampFile = new File(fileName + ".new"); FileUtilities.writeToFile(newLastSyncTimeStampFile, formatter.format(new Date())); - Set<String> notSyncedDataSetCodes = getNotSyncedDataSetCodes(config.getNotSyncedDataSetsFileName()); + String notSyncedDataSetsFileName = config.getNotSyncedDataSetsFileName(); + Set<String> notSyncedDataSetCodes = getNotSyncedDataSetCodes(notSyncedDataSetsFileName); + Set<String> blackListedDataSetCodes = getBlackListedDataSetCodes(notSyncedDataSetsFileName); EntitySynchronizer synchronizer = - new EntitySynchronizer(service, dataStoreCode, storeRoot, lastSyncTimestamp, notSyncedDataSetCodes, context, config, + new EntitySynchronizer(service, dataStoreCode, storeRoot, lastSyncTimestamp, notSyncedDataSetCodes, blackListedDataSetCodes, + context, config, operationLog); synchronizer.syncronizeEntities(); @@ -178,12 +184,12 @@ public class DataSetRegistrationTask<T extends DataSetInformation> implements IM } } - private Set<String> getNotSyncedDataSetCodes(String fileName) + private Set<String> getDataSetCodesFromNotSyncedDataSetsFile(String fileName, ILineFilter linefilter) { File notSyncedDataSetsFile = new File(fileName); if (notSyncedDataSetsFile.exists()) { - List<String> list = FileUtilities.loadToStringList(notSyncedDataSetsFile); + List<String> list = FileUtilities.loadToStringList(notSyncedDataSetsFile, linefilter); return new LinkedHashSet<String>(list); } else @@ -191,6 +197,26 @@ public class DataSetRegistrationTask<T extends DataSetInformation> implements IM return new LinkedHashSet<String>(); } } + + private Set<String> getNotSyncedDataSetCodes(String fileName) + { + return getDataSetCodesFromNotSyncedDataSetsFile(fileName, ExcludeEmptyAndCommentLineFilter.INSTANCE); + } + + private Set<String> getBlackListedDataSetCodes(String fileName) + { + return getDataSetCodesFromNotSyncedDataSetsFile(fileName, new ILineFilter() + { + @Override + public <T> boolean acceptLine(ILine<T> line) + { + assert line != null : "Unspecified line"; + final String trimmed = line.getText().trim(); + return trimmed.length() > 0 && trimmed.startsWith("#") == true; + } + }); + } + private void sendErrorEmail(SyncConfig config, String subject) { if (config.getLogFilePath() != null) diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/EntitySynchronizer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/EntitySynchronizer.java index dc77eca7ef9d9fe1e69f2e6c29d15a43dbd0cbb7..141618a7dc1f8dce9555a2bd785389d9bf0c3efc 100644 --- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/EntitySynchronizer.java +++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/EntitySynchronizer.java @@ -149,8 +149,10 @@ public class EntitySynchronizer private final Logger operationLog; + private final Set<String> blackListedDataSetCodes; + public EntitySynchronizer(IEncapsulatedOpenBISService service, String dataStoreCode, File storeRoot, Date lastSyncTimestamp, - Set<String> dataSetsCodesToRetry, DataSetProcessingContext context, + Set<String> dataSetsCodesToRetry, Set<String> blackListedDataSetCodes, DataSetProcessingContext context, SyncConfig config, Logger operationLog) { this.service = service; @@ -158,6 +160,7 @@ public class EntitySynchronizer this.storeRoot = storeRoot; this.lastSyncTimestamp = lastSyncTimestamp; this.dataSetsCodesToRetry = dataSetsCodesToRetry; + this.blackListedDataSetCodes = blackListedDataSetCodes; this.context = context; this.config = config; this.operationLog = operationLog; @@ -374,6 +377,13 @@ public class EntitySynchronizer ParallelizedExecutor.process(dsList, new DataSetRegistrationTaskExecutor(notRegisteredDataSetCodes), 0.5, 10, "register data sets", 0, false); // backup the current not synced data set codes file, delete the original file + saveNotSyncedDataSetsFile(notRegisteredDataSetCodes); + + return notRegisteredDataSetCodes; + } + + private void saveNotSyncedDataSetsFile(List<String> notRegisteredDataSetCodes) throws IOException + { File notSyncedDataSetsFile = new File(config.getNotSyncedDataSetsFileName()); if (notSyncedDataSetsFile.exists()) { @@ -384,7 +394,11 @@ public class EntitySynchronizer { FileUtilities.appendToFile(notSyncedDataSetsFile, dsCode, true); } - return notRegisteredDataSetCodes; + // append the blacklisted codes to the end of the file + for (String dsCode : blackListedDataSetCodes) + { + FileUtilities.appendToFile(notSyncedDataSetsFile, dsCode, true); + } } private void backupAndResetNotSyncedDataSetsFile(File notSyncedDataSetsFile) throws IOException