From 4aa9166e7f83f33ef9d806f790e1ddb4f1627e8f Mon Sep 17 00:00:00 2001
From: gakin <gakin>
Date: Sun, 29 Jan 2017 21:28:45 +0000
Subject: [PATCH] SSDM-4614 : OpenbisSync Vocabulary term cleanup +
internalnameSpace property fix
SVN: 37638
---
.../synchronizer/EntitySynchronizer.java | 21 ++++-
.../synchronizer/MasterDataParser.java | 22 +++--
.../synchronizer/MasterDataSynchronizer.java | 80 ++++++++++++++-----
3 files changed, 89 insertions(+), 34 deletions(-)
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
index 4ed145a78d0..a1dd6caab80 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
@@ -146,6 +146,8 @@ public class EntitySynchronizer
private final Set<String> blackListedDataSetCodes;
+ private MasterDataSynchronizer masterDataSyncronizer;
+
public EntitySynchronizer(IEncapsulatedOpenBISService service, String dataStoreCode, File storeRoot, Date lastSyncTimestamp,
Set<String> dataSetsCodesToRetry, Set<String> blackListedDataSetCodes, DataSetProcessingContext context,
SyncConfig config, Logger operationLog)
@@ -189,6 +191,8 @@ public class EntitySynchronizer
processDeletions(data);
operationLog.info("registering master data");
+
+ // masterDataSyncronizer = new MasterDataSynchronizer(data.getMasterData());
// registerMasterData(data);
AtomicEntityOperationDetailsBuilder builder = new AtomicEntityOperationDetailsBuilder();
@@ -224,9 +228,23 @@ public class EntitySynchronizer
operationLog.info("start linking/un-linking container and component data sets");
establishDataSetRelationships(data.getDataSetsToProcess(), notRegisteredDataSets, physicalDSMap);
+ // cleanup();
+
return data.getResourceListTimestamp();
}
+ private void cleanup()
+ {
+ operationLog.info("Cleaning up unused master data");
+ try
+ {
+ masterDataSyncronizer.cleanupUnusedMasterData();
+ } catch (Exception e)
+ {
+ operationLog.warn(e.getMessage());
+ }
+ }
+
private void establishDataSetRelationships(Map<String, DataSetWithConnections> dataSetsToProcess,
List<String> notRegisteredDataSets, Map<String, DataSetWithConnections> physicalDSMap)
{
@@ -422,8 +440,7 @@ public class EntitySynchronizer
private void registerMasterData(ResourceListParserData data)
{
- MasterDataSynchronizer sync = new MasterDataSynchronizer(data.getMasterData());
- sync.synchronizeMasterData();
+ masterDataSyncronizer.synchronizeMasterData();
}
private void processDeletions(ResourceListParserData data) throws NoSuchAlgorithmException, UnsupportedEncodingException
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MasterDataParser.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MasterDataParser.java
index 4ab55c835ec..65fb0e249c9 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MasterDataParser.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MasterDataParser.java
@@ -41,6 +41,7 @@ import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetKind;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetType;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataType;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataTypeCode;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.EntityKind;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.EntityType;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ExperimentType;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.MaterialType;
@@ -205,7 +206,7 @@ public class MasterDataParser
materialType.setDescription(getAttribute(materialTypeElement, "description"));
materialTypes.put(materialType.getCode(), materialType);
- parsePropertyAssignments(materialType, materialTypeElement.getElementsByTagName("propertyAssignments"));
+ parsePropertyAssignments(EntityKind.MATERIAL, materialType, materialTypeElement.getElementsByTagName("propertyAssignments"));
}
}
}
@@ -225,7 +226,7 @@ public class MasterDataParser
expType.setDescription(getAttribute(expTypeElement, "description"));
experimentTypes.put(expType.getCode(), expType);
- parsePropertyAssignments(expType, expTypeElement.getElementsByTagName("propertyAssignments"));
+ parsePropertyAssignments(EntityKind.EXPERIMENT, expType, expTypeElement.getElementsByTagName("propertyAssignments"));
}
}
}
@@ -251,7 +252,7 @@ public class MasterDataParser
sampleType.setGeneratedCodePrefix(getAttribute(sampleTypeElement, "generatedCodePrefix"));
sampleTypes.put(sampleType.getCode(), sampleType);
- parsePropertyAssignments(sampleType, sampleTypeElement.getElementsByTagName("propertyAssignments"));
+ parsePropertyAssignments(EntityKind.SAMPLE, sampleType, sampleTypeElement.getElementsByTagName("propertyAssignments"));
}
}
}
@@ -290,15 +291,16 @@ public class MasterDataParser
dataSetType.setDeletionDisallow(Boolean.valueOf(getAttribute(dataSetTypeElement, "deletionDisallowed")));
dataSetTypes.put(dataSetType.getCode(), dataSetType);
- parsePropertyAssignments(dataSetType, dataSetTypeElement.getElementsByTagName("propertyAssignments"));
+ parsePropertyAssignments(EntityKind.DATA_SET, dataSetType, dataSetTypeElement.getElementsByTagName("propertyAssignments"));
}
}
}
- private void parsePropertyAssignments(EntityType entityType, NodeList propertyAssignmentsNode)
+ private void parsePropertyAssignments(EntityKind entityKind, EntityType entityType, NodeList propertyAssignmentsNode)
{
if (propertyAssignmentsNode.getLength() == 1)
{
+ List<NewETPTAssignment> list = new ArrayList<NewETPTAssignment>();
Element propertyAssignmentsElement = (Element) propertyAssignmentsNode.item(0);
NodeList propertyAssignmentNodes = propertyAssignmentsElement.getElementsByTagName("propertyAssignment");
for (int i = 0; i < propertyAssignmentNodes.getLength(); i++)
@@ -314,15 +316,9 @@ public class MasterDataParser
assignment.setSection(getAttribute(propertyAssignmentElement, "section"));
assignment.setOrdinal(Long.valueOf(getAttribute(propertyAssignmentElement, "ordinal")));
assignment.setShownInEditView(Boolean.valueOf(getAttribute(propertyAssignmentElement, "showInEdit")));
-
- List<NewETPTAssignment> list = entityPropertyAssignments.get(entityType.getCode());
- if (list == null)
- {
- list = new ArrayList<NewETPTAssignment>();
- entityPropertyAssignments.put(entityType.getEntityKind().name(), entityType.getCode(), list);
- }
list.add(assignment);
}
+ entityPropertyAssignments.put(entityType.getEntityKind().name(), entityType.getCode(), list);
}
}
@@ -346,7 +342,7 @@ public class MasterDataParser
newPropertyType.setInternalNamespace(Boolean.valueOf(getAttribute(propertyTypeElement, "internalNamespace")));
newPropertyType.setCode(CodeConverter.tryToBusinessLayer(code, newPropertyType.isInternalNamespace()));
- propertyTypes.put(CodeConverter.tryToDatabase(code), newPropertyType);
+ propertyTypes.put(newPropertyType.getCode(), newPropertyType);
if (dataTypeCode.equals(DataTypeCode.CONTROLLEDVOCABULARY))
{
String vocabularyCode = getAttribute(propertyTypeElement, "vocabulary");
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MasterDataSynchronizer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MasterDataSynchronizer.java
index d57feaea6c2..3bd0f9d96c0 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MasterDataSynchronizer.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/MasterDataSynchronizer.java
@@ -56,18 +56,21 @@ public class MasterDataSynchronizer
final ResourceListParserData.MasterData masterData;
+ final Map<TechId, List<VocabularyTerm>> vocabularyTermsToBeDeleted;
+
public MasterDataSynchronizer(ResourceListParserData.MasterData masterData)
{
String openBisServerUrl = ServiceProvider.getConfigProvider().getOpenBisServerUrl();
this.sessionToken = ServiceProvider.getOpenBISService().getSessionToken();
this.commonServer = ServiceFinderUtils.getCommonServer(sessionToken, openBisServerUrl);
this.masterData = masterData;
+ vocabularyTermsToBeDeleted = new HashMap<TechId, List<VocabularyTerm>>();
}
public void synchronizeMasterData() {
MultiKeyMap<String, List<NewETPTAssignment>> propertyAssignmentsToProcess = masterData.getPropertyAssignmentsToProcess();
processVocabularies(masterData.getVocabulariesToProcess());
- // materials are registered but their property assignments are deferred untilafter property types are processed
+ // materials are registered but their property assignments are deferred until after property types are processed
processEntityTypes(masterData.getMaterialTypesToProcess(), propertyAssignmentsToProcess);
processPropertyTypes(masterData.getPropertyTypesToProcess());
processEntityTypes(masterData.getSampleTypesToProcess(), propertyAssignmentsToProcess);
@@ -76,6 +79,15 @@ public class MasterDataSynchronizer
processDeferredMaterialTypePropertyAssignments(propertyAssignmentsToProcess);
}
+ public void cleanupUnusedMasterData()
+ {
+ for (TechId vocabularyId : vocabularyTermsToBeDeleted.keySet())
+ {
+ commonServer.deleteVocabularyTerms(sessionToken, vocabularyId, vocabularyTermsToBeDeleted.get(vocabularyId),
+ Collections.<ch.systemsx.cisd.openbis.generic.shared.basic.dto.VocabularyTermReplacement> emptyList());
+ }
+ }
+
private void processDeferredMaterialTypePropertyAssignments(MultiKeyMap<String, List<NewETPTAssignment>> propertyAssignmentsToProcess)
{
List<? extends EntityType> existingEntityTypes = getExistingEntityTypes(EntityKind.MATERIAL);
@@ -134,7 +146,6 @@ public class MasterDataSynchronizer
List<VocabularyTerm> termsToBeAdded = new ArrayList<VocabularyTerm>();
List<VocabularyTerm> termsToBeUpdated = new ArrayList<VocabularyTerm>();
- List<VocabularyTerm> termsToBeDeleted = new ArrayList<VocabularyTerm>();
for (VocabularyTerm incomingTerm : incomingTerms)
{
String termCode = incomingTerm.getCode();
@@ -152,6 +163,7 @@ public class MasterDataSynchronizer
}
}
+ List<VocabularyTerm> termsToBeDeleted = new ArrayList<VocabularyTerm>();
for (VocabularyTerm term : existingVocabulary.getTerms())
{
String termCode = term.getCode();
@@ -160,16 +172,17 @@ public class MasterDataSynchronizer
termsToBeDeleted.add(term);
}
}
+
+ // defer deletions of vocabulary terms until after the entities referencing the terms are synced
+ if (termsToBeDeleted.isEmpty() == false)
+ {
+ vocabularyTermsToBeDeleted.put(new TechId(existingVocabulary.getId()), termsToBeDeleted);
+ }
commonServer.addVocabularyTerms(sessionToken, new TechId(existingVocabulary.getId()), termsToBeAdded, null, true);
for (VocabularyTerm term : termsToBeUpdated)
{
commonServer.updateVocabularyTerm(sessionToken, term);
}
- if (termsToBeDeleted.isEmpty() == false)
- {
- commonServer.deleteVocabularyTerms(sessionToken, new TechId(existingVocabulary.getId()), termsToBeDeleted,
- Collections.<ch.systemsx.cisd.openbis.generic.shared.basic.dto.VocabularyTermReplacement> emptyList());
- }
}
private VocabularyTerm findTermInVocabulary(Vocabulary existingVocabulary, String termCode)
@@ -236,20 +249,13 @@ public class MasterDataSynchronizer
@SuppressWarnings("rawtypes")
private void processPropertyAssignments(EntityType existingEntityType,
- List<NewETPTAssignment> list)
+ List<NewETPTAssignment> incomingAssignmentsList)
{
- assert list != null : "Null assignments list";
- for (NewETPTAssignment newETPTAssignment : list)
+ assert incomingAssignmentsList != null : "Null assignments list";
+ List<? extends EntityTypePropertyType> assignedPropertyTypes = existingEntityType.getAssignedPropertyTypes();
+ for (NewETPTAssignment newETPTAssignment : incomingAssignmentsList)
{
- List<? extends EntityTypePropertyType> assignedPropertyTypes = existingEntityType.getAssignedPropertyTypes();
- boolean found = false;
- for (EntityTypePropertyType entityTypePropertyType : assignedPropertyTypes)
- {
- if (entityTypePropertyType.getPropertyType().getCode().equals(newETPTAssignment.getPropertyTypeCode()))
- {
- found = true;
- }
- }
+ boolean found = findInExistingPropertyAssignments(newETPTAssignment, assignedPropertyTypes);
if (found)
{
commonServer.updatePropertyTypeAssignment(sessionToken, newETPTAssignment);
@@ -259,6 +265,42 @@ public class MasterDataSynchronizer
commonServer.assignPropertyType(sessionToken, newETPTAssignment);
}
}
+ // remove property assignments that are no longer valid
+ for (EntityTypePropertyType etpt : assignedPropertyTypes)
+ {
+ if (findInIncomingPropertyAssignments(etpt, incomingAssignmentsList) == false)
+ {
+ commonServer.unassignPropertyType(sessionToken, existingEntityType.getEntityKind(), etpt.getPropertyType().getCode(), etpt
+ .getEntityType().getCode());
+ }
+ }
+ }
+
+ @SuppressWarnings("rawtypes")
+ private boolean findInIncomingPropertyAssignments(EntityTypePropertyType existingEtpt, List<NewETPTAssignment> incomingAssignmentsList)
+ {
+ for (NewETPTAssignment newETPTAssignment : incomingAssignmentsList)
+ {
+ if (newETPTAssignment.getPropertyTypeCode().equals(existingEtpt.getPropertyType().getCode()))
+ {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @SuppressWarnings("rawtypes")
+ private boolean findInExistingPropertyAssignments(NewETPTAssignment incomingETPTAssignment,
+ List<? extends EntityTypePropertyType> assignedPropertyTypes)
+ {
+ for (EntityTypePropertyType entityTypePropertyType : assignedPropertyTypes)
+ {
+ if (entityTypePropertyType.getPropertyType().getCode().equals(incomingETPTAssignment.getPropertyTypeCode()))
+ {
+ return true;
+ }
+ }
+ return false;
}
private void registerEntityType(EntityKind entityKind, EntityType incomingEntityType)
--
GitLab