diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/EntityRetriever.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/EntityRetriever.java
index 838e151941cf6e06de69c23e32db7a9f1e4cb4d4..07c7d77ba79fec4581c0720c3f68342915fc065a 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/EntityRetriever.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/EntityRetriever.java
@@ -52,15 +52,17 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.id.SpacePermId;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.search.SpaceSearchCriteria;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.entitygraph.Edge;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.entitygraph.EntityGraph;
+import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.entitygraph.IEntityRetriever;
+import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.entitygraph.INode;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.entitygraph.Node;
//import ch.ethz.sis.openbis.generic.shared.entitygraph.Edge;
//import ch.ethz.sis.openbis.generic.shared.entitygraph.EntityGraph;
//import ch.ethz.sis.openbis.generic.shared.entitygraph.Node;
import ch.systemsx.cisd.openbis.generic.server.jython.api.v1.IMasterDataRegistrationTransaction;
-public class EntityRetriever
+public class EntityRetriever implements IEntityRetriever
{
- private EntityGraph<Node<?>> graph = new EntityGraph<Node<?>>();
+ private EntityGraph<INode> graph = new EntityGraph<INode>();
private final IApplicationServerApi v3Api;
@@ -86,7 +88,8 @@ public class EntityRetriever
return new EntityRetriever(v3Api, sessionToken, null);
}
- public EntityGraph<Node<?>> getEntityGraph(String spaceId)
+ @Override
+ public EntityGraph<INode> getEntityGraph(String spaceId)
{
buildEntityGraph(spaceId);
return graph;
@@ -110,9 +113,9 @@ public class EntityRetriever
return v3Api.searchSpaces(sessionToken, new SpaceSearchCriteria(), new SpaceFetchOptions()).getObjects();
}
- public void buildEntityGraph(String spaceId)
+ private void buildEntityGraph(String spaceId)
{
- graph = new EntityGraph<Node<?>>();
+ graph = new EntityGraph<INode>();
// add shared samples
/*
@@ -383,6 +386,7 @@ public class EntityRetriever
}
}
+ @Override
public List<Material> fetchMaterials()
{
MaterialSearchCriteria criteria = new MaterialSearchCriteria();
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/SkinnyEntityRetriever.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/SkinnyEntityRetriever.java
new file mode 100644
index 0000000000000000000000000000000000000000..87b3d19428ccc44dc1c552c5c85c4ec5c52bc59c
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/SkinnyEntityRetriever.java
@@ -0,0 +1,463 @@
+/*
+ * Copyright 2016 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common;
+
+import static ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.entitygraph.Edge.CHILD;
+import static ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.entitygraph.Edge.COMPONENT;
+import static ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.entitygraph.Edge.CONNECTION;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.TransformerException;
+
+import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.interfaces.ICodeHolder;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.search.SearchResult;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.fetchoptions.DataSetFetchOptions;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.search.DataSetSearchCriteria;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.fetchoptions.ExperimentFetchOptions;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.id.ExperimentIdentifier;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.id.ExperimentPermId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.search.ExperimentSearchCriteria;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.material.Material;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.material.fetchoptions.MaterialFetchOptions;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.material.search.MaterialSearchCriteria;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.project.Project;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.project.fetchoptions.ProjectFetchOptions;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.project.search.ProjectSearchCriteria;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.fetchoptions.SampleFetchOptions;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.id.SampleIdentifier;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.id.SamplePermId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.search.SampleSearchCriteria;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.Space;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.fetchoptions.SpaceFetchOptions;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.id.ISpaceId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.id.SpacePermId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.search.SpaceSearchCriteria;
+import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.entitygraph.Edge;
+import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.entitygraph.EntityGraph;
+import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.entitygraph.IEntityRetriever;
+import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.entitygraph.INode;
+import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.entitygraph.SkinnyNode;
+//import ch.ethz.sis.openbis.generic.shared.entitygraph.Edge;
+//import ch.ethz.sis.openbis.generic.shared.entitygraph.EntityGraph;
+//import ch.ethz.sis.openbis.generic.shared.entitygraph.Node;
+import ch.systemsx.cisd.openbis.generic.server.jython.api.v1.IMasterDataRegistrationTransaction;
+
+public class SkinnyEntityRetriever implements IEntityRetriever
+{
+ private EntityGraph<INode> graph = new EntityGraph<INode>();
+
+ private final IApplicationServerApi v3Api;
+
+ private final IMasterDataRegistrationTransaction masterDataRegistrationTransaction;
+
+ private final String sessionToken;
+
+ private SkinnyEntityRetriever(IApplicationServerApi v3Api, String sessionToken, IMasterDataRegistrationTransaction masterDataRegistrationTransaction)
+ {
+ this.v3Api = v3Api;
+ this.sessionToken = sessionToken;
+ this.masterDataRegistrationTransaction = masterDataRegistrationTransaction;
+ }
+
+ public static SkinnyEntityRetriever createWithMasterDataRegistationTransaction(IApplicationServerApi v3Api, String sessionToken,
+ IMasterDataRegistrationTransaction masterDataRegistrationTransaction)
+ {
+ return new SkinnyEntityRetriever(v3Api, sessionToken, masterDataRegistrationTransaction);
+ }
+
+ public static SkinnyEntityRetriever createWithSessionToken(IApplicationServerApi v3Api, String sessionToken)
+ {
+ return new SkinnyEntityRetriever(v3Api, sessionToken, null);
+ }
+
+ @Override
+ public EntityGraph<INode> getEntityGraph(String spaceId)
+ {
+ buildEntityGraph(spaceId);
+ return graph;
+ }
+
+ public boolean spaceExists(String spaceId)
+ {
+ SpacePermId spacePermId = new SpacePermId(spaceId);
+ Map<ISpaceId, Space> map =
+ v3Api.getSpaces(sessionToken,
+ Arrays.asList(spacePermId),
+ new SpaceFetchOptions());
+ return map.get(spacePermId) != null;
+ }
+
+ /**
+ * Returns spaces the logged in user is allowed to see
+ */
+ public List<Space> getSpaces()
+ {
+ return v3Api.searchSpaces(sessionToken, new SpaceSearchCriteria(), new SpaceFetchOptions()).getObjects();
+ }
+
+ public void buildEntityGraph(String spaceId)
+ {
+ graph = new EntityGraph<INode>();
+
+ // add shared samples
+ /*
+ * adding shared samples to the entity graph for a space means if we are synching from multiple spaces, each entity graph (for each space)
+ * will have the shared entity. When we add them to the RL (Resource List) in the data source servlet, any duplicate will throw an error when
+ * using the Resync library. To work around this we catch the duplicate exceptions where a shared sample is involved.
+ */
+ findSharedSamples();
+
+ // build the graph for the space from top-down starting from projects
+ ProjectSearchCriteria prjCriteria = new ProjectSearchCriteria();
+ prjCriteria.withSpace().withCode().thatEquals(spaceId);
+
+ ProjectFetchOptions projectFetchOptions = new ProjectFetchOptions();
+ projectFetchOptions.withSpace();
+
+ List<Project> projects = v3Api.searchProjects(sessionToken, prjCriteria, projectFetchOptions).getObjects();
+ for (Project project : projects)
+ {
+ SkinnyNode prjNode = new SkinnyNode(project.getPermId().toString(),
+ SyncEntityKind.PROJECT.getLabel(),
+ project.getIdentifier().toString(),
+ null,
+ project.getSpace(),
+ project.getCode());
+ graph.addNode(prjNode);
+ findExperiments(prjNode);
+ }
+
+ // add space samples
+ findSpaceSamples(spaceId);
+
+
+ // TODO logout?
+ // v3.logout(sessionToken);
+ }
+
+ // TODO temporary solution until V3 API SampleSearchCriteria.withoutSpace() is implemented
+ private void findSharedSamples()
+ {
+ SampleFetchOptions fetchOptions = new SampleFetchOptions();
+ fetchOptions.withType();
+ fetchOptions.withSpace();
+
+ List<Sample> samples = v3Api.searchSamples(sessionToken, new SampleSearchCriteria(), fetchOptions).getObjects();
+ for (Sample sample : samples)
+ {
+ if (sample.getSpace() == null)
+ {
+ INode sampleNode = new SkinnyNode(sample.getPermId().toString(),
+ SyncEntityKind.SAMPLE.getLabel(),
+ sample.getIdentifier().toString(),
+ sample.getType().getCode(),
+ null,
+ sample.getCode());
+ graph.addNode(sampleNode);
+ findChildAndComponentSamples(sampleNode);
+ findAndAttachDataSets(sampleNode);
+ }
+ }
+ }
+
+ private void findExperiments(INode prjNode)
+ {
+ ExperimentSearchCriteria criteria = new ExperimentSearchCriteria();
+ criteria.withProject().withCode().thatEquals(prjNode.getCode());
+ criteria.withProject().withSpace().withCode().thatEquals(prjNode.getSpace().getCode());
+ ExperimentFetchOptions fetchOptions = new ExperimentFetchOptions();
+ fetchOptions.withProject().withSpace();
+ fetchOptions.withType();
+
+ List<Experiment> experiments = v3Api.searchExperiments(sessionToken, criteria, fetchOptions).getObjects();
+ for (Experiment exp : experiments)
+ {
+ INode expNode = new SkinnyNode(exp.getPermId().toString(),
+ SyncEntityKind.EXPERIMENT.getLabel(),
+ exp.getIdentifier().toString(),
+ exp.getType().getCode(),
+ exp.getProject().getSpace(),
+ exp.getCode());
+ graph.addEdge(prjNode, expNode, new Edge(CONNECTION));
+ findSamplesForExperiment(expNode);
+ findAndAttachDataSetsForExperiment(expNode);
+ }
+ }
+
+ private void findSpaceSamples(String spaceCode)
+ {
+ SampleSearchCriteria criteria = new SampleSearchCriteria();
+ criteria.withSpace().withCode().thatEquals(spaceCode);
+ criteria.withoutExperiment();
+ criteria.withAndOperator();
+
+ SampleFetchOptions fetchOptions = new SampleFetchOptions();
+ fetchOptions.withType();
+ fetchOptions.withSpace();
+
+ List<Sample> samples = v3Api.searchSamples(sessionToken, criteria, fetchOptions).getObjects();
+ for (Sample sample : samples)
+ {
+ INode sampleNode = new SkinnyNode(sample.getPermId().toString(),
+ SyncEntityKind.SAMPLE.getLabel(),
+ sample.getIdentifier().toString(),
+ sample.getType().getCode(),
+ sample.getSpace(),
+ sample.getCode());
+ graph.addNode(sampleNode);
+ findChildAndComponentSamples(sampleNode);
+ findAndAttachDataSets(sampleNode);
+ }
+ }
+
+ private void findSamplesForExperiment(INode expNode)
+ {
+ SampleSearchCriteria criteria = new SampleSearchCriteria();
+ criteria.withExperiment().withId().thatEquals(new ExperimentPermId(expNode.getPermId()));
+
+ SampleFetchOptions fetchOptions = new SampleFetchOptions();
+ fetchOptions.withDataSets();
+ fetchOptions.withType();
+ fetchOptions.withExperiment();
+ fetchOptions.withSpace();
+
+ List<Sample> samples = v3Api.searchSamples(sessionToken, criteria, fetchOptions).getObjects();
+ for (Sample sample : samples)
+ {
+ INode sampleNode = new SkinnyNode(sample.getPermId().toString(),
+ SyncEntityKind.SAMPLE.getLabel(),
+ sample.getIdentifier().toString(),
+ sample.getType().getCode(),
+ sample.getSpace(),
+ sample.getCode());
+ graph.addEdge(expNode, sampleNode, new Edge(CONNECTION));
+
+ findAndAttachDataSets(sampleNode);
+ findChildAndComponentSamples(sampleNode);
+ }
+ }
+
+ private void findAndAttachDataSetsForExperiment(INode expNode)
+ {
+ DataSetSearchCriteria dsCriteria = new DataSetSearchCriteria();
+ dsCriteria.withExperiment().withId().thatEquals(new ExperimentIdentifier(expNode.getIdentifier()));
+
+ DataSetFetchOptions dsFetchOptions = new DataSetFetchOptions();
+ dsFetchOptions.withType();
+ dsFetchOptions.withSample();
+ dsFetchOptions.withExperiment();
+ // dsFetchOptions.withProperties();
+
+ List<DataSet> dataSets = v3Api.searchDataSets(sessionToken, dsCriteria, dsFetchOptions).getObjects();
+ for (DataSet dataSet : dataSets)
+ {
+ INode dataSetNode = new SkinnyNode(dataSet.getPermId().toString(),
+ SyncEntityKind.DATA_SET.getLabel(),
+ dataSet.getCode().toString(),
+ dataSet.getType().getCode(),
+ null,
+ dataSet.getCode());
+ graph.addEdge(expNode, dataSetNode, new Edge(CONNECTION));
+ findChildAndContainedDataSets(dataSetNode);
+ }
+ }
+
+ private void findAndAttachDataSets(INode sampleNode)
+ {
+ DataSetSearchCriteria dsCriteria = new DataSetSearchCriteria();
+ dsCriteria.withSample().withId().thatEquals(new SampleIdentifier(sampleNode.getIdentifier()));
+
+ DataSetFetchOptions dsFetchOptions = new DataSetFetchOptions();
+ dsFetchOptions.withType();
+ dsFetchOptions.withSample();
+ dsFetchOptions.withExperiment();
+
+ List<DataSet> dataSets = v3Api.searchDataSets(sessionToken, dsCriteria, dsFetchOptions).getObjects();
+ for (DataSet dataSet : dataSets)
+ {
+ INode dataSetNode = new SkinnyNode(dataSet.getPermId().toString(),
+ SyncEntityKind.DATA_SET.getLabel(),
+ dataSet.getCode().toString(),
+ dataSet.getType().getCode(),
+ null,
+ dataSet.getCode());
+ graph.addEdge(sampleNode, dataSetNode, new Edge(CONNECTION));
+ findChildAndContainedDataSets(dataSetNode);
+ }
+ }
+
+ private void findChildAndComponentSamples(INode sampleNode)
+ {
+ SampleFetchOptions fetchOptions = new SampleFetchOptions();
+ fetchOptions.withType();
+ fetchOptions.withDataSets();
+ fetchOptions.withExperiment();
+ fetchOptions.withSpace();
+
+ // first find the children
+ if (graph.isVisitedAsParent(sampleNode.getIdentifier()) == false)
+ {
+ graph.markAsVisitedAsParent(sampleNode.getIdentifier());
+ findChildSamples(sampleNode, fetchOptions);
+ }
+
+ // then find contained samples
+ if (graph.isVisitedAsContainer(sampleNode.getIdentifier()) == false)
+ {
+ graph.markAsVisitedAsContainer(sampleNode.getIdentifier());
+ findComponentSamples(sampleNode, fetchOptions);
+ }
+ }
+
+ private void findComponentSamples(INode sampleNode, SampleFetchOptions fetchOptions)
+ {
+ SampleSearchCriteria criteria = new SampleSearchCriteria();
+ criteria.withContainer().withId().thatEquals(new SamplePermId(sampleNode.getPermId()));
+ List<Sample> components = v3Api.searchSamples(sessionToken, criteria, fetchOptions).getObjects();
+ for (Sample sample : components)
+ {
+ INode subSampleNode = new SkinnyNode(sample.getPermId().toString(),
+ SyncEntityKind.SAMPLE.getLabel(),
+ sample.getIdentifier().toString(),
+ sample.getType().getCode(),
+ sample.getSpace(),
+ sample.getCode());
+ graph.addEdge(sampleNode, subSampleNode, new Edge(COMPONENT));
+
+ findAndAttachDataSets(subSampleNode);
+ findChildAndComponentSamples(subSampleNode);
+ }
+ }
+
+ private void findChildSamples(INode sampleNode, SampleFetchOptions fetchOptions)
+ {
+ SampleSearchCriteria criteria = new SampleSearchCriteria();
+ criteria.withParents().withId().thatEquals(new SamplePermId(sampleNode.getPermId()));
+ List<Sample> children = v3Api.searchSamples(sessionToken, criteria, fetchOptions).getObjects();
+ for (Sample sample : children)
+ {
+ INode subSampleNode = new SkinnyNode(sample.getPermId().toString(),
+ SyncEntityKind.SAMPLE.getLabel(),
+ sample.getIdentifier().toString(),
+ sample.getType().getCode(),
+ sample.getSpace(),
+ sample.getCode());
+ graph.addEdge(sampleNode, subSampleNode, new Edge(CHILD));
+
+ findAndAttachDataSets(subSampleNode);
+ findChildAndComponentSamples(subSampleNode);
+ }
+ }
+
+ private void findChildAndContainedDataSets(INode dsNode)
+ {
+ DataSetFetchOptions dsFetchOptions = new DataSetFetchOptions();
+ dsFetchOptions.withType();
+ dsFetchOptions.withSample();
+ dsFetchOptions.withExperiment();
+
+ // first find the children
+ if (graph.isVisitedAsParent(dsNode.getIdentifier()) == false)
+ {
+ graph.markAsVisitedAsParent(dsNode.getIdentifier());
+ findChildDataSets(dsNode, dsFetchOptions);
+ }
+
+ // then find contained data sets
+ if (graph.isVisitedAsContainer(dsNode.getIdentifier()) == false)
+ {
+ graph.markAsVisitedAsContainer(dsNode.getIdentifier());
+ findComponentDataSets(dsNode, dsFetchOptions);
+ }
+ }
+
+ private void findComponentDataSets(INode dsNode, DataSetFetchOptions dsFetchOptions)
+ {
+ DataSetSearchCriteria criteria = new DataSetSearchCriteria();
+ criteria.withContainer().withId().thatEquals(new DataSetPermId(dsNode.getPermId()));
+ List<DataSet> components = v3Api.searchDataSets(sessionToken, criteria, dsFetchOptions).getObjects();
+ for (DataSet ds : components)
+ {
+ INode containedDsNode = new SkinnyNode(ds.getPermId().toString(),
+ SyncEntityKind.DATA_SET.getLabel(),
+ ds.getCode().toString(),
+ ds.getType().getCode(),
+ null,
+ ds.getCode());
+ graph.addEdge(dsNode, containedDsNode, new Edge(COMPONENT));
+ findChildAndContainedDataSets(containedDsNode);
+ }
+ }
+
+ private void findChildDataSets(INode dsNode, DataSetFetchOptions dsFetchOptions)
+ {
+ DataSetSearchCriteria criteria = new DataSetSearchCriteria();
+ criteria.withParents().withId().thatEquals(new DataSetPermId(dsNode.getPermId()));
+ List<DataSet> children = v3Api.searchDataSets(sessionToken, criteria, dsFetchOptions).getObjects();
+ for (DataSet ds : children)
+ {
+ INode childDsNode = new SkinnyNode(ds.getPermId().toString(),
+ SyncEntityKind.DATA_SET.getLabel(),
+ ds.getCode().toString(),
+ ds.getType().getCode(),
+ null,
+ ds.getCode());
+ graph.addEdge(dsNode, childDsNode, new Edge(CHILD));
+
+ findChildAndContainedDataSets(childDsNode);
+ }
+ }
+
+ @Override
+ public List<Material> fetchMaterials()
+ {
+ MaterialSearchCriteria criteria = new MaterialSearchCriteria();
+
+ final MaterialFetchOptions fetchOptions = new MaterialFetchOptions();
+ fetchOptions.withType();
+
+ SearchResult<Material> searchResult =
+ v3Api.searchMaterials(sessionToken, criteria, fetchOptions);
+
+ return searchResult.getObjects();
+ }
+
+ public String fetchMasterDataAsXML() throws ParserConfigurationException, TransformerException
+ {
+ MasterDataExtractor masterDataExtractor = new MasterDataExtractor(v3Api, sessionToken, masterDataRegistrationTransaction);
+ return masterDataExtractor.fetchAsXmlString();
+ }
+
+ protected List<String> extractCodes(List<? extends ICodeHolder> codeHolders)
+ {
+ List<String> codes = new ArrayList<>();
+ for (ICodeHolder codeHolder : codeHolders)
+ {
+ codes.add(codeHolder.getCode());
+ }
+ return codes;
+ }
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/entitygraph/EdgeNodePair.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/entitygraph/EdgeNodePair.java
index b2994b91579b7af66aef22f5813390078f70920b..663d9bf12b9435a91e3954a55842c057e671e266 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/entitygraph/EdgeNodePair.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/entitygraph/EdgeNodePair.java
@@ -19,9 +19,9 @@ public class EdgeNodePair
{
private final Edge edge;
- private final Node<?> node;
+ private final INode node;
- public EdgeNodePair(Edge edge, Node<?> node)
+ public EdgeNodePair(Edge edge, INode node)
{
this.edge = edge;
this.node = node;
@@ -32,7 +32,7 @@ public class EdgeNodePair
return edge;
}
- public Node<?> getNode()
+ public INode getNode()
{
return node;
}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/entitygraph/EntityGraph.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/entitygraph/EntityGraph.java
index 62f73ccf967bbea6d40d1b180a5ca82121cdfae8..73a156514e9811849b51cadf19de344faf131c54 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/entitygraph/EntityGraph.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/entitygraph/EntityGraph.java
@@ -27,11 +27,11 @@ import java.util.Set;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.SyncEntityKind;
-public class EntityGraph<N extends Node<?>>
+public class EntityGraph<N extends INode>
{
- private final Map<String, N> nodes;
+ private final Map<String, INode> nodes;
- private final Map<N, List<EdgeNodePair>> adjacencyMap;
+ private final Map<INode, List<EdgeNodePair>> adjacencyMap;
// the following flags make sure then we don't end up in a cycle when a sample is the component of a its child sample
// or when a data set is the component of its child data set
@@ -41,8 +41,8 @@ public class EntityGraph<N extends Node<?>>
public EntityGraph()
{
- this.nodes = new HashMap<String, N>();
- this.adjacencyMap = new HashMap<N, List<EdgeNodePair>>();
+ this.nodes = new HashMap<String, INode>();
+ this.adjacencyMap = new HashMap<INode, List<EdgeNodePair>>();
}
public boolean isVisitedAsParent(String identifier)
@@ -65,7 +65,7 @@ public class EntityGraph<N extends Node<?>>
visitedContainers.add(identifier);
}
- public void addEdge(N startNode, N endNode, Edge edge)
+ public void addEdge(INode startNode, INode endNode, Edge edge)
{
List<EdgeNodePair> adjacencyList = adjacencyMap.get(startNode);
if (adjacencyList == null)
@@ -86,12 +86,12 @@ public class EntityGraph<N extends Node<?>>
}
}
EdgeNodePair enPair = new EdgeNodePair(edge, endNode);
- startNode.addConnection(enPair);
+ // startNode.addConnection(enPair);
adjacencyList.add(enPair);
addNode(endNode);
}
- public void addNode(N node)
+ public void addNode(INode node)
{
String identifier = node.getIdentifier();
if (nodes.containsKey(identifier) == false)
@@ -103,9 +103,9 @@ public class EntityGraph<N extends Node<?>>
}
- public List<N> getNodes()
+ public List<INode> getNodes()
{
- return new ArrayList<N>(nodes.values());
+ return new ArrayList<INode>(nodes.values());
}
public String getEdgesForDOTRepresentation()
@@ -121,7 +121,7 @@ public class EntityGraph<N extends Node<?>>
private String getEdgesForNode(boolean forTest)
{
StringBuffer sb = new StringBuffer();
- for (Node<?> node : getNodes())
+ for (INode node : getNodes())
{
List<EdgeNodePair> list = adjacencyMap.get(node);
if (list.isEmpty() && node.getEntityKind().equals("DATA_SET") == false)
@@ -136,7 +136,7 @@ public class EntityGraph<N extends Node<?>>
}
for (EdgeNodePair edgeNodePair : list)
{
- Node<?> neighbourNode = edgeNodePair.getNode();
+ INode neighbourNode = edgeNodePair.getNode();
sb.append("\"" + node.getCode() + "(" + getDifferentiatorStr(node, forTest) + ")\" -> "
+ getRightHandNodeRep(neighbourNode, forTest));
if (edgeNodePair.getEdge().getType().equals(Edge.COMPONENT))
@@ -154,24 +154,20 @@ public class EntityGraph<N extends Node<?>>
return sb.toString();
}
- private String getRightHandNodeRep(Node<?> node, boolean forTest)
+ private String getRightHandNodeRep(INode node, boolean forTest)
{
return "\"" + node.getCode() + "(" + getDifferentiatorStr(node, forTest) + ")\"";
}
- private String getDifferentiatorStr(Node<?> node, boolean forTest)
+ private String getDifferentiatorStr(INode node, boolean forTest)
{
if (forTest == false)
{
String differentiatorStr = "";
- if (node.getEntityKind().equals(SyncEntityKind.EXPERIMENT.getLabel()) || node.getEntityKind().equals(SyncEntityKind.PROJECT.getLabel())) // in
- // order
- // to
- // differentiate
- // between
- // experiments/projects in the same space but
- // under different
+ // in order
+ // to differentiate between experiments/projects in the same space but under different
// projects/spaces
+ if (node.getEntityKind().equals(SyncEntityKind.EXPERIMENT.getLabel()) || node.getEntityKind().equals(SyncEntityKind.PROJECT.getLabel()))
{
differentiatorStr = node.getIdentifier();
;
@@ -218,10 +214,10 @@ public class EntityGraph<N extends Node<?>>
public void printGraph(String space)
{
- for (Node<?> node : nodes.values())
- {
- // printNeighboursForNode(node);
- }
+ // for (INode node : nodes.values())
+ // {
+ // printNeighboursForNode(node);
+ // }
printGraphInDOT(space);
}
@@ -247,7 +243,7 @@ public class EntityGraph<N extends Node<?>>
List<EdgeNodePair> neighboursForEntityWithIdentifier = getNeighboursForEntityWithIdentifier(nodeIdentifierFrom, null);
for (EdgeNodePair edgeNodePair : neighboursForEntityWithIdentifier)
{
- Node<?> connectedNode = edgeNodePair.getNode();
+ INode connectedNode = edgeNodePair.getNode();
String connectionType = edgeNodePair.getEdge().getType();
if (connectedNode.getIdentifier().equals(nodeIdentifierTo)
&& connType.equals(connectionType))
@@ -270,13 +266,13 @@ public class EntityGraph<N extends Node<?>>
public List<EdgeNodePair> getNeighboursForEntityWithIdentifier(String identifier, String connType)
{
- Node<?> node = getNodeForIdentifier(identifier);
+ INode node = getNodeForIdentifier(identifier);
return getNeighboursForEntity(node, connType);
}
public List<EdgeNodePair> getNeighboursForEntity(String permId, String connType)
{
- Node<?> node = getNodeWithPermId(permId);
+ INode node = getNodeWithPermId(permId);
// TODO throw an exception if node not found
if (node == null)
{
@@ -285,7 +281,7 @@ public class EntityGraph<N extends Node<?>>
return getNeighboursForEntity(node, connType);
}
- private List<EdgeNodePair> getNeighboursForEntity(Node<?> node, String connType)
+ private List<EdgeNodePair> getNeighboursForEntity(INode node, String connType)
{
if (adjacencyMap.containsKey(node) == false)
{
@@ -304,10 +300,10 @@ public class EntityGraph<N extends Node<?>>
return list;
}
- public Node<?> getNodeWithPermId(String permId)
+ public INode getNodeWithPermId(String permId)
{
// TODO make this more efficient by mapping nodes by permId as well
- for (Node<?> node : nodes.values())
+ for (INode node : nodes.values())
{
if (node.getPermId().equals(permId))
{
@@ -317,7 +313,7 @@ public class EntityGraph<N extends Node<?>>
return null;
}
- public Node<?> getNodeForIdentifier(String identifier)
+ public INode getNodeForIdentifier(String identifier)
{
return nodes.get(identifier);
// for (Node<?> node : nodes.values())
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/entitygraph/IEntityRetriever.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/entitygraph/IEntityRetriever.java
new file mode 100644
index 0000000000000000000000000000000000000000..0e78490e82e153412598162f55e6755d4ed3e2c1
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/entitygraph/IEntityRetriever.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2017 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.entitygraph;
+
+import java.util.List;
+
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.material.Material;
+
+/**
+ *
+ *
+ * @author Ganime Betul Akin
+ */
+public interface IEntityRetriever
+{
+ public EntityGraph<INode> getEntityGraph(String spaceId);
+
+ public List<Material> fetchMaterials();
+}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/entitygraph/INode.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/entitygraph/INode.java
new file mode 100644
index 0000000000000000000000000000000000000000..122f7f504531b4fab4eb44bbed38097d7076060d
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/entitygraph/INode.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2017 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.entitygraph;
+
+import java.util.Map;
+
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.Space;
+
+/**
+ *
+ *
+ * @author Ganime Betul Akin
+ */
+public interface INode
+{
+ public String getPermId();
+
+ public String getEntityKind();
+
+ public String getIdentifier();
+
+ public String getCode();
+
+ public String getTypeCodeOrNull();
+
+ public Space getSpace();
+
+ public Map<String, String> getPropertiesOrNull();
+
+ public void addConnection(EdgeNodePair enPair);
+}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/entitygraph/Node.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/entitygraph/Node.java
index ca062f7a3e1f57cafc303ab74384933cf6e01ec6..0468aba9952ee15d0961387a00cfa2b8d0d384e7 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/entitygraph/Node.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/entitygraph/Node.java
@@ -40,14 +40,15 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.Space;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.SyncEntityKind;
-public class Node<T extends ICodeHolder & IModificationDateHolder & IModifierHolder & IPermIdHolder & IRegistrationDateHolder & IRegistratorHolder>
+public class Node<T extends IModificationDateHolder & IModifierHolder & IRegistrationDateHolder & IRegistratorHolder & IPermIdHolder & ICodeHolder>
+ implements INode
{
- private final T entity;
-
private final List<EdgeNodePair> connections;
private final List<Attachment> attachments;
+ protected final T entity;
+
public T getEntity()
{
return entity;
@@ -93,16 +94,7 @@ public class Node<T extends ICodeHolder & IModificationDateHolder & IModifierHol
return true;
}
- public String getCode()
- {
- return this.entity.getCode();
- }
-
- public String getPermId()
- {
- return this.entity.getPermId().toString();
- }
-
+ @Override
public Map<String, String> getPropertiesOrNull()
{
if (IPropertiesHolder.class.isAssignableFrom(entity.getClass()) == false)
@@ -114,14 +106,7 @@ public class Node<T extends ICodeHolder & IModificationDateHolder & IModifierHol
public String getSpaceOrNull()
{
- if(entity instanceof Experiment) {
- return ((Experiment)entity).getProject().getSpace().getCode();
- }
- if (ISpaceHolder.class.isAssignableFrom(entity.getClass()) == false)
- {
- return null;
- }
- Space space = ((ISpaceHolder) entity).getSpace();
+ Space space = getSpace();
if (space == null)
{
return null;
@@ -182,49 +167,96 @@ public class Node<T extends ICodeHolder & IModificationDateHolder & IModifierHol
return null;
}
- public String getIdentifier()
+ @Override
+ public String getTypeCodeOrNull()
{
- if (entity instanceof Project)
+ if (IPropertiesHolder.class.isAssignableFrom(entity.getClass()) == false)
{
- return ((Project) entity).getIdentifier().getIdentifier();
+ return null;
}
- if (entity instanceof Experiment)
- {
- return ((Experiment) entity).getIdentifier().getIdentifier();
+ if(entity instanceof Sample) {
+ return ((Sample) entity).getType().getCode();
}
- if (entity instanceof Sample)
+ else if (entity instanceof Experiment)
{
- return ((Sample) entity).getIdentifier().getIdentifier();
+ return ((Experiment) entity).getType().getCode();
}
else if (entity instanceof DataSet)
{
- return ((DataSet) entity).getPermId().toString();
+ return ((DataSet) entity).getType().getCode();
}
// TODO exception
return null;
}
- public String getTypeCodeOrNull()
+ @Override
+ public void addConnection(EdgeNodePair enPair)
{
- if (IPropertiesHolder.class.isAssignableFrom(entity.getClass()) == false)
+ connections.add(enPair);
+ }
+
+ public List<EdgeNodePair> getConnections()
+ {
+ return connections;
+ }
+
+ public void setAttachments(List<Attachment> attachmentList)
+ {
+ attachments.clear();
+ attachments.addAll(attachmentList);
+ }
+
+ public List<Attachment> getAttachmentsOrNull()
+ {
+ if (entity instanceof IAttachmentsHolder)
{
- return null;
+ IAttachmentsHolder holder = (IAttachmentsHolder) entity;
+ return holder.getAttachments();
}
- if(entity instanceof Sample) {
- return ((Sample) entity).getType().getCode();
+ return null;
+ }
+
+ @Override
+ public String getPermId()
+ {
+ return this.entity.getPermId().toString();
+ }
+
+ @Override
+ public String getIdentifier()
+ {
+ if (entity instanceof Project)
+ {
+ return ((Project) entity).getIdentifier().getIdentifier();
}
- else if (entity instanceof Experiment)
+ if (entity instanceof Experiment)
{
- return ((Experiment) entity).getType().getCode();
+ return ((Experiment) entity).getIdentifier().getIdentifier();
+ }
+ if (entity instanceof Sample)
+ {
+ return ((Sample) entity).getIdentifier().getIdentifier();
}
else if (entity instanceof DataSet)
{
- return ((DataSet) entity).getType().getCode();
+ return ((DataSet) entity).getPermId().toString();
}
// TODO exception
return null;
}
+ @Override
+ public String getCode()
+ {
+ return this.entity.getCode();
+ }
+
+ public Date getRegistrationDate()
+ {
+ return entity.getRegistrationDate();
+ }
+
+ @Override
public String getEntityKind()
{
if (entity instanceof Project)
@@ -247,34 +279,22 @@ public class Node<T extends ICodeHolder & IModificationDateHolder & IModifierHol
return null;
}
- public Date getRegistrationDate()
- {
- return entity.getRegistrationDate();
- }
-
- public void addConnection(EdgeNodePair enPair)
- {
- connections.add(enPair);
- }
-
- public List<EdgeNodePair> getConnections()
- {
- return connections;
- }
-
- public void setAttachments(List<Attachment> attachmentList)
- {
- attachments.clear();
- attachments.addAll(attachmentList);
- }
-
- public List<Attachment> getAttachmentsOrNull()
+ @Override
+ public Space getSpace()
{
- if (entity instanceof IAttachmentsHolder)
+ if (entity instanceof Experiment)
{
- IAttachmentsHolder holder = (IAttachmentsHolder) entity;
- return holder.getAttachments();
+ return ((Experiment) entity).getProject().getSpace();
}
- return null;
+ if (ISpaceHolder.class.isAssignableFrom(entity.getClass()) == false)
+ {
+ return null;
+ }
+ Space space = ((ISpaceHolder) entity).getSpace();
+ if (space == null)
+ {
+ return null;
+ }
+ return space;
}
}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/entitygraph/SkinnyNode.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/entitygraph/SkinnyNode.java
new file mode 100644
index 0000000000000000000000000000000000000000..278c2462c101f967772f9ee30876c334aa6524cb
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/common/entitygraph/SkinnyNode.java
@@ -0,0 +1,96 @@
+/*
+ * Copyright 2017 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.entitygraph;
+
+import java.util.Map;
+
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.Space;
+
+/**
+ * @author Ganime Betul Akin
+ */
+public class SkinnyNode implements INode
+{
+ private final String permId;
+
+ private final String entityKind;
+
+ private final String identifier;
+
+ private final String typeCodeOrNull;
+
+ private final Space space;
+
+ private final String code;
+
+ public Space getSpace()
+ {
+ return space;
+ }
+
+ public SkinnyNode(String permId, String entityKind, String identifier, String typeCodeOrNull, Space space, String code)
+ {
+ this.permId = permId;
+ this.entityKind = entityKind;
+ this.identifier = identifier;
+ this.typeCodeOrNull = typeCodeOrNull;
+ this.space = space;
+ this.code = code;
+ }
+ @Override
+ public String getPermId()
+ {
+ return permId;
+ }
+
+ @Override
+ public String getEntityKind()
+ {
+ return entityKind;
+ }
+
+ @Override
+ public String getIdentifier()
+ {
+ return identifier;
+ }
+
+ @Override
+ public String getTypeCodeOrNull()
+ {
+ return typeCodeOrNull;
+ }
+
+ @Override
+ public String getCode()
+ {
+ return this.code;
+ }
+
+ @Override
+ public Map<String, String> getPropertiesOrNull()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void addConnection(EdgeNodePair enPair)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/HarvesterMaintenanceTask.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/HarvesterMaintenanceTask.java
index 468b0e11a36d51ac3269a08d6672a4704989995f..8eb70491134647ce980d544f0fc36b9dd9652b79 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/HarvesterMaintenanceTask.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/HarvesterMaintenanceTask.java
@@ -199,7 +199,8 @@ public class HarvesterMaintenanceTask<T extends DataSetInformation> implements I
Date newCutOffTimestamp = new Date();
EntitySynchronizer synchronizer =
- new EntitySynchronizer(service, dataStoreCode, storeRoot, cutOffTimestamp, notSyncedDataSetCodes,
+ new EntitySynchronizer(service, dataStoreCode, storeRoot, cutOffTimestamp, timestamps.lastIncSyncTimestamp,
+ notSyncedDataSetCodes,
blackListedDataSetCodes,
notSyncedAttachmentHolderCodes,
context, config,
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
index 6c1b5f6b6f06f6d6da27fcf49b31458999d1f71a..5fe089be8e984157f09eb7b1c2aaa0de2fd3363a 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/EntitySynchronizer.java
@@ -59,10 +59,11 @@ import ch.ethz.sis.openbis.generic.dssapi.v3.IDataStoreServerApi;
import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile;
import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.fetchoptions.DataSetFileFetchOptions;
import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.search.DataSetFileSearchCriteria;
-import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.EntityRetriever;
+import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.SkinnyEntityRetriever;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.SyncEntityKind;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.entitygraph.EntityGraph;
-import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.entitygraph.Node;
+import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.entitygraph.IEntityRetriever;
+import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.common.entitygraph.INode;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.config.ParallelizedExecutionPreferences;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.config.SyncConfig;
import ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.ResourceListParserData.Connection;
@@ -150,6 +151,10 @@ public class EntitySynchronizer
private final Date lastSyncTimestamp;
+ // the following indicates the actual value in the file, not the cutoff calculated according to full sync prefs
+ // it is later read in data set deletions.
+ private final Date lastIncSyncTimestamp;
+
private final Set<String> dataSetsCodesToRetry;
private final Set<String> attachmentHolderCodesToRetry;
@@ -161,7 +166,8 @@ public class EntitySynchronizer
private final Set<String> blackListedDataSetCodes;
public EntitySynchronizer(IEncapsulatedOpenBISService service, String dataStoreCode, File storeRoot, Date lastSyncTimestamp,
- Set<String> dataSetsCodesToRetry, Set<String> blackListedDataSetCodes, Set<String> attachmentHolderCodesToRetry,
+ Date lastIncSyncTimestamp, Set<String> dataSetsCodesToRetry, Set<String> blackListedDataSetCodes,
+ Set<String> attachmentHolderCodesToRetry,
DataSetProcessingContext context,
SyncConfig config, Logger operationLog)
{
@@ -169,6 +175,7 @@ public class EntitySynchronizer
this.dataStoreCode = dataStoreCode;
this.storeRoot = storeRoot;
this.lastSyncTimestamp = lastSyncTimestamp;
+ this.lastIncSyncTimestamp = lastIncSyncTimestamp;
this.dataSetsCodesToRetry = dataSetsCodesToRetry;
this.attachmentHolderCodesToRetry = attachmentHolderCodesToRetry;
this.blackListedDataSetCodes = blackListedDataSetCodes;
@@ -232,9 +239,11 @@ public class EntitySynchronizer
verboseLogEntityOperations(builder.getDetails());
}
+ // MultiKeyMap<String, String> newEntities = new MultiKeyMap<String, String>();
if (config.isDryRun() == false)
{
AtomicEntityOperationResult operationResult = service.performEntityOperations(builder.getDetails());
+ // newEntities = getNewEntities(builder);
operationLog.info("Entity operation result: " + operationResult);
}
operationLog.info("\n");
@@ -244,7 +253,7 @@ public class EntitySynchronizer
if (config.isVerbose())
{
- verboseLogProcessAttachments(attachmentHoldersToProcess);
+ verboseLogProcessAttachments(attachmentHoldersToProcess); // newEntities
}
List<String> notSyncedAttachmentsHolders = new ArrayList<String>();
if (config.isDryRun() == false)
@@ -306,11 +315,32 @@ public class EntitySynchronizer
return data.getResourceListTimestamp();
}
+ private MultiKeyMap<String, String> getNewEntities(AtomicEntityOperationDetailsBuilder builder)
+ {
+ MultiKeyMap<String, String> newEntities = new MultiKeyMap<String, String>();
+ List<NewSample> sampleRegistrations = builder.getDetails().getSampleRegistrations();
+ for (NewSample newSample : sampleRegistrations)
+ {
+ newEntities.put(SyncEntityKind.SAMPLE.getLabel(), newSample.getPermID(), newSample.getIdentifier());
+ }
+ List<NewExperiment> experimentRegistrations = builder.getDetails().getExperimentRegistrations();
+ for (NewExperiment newExperiment : experimentRegistrations)
+ {
+ newEntities.put(SyncEntityKind.EXPERIMENT.getLabel(), newExperiment.getPermID(), newExperiment.getIdentifier());
+ }
+ List<NewProject> projectRegistrations = builder.getDetails().getProjectRegistrations();
+ for (NewProject newProject : projectRegistrations)
+ {
+ newEntities.put(SyncEntityKind.PROJECT.getLabel(), newProject.getPermID(), newProject.getIdentifier());
+ }
+ return newEntities;
+ }
+
private void verboseLogPhysicalDataSetRegistrations(Map<String, IncomingDataSet> physicalDSMap)
{
if (physicalDSMap.isEmpty() == false)
{
- operationLog.info("-------The following physical data sets will be registered-------");
+ operationLog.info("-------The following physical data sets will be processed-------");
for (String dsCode : physicalDSMap.keySet())
{
operationLog.info(dsCode);
@@ -318,13 +348,22 @@ public class EntitySynchronizer
}
}
- private void verboseLogProcessAttachments(List<IncomingEntity<?>> attachmentHoldersToProcess)
+ private void verboseLogProcessAttachments(List<IncomingEntity<?>> attachmentHoldersToProcess) // MultiKeyMap<String, String> newEntities
{
if (attachmentHoldersToProcess.isEmpty() == false)
{
operationLog.info("-------Attachments for the following entities will be processed-------");
for (IncomingEntity<?> holder : attachmentHoldersToProcess)
{
+// // the following is done to not list holders in the log when they are just being created and have no attachments
+// // updated ones will logged because the attachments might have been deleted.
+// if (newEntities.containsKey(holder.getEntityKind().getLabel(), holder.getPermID()) == true)
+// {
+// if (holder.hasAttachments() == false)
+// {
+// continue;
+// }
+// }
operationLog.info(holder.getIdentifer());
}
}
@@ -731,8 +770,8 @@ public class EntitySynchronizer
private void processDeletions(ResourceListParserData data) throws NoSuchAlgorithmException, UnsupportedEncodingException
{
String sessionToken = ServiceProvider.getOpenBISService().getSessionToken();
- EntityRetriever entityRetriever =
- EntityRetriever.createWithSessionToken(ServiceProvider.getV3ApplicationService(), sessionToken);
+ IEntityRetriever entityRetriever =
+ SkinnyEntityRetriever.createWithSessionToken(ServiceProvider.getV3ApplicationService(), sessionToken);
Set<String> incomingProjectPermIds = data.getProjectsToProcess().keySet();
Set<String> incomingExperimentPermIds = data.getExperimentsToProcess().keySet();
@@ -752,9 +791,9 @@ public class EntitySynchronizer
// first find out the entities to be deleted
for (String harvesterSpaceId : data.getHarvesterSpaceList())
{
- EntityGraph<Node<?>> harvesterEntityGraph = entityRetriever.getEntityGraph(harvesterSpaceId);
- List<Node<?>> entities = harvesterEntityGraph.getNodes();
- for (Node<?> entity : entities)
+ EntityGraph<INode> harvesterEntityGraph = entityRetriever.getEntityGraph(harvesterSpaceId);
+ List<INode> entities = harvesterEntityGraph.getNodes();
+ for (INode entity : entities)
{
String permId = entity.getPermId();
String identifier = entity.getIdentifier();
@@ -817,12 +856,14 @@ public class EntitySynchronizer
}
else
{
- if (dsWithConns.getKind() == DataSetKind.PHYSICAL && dsWithConns.getLastModificationDate().after(lastSyncTimestamp))
+ if (dsWithConns.getKind() == DataSetKind.PHYSICAL && dsWithConns.getLastModificationDate().after(lastIncSyncTimestamp))
{
PhysicalDataSet physicalDS = service.tryGetDataSet(permId).tryGetAsDataSet();
sameDS = deepCompareDataSets(permId);
if (sameDS == false)
+ {
physicalDataSetsDelete.add(physicalDS);
+ }
}
}
if (sameDS == false)
@@ -909,32 +950,36 @@ public class EntitySynchronizer
operationLog.warn("One or more materials could not be deleted due to: " + e.getMessage());
}
- // StringBuffer summary = new StringBuffer();
- // if (projectsToDelete.size() > 0)
- // {
- // summary.append(projectsToDelete.size() + " projects,");
- // }
- // if (materialdsToDelete.size() > 0)
- // {
- // summary.append(materialdsToDelete.size() + " materials,");
- // }
- // if (expDeletionId != null)
- // {
- // summary.append(experimentsToDelete.size() + " experiments,");
- // }
- // if (smpDeletionId != null)
- // {
- // summary.append(samplesToDelete.size() + " samples,");
- // }
- // if (dsDeletionId != null)
- // {
- // summary.append(dataSetsToDelete.size() + " data sets");
- // }
- //
- // if (summary.length() > 0)
- // {
- // operationLog.info(summary.substring(0, summary.length() - 1) + " have been deleted:");
- // }
+ // The following summary is not accurate if an error occurs in material deletions
+ StringBuffer summary = new StringBuffer();
+ if (projectsToDelete.size() > 0)
+ {
+ summary.append(projectsToDelete.size() + " projects,");
+ }
+ if (materialsToDelete.size() > 0)
+ {
+ summary.append(materialsToDelete.size() + " materials,");
+ }
+ if (expDeletionId != null)
+ {
+ summary.append(experimentsToDelete.size() + " experiments,");
+ }
+ if (smpDeletionId != null)
+ {
+ summary.append(samplesToDelete.size() + " samples,");
+ }
+ if (dsDeletionId != null)
+ {
+ summary.append(dataSetsToDelete.size() + " data sets");
+ }
+ if (summary.length() > 0)
+ {
+ operationLog.info(summary.substring(0, summary.length() - 1) + " have been deleted:");
+ }
+ else
+ {
+ operationLog.info("Nothing has been deleted:");
+ }
for (PhysicalDataSet physicalDS : physicalDataSetsDelete)
{
operationLog.info("Is going to delete the location: " + physicalDS.getLocation());
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParser.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParser.java
index c1bcc519236254d55a06f1dd2d1914579f8274bf..808aa9296935ed1ddcc0c3e9ad31bd553a95eadd 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParser.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/ResourceListParser.java
@@ -446,6 +446,7 @@ public class ResourceListParser
{
Element docElement = (Element) xdNode;
NodeList connsNode = docElement.getElementsByTagName("x:binaryData");
+ // if a sample/experiment/project node has binaryData element, it can only be because of attachments
if (connsNode.getLength() == 1)
{
return true;