From 2fd306ff84d63cc229558e5fde942a8ee4c72f89 Mon Sep 17 00:00:00 2001
From: gakin <gakin>
Date: Mon, 3 Oct 2016 15:10:14 +0000
Subject: [PATCH] SSDM-4197 : Refactoring
SVN: 37142
---
.../config/BasicAuthCredentials.java | 68 ++++++++++
.../plugins/harvester/config/SyncConfig.java | 58 ++++-----
.../config/SynchronizationConfigReader.java | 9 +-
.../harvester/synchronizer/DSSFileUtils.java | 80 ++++++++++++
.../DataSetRegistrationIngestionService.java | 25 +---
.../synchronizer/DataSourceConnector.java | 118 ++++++++++++++++++
.../synchronizer/EntitySynchronizer.java | 96 +++-----------
7 files changed, 313 insertions(+), 141 deletions(-)
create mode 100644 datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/config/BasicAuthCredentials.java
create mode 100644 datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/DSSFileUtils.java
create mode 100644 datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/DataSourceConnector.java
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/config/BasicAuthCredentials.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/config/BasicAuthCredentials.java
new file mode 100644
index 00000000000..396c88fea6e
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/config/BasicAuthCredentials.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2016 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.harvester.config;
+
+/**
+ *
+ *
+ * @author Ganime Betul Akin
+ */
+public class BasicAuthCredentials
+{
+ private String realm;
+
+ private String user;
+
+ private String password;
+
+ public BasicAuthCredentials(String realm, String user, String pass)
+ {
+ this.realm = realm;
+ this.user = user;
+ this.password = pass;
+ }
+
+ public String getRealm()
+ {
+ return realm;
+ }
+
+ public void setRealm(String realm)
+ {
+ this.realm = realm;
+ }
+
+ public String getUser()
+ {
+ return user;
+ }
+
+ public void setUser(String user)
+ {
+ this.user = user;
+ }
+
+ public String getPassword()
+ {
+ return password;
+ }
+
+ public void setPassword(String pass)
+ {
+ this.password = pass;
+ }
+}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/config/SyncConfig.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/config/SyncConfig.java
index 3185266c67c..29329b4297c 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/config/SyncConfig.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/config/SyncConfig.java
@@ -62,36 +62,6 @@ public class SyncConfig
this.dataSourceDSSURL = dataSourceDSSURL;
}
- public String getRealm()
- {
- return realm;
- }
-
- public void setRealm(String realm)
- {
- this.realm = realm;
- }
-
- public String getUser()
- {
- return user;
- }
-
- public void setUser(String user)
- {
- this.user = user;
- }
-
- public String getPass()
- {
- return pass;
- }
-
- public void setPass(String pass)
- {
- this.pass = pass;
- }
-
public String getLastSyncTimestampFileName()
{
return lastSyncTimestampFileName;
@@ -171,16 +141,12 @@ public class SyncConfig
}
}
+ private BasicAuthCredentials auth;
+
private String dataSourceOpenbisURL;
private String dataSourceDSSURL;
- private String realm;
-
- private String user;
-
- private String pass;
-
private String lastSyncTimestampFileName;
private String dataSourcePrefix;
@@ -226,4 +192,24 @@ public class SyncConfig
this.emailAddresses.add(new EMailAddress(token.trim()));
}
}
+
+ public void setAuthCredentials(String realm, String user, String pass)
+ {
+ this.auth = new BasicAuthCredentials(realm, user, pass);
+ }
+
+ public BasicAuthCredentials getAuthenticationCredentials()
+ {
+ return auth;
+ }
+
+ public String getUser()
+ {
+ return this.auth.getUser();
+ }
+
+ public String getPassword()
+ {
+ return this.auth.getPassword();
+ }
}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/config/SynchronizationConfigReader.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/config/SynchronizationConfigReader.java
index 79ded6289bf..10e75638646 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/config/SynchronizationConfigReader.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/config/SynchronizationConfigReader.java
@@ -81,15 +81,16 @@ public class SynchronizationConfigReader
config.setLogFilePath(reader.getString(section, LOG_FILE_PROPERTY_NAME, DEFAULT_LOG_FILE_NAME, false));
if (config.getLogFilePath() != null)
{
- // configureFileAppender(config, logger);
+ configureFileAppender(config, logger);
}
config.setDataSourceURI(reader.getString(section, DATA_SOURCE_URL_PROPERTY_NAME, null, true));
config.setDataSourceOpenbisURL(reader.getString(section, DATA_SOURCE_OPENBIS_URL_PROPERTY_NAME, null, true));
config.setDataSourceDSSURL(reader.getString(section, DATA_SOURCE_DSS_URL_PROPERTY_NAME, null, true));
- config.setRealm(reader.getString(section, DATA_SOURCE_AUTH_REALM_PROPERTY_NAME, null, true));
- config.setUser(reader.getString(section, DATA_SOURCE_AUTH_USER_PROPERTY_NAME, null, true));
- config.setPass(reader.getString(section, DATA_SOURCE_AUTH_PASS_PROPERTY_NAME, null, true));
+ String realm = reader.getString(section, DATA_SOURCE_AUTH_REALM_PROPERTY_NAME, null, true);
+ String user = reader.getString(section, DATA_SOURCE_AUTH_USER_PROPERTY_NAME, null, true);
+ String pass = reader.getString(section, DATA_SOURCE_AUTH_PASS_PROPERTY_NAME, null, true);
+ config.setAuthCredentials(realm, user, pass);
config.setDataSourceSpaces(reader.getString(section, DATA_SOURCE_SPACES_PROPERTY_NAME, null, true));
config.setHarvesterSpaces(reader.getString(section, HARVESTER_SPACES_PROPERTY_NAME, null, true));
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/DSSFileUtils.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/DSSFileUtils.java
new file mode 100644
index 00000000000..18f1dd76822
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/DSSFileUtils.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2016 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.harvester.synchronizer;
+
+import java.io.InputStream;
+import java.util.List;
+
+import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.search.SearchResult;
+import ch.ethz.sis.openbis.generic.dssapi.v3.IDataStoreServerApi;
+import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile;
+import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.download.DataSetFileDownloadOptions;
+import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.fetchoptions.DataSetFileFetchOptions;
+import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.id.IDataSetFileId;
+import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.search.DataSetFileSearchCriteria;
+import ch.systemsx.cisd.common.spring.HttpInvokerUtils;
+import ch.systemsx.cisd.common.ssl.SslCertificateHelper;
+
+/**
+ *
+ *
+ * @author Ganime Betul Akin
+ */
+class DSSFileUtils
+{
+ public static final int TIMEOUT = 10000;
+
+ private final IDataStoreServerApi dss;
+ private final IApplicationServerApi as;
+
+ public static DSSFileUtils create(String asUrl, String dssUrl)
+ {
+ return new DSSFileUtils(asUrl, dssUrl, TIMEOUT);
+ }
+
+ private DSSFileUtils (String asUrl, String dssUrl, int timeout)
+ {
+ SslCertificateHelper.trustAnyCertificate(asUrl);
+ SslCertificateHelper.trustAnyCertificate(dssUrl);
+
+ this.dss =HttpInvokerUtils.createStreamSupportingServiceStub(IDataStoreServerApi.class, dssUrl + IDataStoreServerApi.SERVICE_URL, timeout);
+ this.as = HttpInvokerUtils.createServiceStub(IApplicationServerApi.class, asUrl + IApplicationServerApi.SERVICE_URL, timeout);
+ }
+
+ public SearchResult<DataSetFile> searchFiles(String sessionToken, DataSetFileSearchCriteria criteria, DataSetFileFetchOptions dsFileFetchOptions)
+ {
+ return dss.searchFiles(sessionToken, criteria, dsFileFetchOptions);
+ }
+
+ public SearchResult<DataSetFile> searchWithDataSetCode(String sessionToken, String dataSetCode, DataSetFileFetchOptions dsFileFetchOptions)
+ {
+ DataSetFileSearchCriteria criteria = new DataSetFileSearchCriteria();
+ criteria.withDataSet().withCode().thatEquals(dataSetCode);
+ return searchFiles(sessionToken, criteria, dsFileFetchOptions);
+ }
+
+ public InputStream downloadFiles(String sessionToken, List<IDataSetFileId> fileIds, DataSetFileDownloadOptions options)
+ {
+ return dss.downloadFiles(sessionToken, fileIds, options);
+ }
+
+ public String login(String loginUser, String loginPass)
+ {
+ return as.login(loginUser, loginPass);
+ }
+}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/DataSetRegistrationIngestionService.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/DataSetRegistrationIngestionService.java
index 152b5660539..da1f79bf9f9 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/DataSetRegistrationIngestionService.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/DataSetRegistrationIngestionService.java
@@ -27,19 +27,14 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
-import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.search.SearchResult;
-import ch.ethz.sis.openbis.generic.dssapi.v3.IDataStoreServerApi;
import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile;
import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.download.DataSetFileDownload;
import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.download.DataSetFileDownloadOptions;
import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.download.DataSetFileDownloadReader;
import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.fetchoptions.DataSetFileFetchOptions;
import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.id.IDataSetFileId;
-import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.search.DataSetFileSearchCriteria;
import ch.systemsx.cisd.common.parser.MemorySizeFormatter;
-import ch.systemsx.cisd.common.spring.HttpInvokerUtils;
-import ch.systemsx.cisd.common.ssl.SslCertificateHelper;
import ch.systemsx.cisd.etlserver.registrator.api.v2.IDataSet;
import ch.systemsx.cisd.etlserver.registrator.api.v2.IDataSetRegistrationTransactionV2;
import ch.systemsx.cisd.etlserver.registrator.api.v2.IDataSetUpdatable;
@@ -54,7 +49,6 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.NewProperty;
class DataSetRegistrationIngestionService extends IngestionService<DataSetInformation>
{
-
private static final long serialVersionUID = 1L;
private List<String> dataSetCodes;
@@ -144,20 +138,9 @@ class DataSetRegistrationIngestionService extends IngestionService<DataSetInform
private void downloadDataSetFiles(File dir, String dataSetCode)
{
- SslCertificateHelper.trustAnyCertificate(asUrl);
- SslCertificateHelper.trustAnyCertificate(dssUrl);
-
- IDataStoreServerApi dss =
- HttpInvokerUtils.createStreamSupportingServiceStub(IDataStoreServerApi.class,
- dssUrl + IDataStoreServerApi.SERVICE_URL, 10000);
- IApplicationServerApi as = HttpInvokerUtils
- .createServiceStub(IApplicationServerApi.class, asUrl
- + IApplicationServerApi.SERVICE_URL, 10000);
- String sessionToken = as.login(loginUser, loginPass);
-
- DataSetFileSearchCriteria criteria = new DataSetFileSearchCriteria();
- criteria.withDataSet().withCode().thatEquals(dataSetCode);
- SearchResult<DataSetFile> result = dss.searchFiles(sessionToken, criteria, new DataSetFileFetchOptions());
+ DSSFileUtils dssFileUtils = DSSFileUtils.create(asUrl, dssUrl);
+ String sessionToken = dssFileUtils.login(loginUser, loginPass);
+ SearchResult<DataSetFile> result = dssFileUtils.searchWithDataSetCode(sessionToken, dataSetCode, new DataSetFileFetchOptions());
List<DataSetFile> files = result.getObjects();
List<IDataSetFileId> fileIds = new LinkedList<IDataSetFileId>();
@@ -168,7 +151,7 @@ class DataSetRegistrationIngestionService extends IngestionService<DataSetInform
// Download the files & print the contents
DataSetFileDownloadOptions options = new DataSetFileDownloadOptions();
options.setRecursive(false);
- InputStream stream = dss.downloadFiles(sessionToken, fileIds, options);
+ InputStream stream = dssFileUtils.downloadFiles(sessionToken, fileIds, options);
DataSetFileDownloadReader reader = new DataSetFileDownloadReader(stream);
DataSetFileDownload fileDownload = null;
while ((fileDownload = reader.read()) != null)
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/DataSourceConnector.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/DataSourceConnector.java
new file mode 100644
index 00000000000..f891867b643
--- /dev/null
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/DataSourceConnector.java
@@ -0,0 +1,118 @@
+/*
+ * Copyright 2016 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.dss.plugins.harvester.synchronizer;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeoutException;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.xpath.XPathExpressionException;
+
+import org.eclipse.jetty.client.HttpClient;
+import org.eclipse.jetty.client.api.AuthenticationStore;
+import org.eclipse.jetty.client.api.ContentResponse;
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.client.util.BasicAuthentication;
+import org.eclipse.jetty.http.HttpStatus;
+import org.w3c.dom.Document;
+import org.xml.sax.SAXException;
+
+import ch.ethz.sis.openbis.generic.server.dss.plugins.harvester.config.BasicAuthCredentials;
+import ch.systemsx.cisd.common.http.JettyHttpClientFactory;
+
+/**
+ *
+ *
+ * @author Ganime Betul Akin
+ */
+public class DataSourceConnector
+{
+ private final String dataSourceUrl;
+
+ private final BasicAuthCredentials authCredentials;
+
+ public DataSourceConnector(String url, BasicAuthCredentials authCredentials)
+ {
+ this.dataSourceUrl = url;
+ this.authCredentials = authCredentials;
+ }
+
+ public Document getResourceListAsXMLDoc(List<String> spaces) throws ParserConfigurationException, SAXException, IOException,
+ XPathExpressionException,
+ URISyntaxException,
+ InterruptedException, TimeoutException, ExecutionException
+ {
+ HttpClient client = JettyHttpClientFactory.getHttpClient();
+ addAuthenticationCredentials(client);
+ Request requestEntity = createNewHttpRequest(client, spaces);
+ ContentResponse contentResponse = getResponse(requestEntity);
+ return parseResponse(contentResponse);
+ }
+
+ private Document parseResponse(ContentResponse contentResponse) throws ParserConfigurationException, SAXException, IOException
+ {
+ DocumentBuilderFactory domFactory = DocumentBuilderFactory.newInstance();
+ domFactory.setNamespaceAware(true);
+ byte[] content = contentResponse.getContent();
+ ByteArrayInputStream bis = new ByteArrayInputStream(content);
+ DocumentBuilder builder = domFactory.newDocumentBuilder();
+ Document doc = builder.parse(bis);
+ return doc;
+ }
+
+ private ContentResponse getResponse(Request requestEntity) throws InterruptedException, TimeoutException, ExecutionException, IOException
+ {
+ ContentResponse contentResponse;
+ contentResponse = requestEntity.send();
+ int statusCode = contentResponse.getStatus();
+
+ if (statusCode != HttpStatus.Code.OK.getCode())
+ {
+ throw new IOException("Status Code was " + statusCode + " instead of " + HttpStatus.Code.OK.getCode());
+ }
+ return contentResponse;
+ }
+
+ private Request createNewHttpRequest(HttpClient client, List<String> spaces)
+ {
+ String spacesParam = "";
+ for (String dataSourceSpace : spaces)
+ {
+ spacesParam += dataSourceSpace + ",";
+ }
+ if (spacesParam.isEmpty() == false)
+ {
+ spacesParam = spacesParam.substring(0, spacesParam.length() - 1);
+ }
+ Request requestEntity = client.newRequest(dataSourceUrl + "?verb=resourcelist.xml&spaces=" + spacesParam).method("GET");
+ return requestEntity;
+ }
+
+ private void addAuthenticationCredentials(HttpClient client) throws URISyntaxException
+ {
+ AuthenticationStore auth = client.getAuthenticationStore();
+ auth.addAuthentication(new BasicAuthentication(new URI(dataSourceUrl), authCredentials.getRealm(), authCredentials.getUser(), authCredentials
+ .getPassword()));
+ }
+}
diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/EntitySynchronizer.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/EntitySynchronizer.java
index 6a49921a23e..a8a4e1a5c71 100644
--- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/EntitySynchronizer.java
+++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/harvester/synchronizer/EntitySynchronizer.java
@@ -16,11 +16,9 @@
package ch.ethz.sis.openbis.generic.server.dss.plugins.harvester.synchronizer;
-import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
-import java.net.URI;
import java.net.URISyntaxException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
@@ -38,19 +36,11 @@ import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeoutException;
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPathExpressionException;
import org.apache.commons.codec.binary.Hex;
import org.apache.log4j.Logger;
-import org.eclipse.jetty.client.HttpClient;
-import org.eclipse.jetty.client.api.AuthenticationStore;
-import org.eclipse.jetty.client.api.ContentResponse;
-import org.eclipse.jetty.client.api.Request;
-import org.eclipse.jetty.client.util.BasicAuthentication;
-import org.eclipse.jetty.http.HttpStatus;
import org.w3c.dom.Document;
import org.xml.sax.SAXException;
@@ -85,10 +75,7 @@ import ch.ethz.sis.openbis.generic.shared.entitygraph.Node;
import ch.systemsx.cisd.common.concurrent.ITaskExecutor;
import ch.systemsx.cisd.common.concurrent.ParallelizedExecutor;
import ch.systemsx.cisd.common.exceptions.Status;
-import ch.systemsx.cisd.common.http.JettyHttpClientFactory;
import ch.systemsx.cisd.common.logging.Log4jSimpleLogger;
-import ch.systemsx.cisd.common.spring.HttpInvokerUtils;
-import ch.systemsx.cisd.common.ssl.SslCertificateHelper;
import ch.systemsx.cisd.etlserver.registrator.api.v1.impl.ConversionUtils;
import ch.systemsx.cisd.openbis.dss.generic.shared.DataSetDirectoryProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.DataSetProcessingContext;
@@ -171,7 +158,8 @@ public class EntitySynchronizer
// retrieve the document from the data source
operationLog.info("Retrieving the resource list..");
- Document doc = getResourceListAsXMLDoc();
+ DataSourceConnector connector = new DataSourceConnector(config.getDataSourceURI(), config.getAuthenticationCredentials());
+ Document doc = connector.getResourceListAsXMLDoc(new ArrayList<String>(config.getSpaceMappings().keySet()));
// Parse the resource list: This sends back all projects,
// experiments, samples and data sets contained in the XML together with their last modification date to be used for filtering
@@ -179,11 +167,9 @@ public class EntitySynchronizer
ResourceListParser parser = ResourceListParser.create(config.getSpaceMappings()); // , lastSyncTimestamp
ResourceListParserData data = parser.parseResourceListDocument(doc);
- operationLog.info("Processing deletions");
processDeletions(data);
AtomicEntityOperationDetailsBuilder builder = new AtomicEntityOperationDetailsBuilder();
-
processMetaData(data, builder);
operationLog.info("Registering meta data...");
@@ -192,9 +178,7 @@ public class EntitySynchronizer
// register physical data sets
operationLog.info("Registering data sets...");
-
Map<String, DataSetWithConnections> physicalDSMap = data.filterPhysicalDataSetsByLastModificationDate(lastSyncTimestamp);
-
registerPhysicalDataSets(physicalDSMap);
// link physical data sets registered above to container data sets
@@ -316,7 +300,6 @@ public class EntitySynchronizer
builder.dataSetUpdate(dsBatchUpdatesDTO);
}
operationResult = service.performEntityOperations(builder.getDetails());
- System.err.println("entity operation result: " + operationResult);
operationLog.info("entity operation result: " + operationResult);
}
@@ -351,50 +334,11 @@ public class EntitySynchronizer
// service.commit();
}
- private Document getResourceListAsXMLDoc() throws ParserConfigurationException, SAXException, IOException, XPathExpressionException,
- URISyntaxException,
- InterruptedException, TimeoutException, ExecutionException
- {
-
- HttpClient client = JettyHttpClientFactory.getHttpClient();
- // Add authentication credentials
- AuthenticationStore auth = client.getAuthenticationStore();
- auth.addAuthentication(new BasicAuthentication(new URI(config.getDataSourceURI()), config.getRealm(), config.getUser(), config.getPass()));
-
- String spacesParam = "";
- for (String dataSourceSpace : config.getSpaceMappings().keySet())
- {
- spacesParam += dataSourceSpace + ",";
- }
- if (spacesParam.isEmpty() == false)
- {
- spacesParam = spacesParam.substring(0, spacesParam.length() - 1);
- }
- Request requestEntity = client.newRequest(config.getDataSourceURI() + "?verb=resourcelist.xml&spaces=" + spacesParam).method("GET");
-
- ContentResponse contentResponse;
- contentResponse = requestEntity.send();
- int statusCode = contentResponse.getStatus();
-
- if (statusCode != HttpStatus.Code.OK.getCode())
- {
- throw new IOException("Status Code was " + statusCode + " instead of " + HttpStatus.Code.OK.getCode());
- }
-
- byte[] content = contentResponse.getContent();
- ByteArrayInputStream bis = new ByteArrayInputStream(content);
-
- // System.out.println(new String(content));
- DocumentBuilderFactory domFactory =
- DocumentBuilderFactory.newInstance();
- domFactory.setNamespaceAware(true);
- DocumentBuilder builder = domFactory.newDocumentBuilder();
- return builder.parse(bis);
- }
private void processDeletions(ResourceListParserData data) throws NoSuchAlgorithmException, UnsupportedEncodingException
{
+ operationLog.info("Processing deletions");
String sessionToken = ServiceProvider.getOpenBISService().getSessionToken();
EntityRetriever entityRetriever =
EntityRetriever.createWithSessionToken(ServiceProvider.getV3ApplicationService(), sessionToken);
@@ -564,7 +508,7 @@ public class EntitySynchronizer
experiment = service.tryGetExperimentByPermId(newIncomingExp.getPermID());
} catch (Exception e)
{
- // TODO doing nothing because when the experiment with the perm id not found
+ // doing nothing because when the experiment with the perm id not found
// an exception will be thrown. Seems to be the same with entity kinds
}
@@ -580,11 +524,11 @@ public class EntitySynchronizer
builder.experimentUpdate(expUpdate);
}
}
- handleSampleConnections(data, exp, newIncomingExp);
+ handleExperimentConnections(data, exp, newIncomingExp);
}
}
- private void handleSampleConnections(ResourceListParserData data, ExperimentWithConnections exp, NewExperiment newIncomingExp)
+ private void handleExperimentConnections(ResourceListParserData data, ExperimentWithConnections exp, NewExperiment newIncomingExp)
{
Map<String, SampleWithConnections> samplesToProcess = data.getSamplesToProcess();
Map<String, DataSetWithConnections> dataSetsToProcess = data.getDataSetsToProcess();
@@ -670,7 +614,7 @@ public class EntitySynchronizer
builder.projectUpdate(createProjectUpdateDTO(incomingProject, project));
}
}
- handleProjectConnections(data, prj);
+ // handleProjectConnections(data, prj);
}
}
@@ -686,6 +630,7 @@ public class EntitySynchronizer
// the project is connected to an experiment
ExperimentWithConnections exp = experimentsToProcess.get(connectedExpPermId);
NewExperiment newExp = exp.getExperiment();
+ Experiment experiment = service.tryGetExperimentByPermId(connectedExpPermId);
// check if our local graph has the same connection
if (service.tryGetExperiment(ExperimentIdentifierFactory.parse(newExp.getIdentifier())) == null)
{
@@ -693,16 +638,14 @@ public class EntitySynchronizer
String oldIdentifier = newExp.getIdentifier();
int index = oldIdentifier.lastIndexOf('/');
String expCode = oldIdentifier.substring(index + 1);
-
newExp.setIdentifier(prj.getProject().getIdentifier() + "/" + expCode);
// add new experiment node
}
}
else
{
- // TODO This means the XML contains the connection but not the connected entity.
- // These sort of problems maybe recorded in a separate synchronization log?
- // ???????????????
+ // This means the XML contains the connection but not the connected entity.
+ // This is an unlikely scenario.
operationLog.info("Connected experiment with permid : " + connectedExpPermId + " is missing");
}
}
@@ -739,7 +682,7 @@ public class EntitySynchronizer
sampleWithExperiment = service.tryGetSampleByPermId(incomingSample.getPermID());
} catch (Exception e)
{
- // TODO doing nothing because when the sample with the perm is not found
+ // doing nothing because when the sample with the perm is not found
// an exception will be thrown. See the same issue for projects
}
if (sampleWithExperiment == null)
@@ -877,7 +820,7 @@ public class EntitySynchronizer
{
Properties props = new Properties();
props.setProperty("user", EntitySynchronizer.this.config.getUser());
- props.setProperty("pass", EntitySynchronizer.this.config.getPass());
+ props.setProperty("pass", EntitySynchronizer.this.config.getPassword());
props.setProperty("as-url", EntitySynchronizer.this.config.getDataSourceOpenbisURL());
props.setProperty("dss-url", EntitySynchronizer.this.config.getDataSourceDSSURL());
props.setProperty("harvester-temp-dir", EntitySynchronizer.this.config.getHarvesterTempDir());
@@ -892,21 +835,14 @@ public class EntitySynchronizer
String asUrl = config.getDataSourceOpenbisURL();
String dssUrl = config.getDataSourceDSSURL();
- SslCertificateHelper.trustAnyCertificate(dssUrl);
- SslCertificateHelper.trustAnyCertificate(asUrl);
-
- IDataStoreServerApi dss =
- HttpInvokerUtils.createStreamSupportingServiceStub(IDataStoreServerApi.class,
- dssUrl + IDataStoreServerApi.SERVICE_URL, 10000);
- IApplicationServerApi as = HttpInvokerUtils
- .createServiceStub(IApplicationServerApi.class, asUrl
- + IApplicationServerApi.SERVICE_URL, 10000);
- String sessionToken = as.login(config.getUser(), config.getPass());
+ DSSFileUtils dssFileUtils = DSSFileUtils.create(asUrl, dssUrl);
+ String sessionToken = dssFileUtils.login(config.getUser(), config.getPassword());
DataSetFileSearchCriteria criteria = new DataSetFileSearchCriteria();
criteria.withDataSet().withCode().thatEquals(dataSetCode);
- SearchResult<DataSetFile> result = dss.searchFiles(sessionToken, criteria, new DataSetFileFetchOptions());
+ SearchResult<DataSetFile> result = dssFileUtils.searchFiles(sessionToken, criteria, new DataSetFileFetchOptions());
+ // get the file nodes in the harvester openbis
IDataStoreServerApi dssharvester = (IDataStoreServerApi) ServiceProvider.getDssServiceV3().getService();
SearchResult<DataSetFile> resultHarvester =
dssharvester.searchFiles(ServiceProvider.getOpenBISService().getSessionToken(), criteria, new DataSetFileFetchOptions());
--
GitLab