From ac0a2eb251110c4ace054521d16f9072d7a36c1b Mon Sep 17 00:00:00 2001
From: Viktor Kovtun <viktor.kovtun@id.ethz.ch>
Date: Sun, 18 Aug 2019 11:52:23 +0200
Subject: [PATCH] SSDM-8405 Made the Zenodo exporting plugin do actual export.
---
.../eln-lims/html/js/server/ServerFacade.js | 45 ++++++
.../ZenodoExport/ZenodoExportController.js | 8 +-
.../exports-api/exportsApi.py | 31 +++-
.../rc-exports-api/rcExports.py | 34 +----
.../zenodo-exports-api/exportsApi.py | 1 +
.../zenodo-exports-api/plugin.properties | 1 +
.../zenodo-exports-api/zenodoExports.py | 137 +++++++++++++++++-
7 files changed, 214 insertions(+), 43 deletions(-)
create mode 120000 openbis_standard_technologies/dist/core-plugins/eln-lims/1/dss/reporting-plugins/zenodo-exports-api/exportsApi.py
diff --git a/openbis_standard_technologies/dist/core-plugins/eln-lims/1/as/webapps/eln-lims/html/js/server/ServerFacade.js b/openbis_standard_technologies/dist/core-plugins/eln-lims/1/as/webapps/eln-lims/html/js/server/ServerFacade.js
index 54e465edf47..b80412313fe 100644
--- a/openbis_standard_technologies/dist/core-plugins/eln-lims/1/as/webapps/eln-lims/html/js/server/ServerFacade.js
+++ b/openbis_standard_technologies/dist/core-plugins/eln-lims/1/as/webapps/eln-lims/html/js/server/ServerFacade.js
@@ -378,6 +378,51 @@ function ServerFacade(openbisServer) {
});
};
+ this.exportZenodo = function(entities, includeRoot, metadataOnly, userInformation, callbackFunction) {
+ this.asyncExportZenodo({
+ "method": "exportAll",
+ "includeRoot": includeRoot,
+ "entities": entities,
+ "metadataOnly": metadataOnly,
+ "userInformation": userInformation,
+ "originUrl": window.location.origin,
+ "sessionToken": this.openbisServer.getSession(),
+ }, callbackFunction, "zenodo-exports-api");
+ };
+
+ this.asyncExportZenodo = function(parameters, callbackFunction, serviceId) {
+ require(["as/dto/service/execute/ExecuteAggregationServiceOperation",
+ "as/dto/operation/AsynchronousOperationExecutionOptions", "as/dto/service/id/DssServicePermId",
+ "as/dto/datastore/id/DataStorePermId", "as/dto/service/execute/AggregationServiceExecutionOptions"],
+ function(ExecuteAggregationServiceOperation, AsynchronousOperationExecutionOptions, DssServicePermId, DataStorePermId,
+ AggregationServiceExecutionOptions) {
+ var dataStoreCode = profile.getDefaultDataStoreCode();
+ var dataStoreId = new DataStorePermId(dataStoreCode);
+ var dssServicePermId = new DssServicePermId(serviceId, dataStoreId);
+ var options = new AggregationServiceExecutionOptions();
+
+ options.withParameter("sessionToken", parameters["sessionToken"]);
+
+ options.withParameter("entities", parameters["entities"]);
+ options.withParameter("includeRoot", parameters["includeRoot"]);
+ options.withParameter("metadataOnly", parameters["metadataOnly"]);
+ options.withParameter("method", parameters["method"]);
+ options.withParameter("originUrl", parameters["originUrl"]);
+ options.withParameter("submissionType", parameters["submissionType"]);
+ options.withParameter("submissionUrl", parameters["submissionUrl"]);
+ options.withParameter("entities", parameters["entities"]);
+ options.withParameter("userId", parameters["userInformation"]["id"]);
+ options.withParameter("userEmail", parameters["userInformation"]["email"]);
+ options.withParameter("userFirstName", parameters["userInformation"]["firstName"]);
+ options.withParameter("userLastName", parameters["userInformation"]["lastName"]);
+
+ var operation = new ExecuteAggregationServiceOperation(dssServicePermId, options);
+ mainController.openbisV3.executeOperations([operation], new AsynchronousOperationExecutionOptions()).done(function(results) {
+ callbackFunction(results.executionId.permId);
+ });
+ });
+ };
+
//
// Gets submission types
//
diff --git a/openbis_standard_technologies/dist/core-plugins/eln-lims/1/as/webapps/eln-lims/html/js/views/ZenodoExport/ZenodoExportController.js b/openbis_standard_technologies/dist/core-plugins/eln-lims/1/as/webapps/eln-lims/html/js/views/ZenodoExport/ZenodoExportController.js
index c79e5e40812..99e24671fe9 100644
--- a/openbis_standard_technologies/dist/core-plugins/eln-lims/1/as/webapps/eln-lims/html/js/views/ZenodoExport/ZenodoExportController.js
+++ b/openbis_standard_technologies/dist/core-plugins/eln-lims/1/as/webapps/eln-lims/html/js/views/ZenodoExport/ZenodoExportController.js
@@ -26,10 +26,6 @@ function ZenodoExportController(parentController) {
var _this = this;
var selectedNodes = $(exportModel.tree).fancytree('getTree').getSelectedNodes();
- var selectedOption = exportView.$submissionTypeDropdown.find(":selected");
- var submissionUrl = selectedOption.val();
- var submissionType = selectedOption.text();
-
var toExport = [];
for (var eIdx = 0; eIdx < selectedNodes.length; eIdx++) {
var node = selectedNodes[eIdx];
@@ -38,12 +34,10 @@ function ZenodoExportController(parentController) {
if (toExport.length === 0) {
Util.showInfo('First select something to export.');
- } else if (!submissionUrl) {
- Util.showInfo('First select submission type.');
} else {
Util.blockUI();
this.getUserInformation(function(userInformation) {
- mainController.serverFacade.exportRc(toExport, true, false, submissionUrl, submissionType, userInformation,
+ mainController.serverFacade.exportZenodo(toExport, true, false, userInformation,
function(operationExecutionPermId) {
_this.waitForOpExecutionResponse(operationExecutionPermId, function(error, result) {
Util.unblockUI();
diff --git a/openbis_standard_technologies/dist/core-plugins/eln-lims/1/dss/reporting-plugins/exports-api/exportsApi.py b/openbis_standard_technologies/dist/core-plugins/eln-lims/1/dss/reporting-plugins/exports-api/exportsApi.py
index 932c0fa3023..7af095aa70a 100644
--- a/openbis_standard_technologies/dist/core-plugins/eln-lims/1/dss/reporting-plugins/exports-api/exportsApi.py
+++ b/openbis_standard_technologies/dist/core-plugins/eln-lims/1/dss/reporting-plugins/exports-api/exportsApi.py
@@ -645,4 +645,33 @@ def getConfigurationProperty(transaction, propertyName):
try:
return threadProperties.getProperty(propertyName);
except:
- return None
\ No newline at end of file
+ return None
+
+
+def generateZipFile(entities, params, tempDirPath, tempZipFilePath):
+ # Generates ZIP file with selected item for export
+
+ sessionToken = params.get('sessionToken')
+ includeRoot = params.get('includeRoot')
+
+ fos = None
+ zos = None
+ try:
+ fos = FileOutputStream(tempZipFilePath)
+ zos = ZipOutputStream(fos)
+
+ fileMetadata = generateFilesInZip(zos, entities, includeRoot, sessionToken, tempDirPath)
+ finally:
+ if zos is not None:
+ zos.close()
+ if fos is not None:
+ fos.close()
+
+ return fileMetadata
+
+
+def checkResponseStatus(response):
+ status = response.getStatus()
+ if status >= 300:
+ reason = response.getReason()
+ raise ValueError('Unsuccessful response from the server: %s %s' % (status, reason))
\ No newline at end of file
diff --git a/openbis_standard_technologies/dist/core-plugins/eln-lims/1/dss/reporting-plugins/rc-exports-api/rcExports.py b/openbis_standard_technologies/dist/core-plugins/eln-lims/1/dss/reporting-plugins/rc-exports-api/rcExports.py
index 99f41ac9578..66e91990f86 100644
--- a/openbis_standard_technologies/dist/core-plugins/eln-lims/1/dss/reporting-plugins/rc-exports-api/rcExports.py
+++ b/openbis_standard_technologies/dist/core-plugins/eln-lims/1/dss/reporting-plugins/rc-exports-api/rcExports.py
@@ -35,7 +35,8 @@ from org.eclipse.jetty.client.util import BasicAuthentication
from org.eclipse.jetty.http import HttpMethod
from org.eclipse.jetty.util.ssl import SslContextFactory
-from exportsApi import displayResult, findEntitiesToExport, validateDataSize, getConfigurationProperty, generateFilesInZip, addToZipFile, cleanUp
+from exportsApi import displayResult, findEntitiesToExport, validateDataSize, getConfigurationProperty, generateFilesInZip, addToZipFile, cleanUp, \
+ generateZipFile, checkResponseStatus
operationLog = Logger.getLogger(str(LogCategory.OPERATION) + '.rcExports.py')
@@ -106,7 +107,7 @@ def export(entities, tr, params, userInformation):
exportZipFilePath = exportDirPath + '.zip'
exportZipFileName = exportDirName + '.zip'
- generateInternalZipFile(entities, params, contentDirPath, contentZipFilePath)
+ generateZipFile(entities, params, contentDirPath, contentZipFilePath)
FileUtils.forceDelete(File(contentDirPath))
generateExternalZipFile(params=params, exportDirPath=exportDirPath, contentZipFilePath=contentZipFilePath, contentZipFileName=contentZipFileName,
@@ -189,35 +190,6 @@ def fetchServiceDocument(url, httpClient):
return json.dumps(map(collectionToDictionaryMapper, collections))
-def checkResponseStatus(response):
- status = response.getStatus()
- if status >= 300:
- reason = response.getReason()
- raise ValueError('Unsuccessful response from the server: %s %s' % (status, reason))
-
-
-def generateInternalZipFile(entities, params, tempDirPath, tempZipFilePath):
- # Generates ZIP file with selected item for export
-
- sessionToken = params.get('sessionToken')
- includeRoot = params.get('includeRoot')
-
- fos = None
- zos = None
- try:
- fos = FileOutputStream(tempZipFilePath)
- zos = ZipOutputStream(fos)
-
- fileMetadata = generateFilesInZip(zos, entities, includeRoot, sessionToken, tempDirPath)
- finally:
- if zos is not None:
- zos.close()
- if fos is not None:
- fos.close()
-
- return fileMetadata
-
-
def generateExternalZipFile(params, exportDirPath, contentZipFilePath, contentZipFileName, exportZipFileName, userInformation, entities):
# Generates ZIP file which will go to the research collection server
diff --git a/openbis_standard_technologies/dist/core-plugins/eln-lims/1/dss/reporting-plugins/zenodo-exports-api/exportsApi.py b/openbis_standard_technologies/dist/core-plugins/eln-lims/1/dss/reporting-plugins/zenodo-exports-api/exportsApi.py
new file mode 120000
index 00000000000..ccb8a49feb3
--- /dev/null
+++ b/openbis_standard_technologies/dist/core-plugins/eln-lims/1/dss/reporting-plugins/zenodo-exports-api/exportsApi.py
@@ -0,0 +1 @@
+../exports-api/exportsApi.py
\ No newline at end of file
diff --git a/openbis_standard_technologies/dist/core-plugins/eln-lims/1/dss/reporting-plugins/zenodo-exports-api/plugin.properties b/openbis_standard_technologies/dist/core-plugins/eln-lims/1/dss/reporting-plugins/zenodo-exports-api/plugin.properties
index e294424d54a..a9094b31167 100644
--- a/openbis_standard_technologies/dist/core-plugins/eln-lims/1/dss/reporting-plugins/zenodo-exports-api/plugin.properties
+++ b/openbis_standard_technologies/dist/core-plugins/eln-lims/1/dss/reporting-plugins/zenodo-exports-api/plugin.properties
@@ -2,3 +2,4 @@ label = Zenodo Exports API
class = ch.systemsx.cisd.openbis.dss.generic.server.plugins.jython.JythonIngestionService
script-path = zenodoExports.py
service-document-url=https://test.research-collection.ethz.ch/swordv2/servicedocument
+accessToken=
\ No newline at end of file
diff --git a/openbis_standard_technologies/dist/core-plugins/eln-lims/1/dss/reporting-plugins/zenodo-exports-api/zenodoExports.py b/openbis_standard_technologies/dist/core-plugins/eln-lims/1/dss/reporting-plugins/zenodo-exports-api/zenodoExports.py
index 9e00cec83ab..f39c6eefa69 100644
--- a/openbis_standard_technologies/dist/core-plugins/eln-lims/1/dss/reporting-plugins/zenodo-exports-api/zenodoExports.py
+++ b/openbis_standard_technologies/dist/core-plugins/eln-lims/1/dss/reporting-plugins/zenodo-exports-api/zenodoExports.py
@@ -14,10 +14,27 @@
# limitations under the License.
#
+from __future__ import print_function
+
+import traceback
+
+import time
from ch.systemsx.cisd.common.logging import LogCategory
+from java.io import File
+from java.nio.file import Paths
+from org.apache.commons.io import FileUtils
from org.apache.log4j import Logger
+from org.eclipse.jetty.client import HttpClient
+from org.eclipse.jetty.client.util import MultiPartContentProvider
+from org.eclipse.jetty.client.util import PathContentProvider
+from org.eclipse.jetty.client.util import StringContentProvider
+from org.eclipse.jetty.http import HttpMethod
+from org.eclipse.jetty.util.ssl import SslContextFactory
+from org.json import JSONObject
+
+from exportsApi import findEntitiesToExport, validateDataSize, getConfigurationProperty, generateZipFile, checkResponseStatus
-operationLog = Logger.getLogger(str(LogCategory.OPERATION) + '.rcExports.py')
+operationLog = Logger.getLogger(str(LogCategory.OPERATION) + '.zenodoExports.py')
def process(tr, params, tableBuilder):
@@ -27,8 +44,120 @@ def process(tr, params, tableBuilder):
tr.setUserId(userId)
if method == 'exportAll':
- resultUrl = export(tr, params)
+ resultUrl = expandAndExport(tr, params)
+
+
+def expandAndExport(tr, params):
+ entitiesToExport = findEntitiesToExport(params)
+ validateDataSize(entitiesToExport, tr)
+
+ userInformation = {
+ 'firstName': params.get('userFirstName'),
+ 'lastName': params.get('userLastName'),
+ 'email': params.get('userEmail'),
+ }
+
+ operationLog.info('Found ' + str(len(entitiesToExport)) + ' entities to export')
+ return export(entities=entitiesToExport, tr=tr, params=params, userInformation=userInformation)
+
+
+def export(entities, tr, params, userInformation):
+ #Create temporal folder
+ timeNow = time.time()
+
+ exportDirName = 'export_' + str(timeNow)
+ exportDir = File.createTempFile(exportDirName, None)
+ exportDirPath = exportDir.getCanonicalPath()
+ exportDir.delete()
+ exportDir.mkdir()
+
+ contentZipFileName = 'content.zip'
+ contentDirName = 'content_' + str(timeNow)
+ contentDir = File.createTempFile(contentDirName, None, exportDir)
+ contentDirPath = contentDir.getCanonicalPath()
+ contentDir.delete()
+ contentDir.mkdir()
+
+ contentZipFilePath = exportDirPath + '/' + contentZipFileName
+
+ exportZipFilePath = exportDirPath + '.zip'
+ exportZipFileName = exportDirName + '.zip'
+
+ generateZipFile(entities, params, contentDirPath, contentZipFilePath)
+ FileUtils.forceDelete(File(contentDirPath))
+
+ resultUrl = sendToZenodo(tr=tr, params=params, tempZipFileName=contentZipFileName, tempZipFilePath=contentZipFilePath)
+ # cleanUp(exportDirPath, exportZipFilePath)
+ return resultUrl
+
+
+def sendToZenodo(tr, params, tempZipFileName, tempZipFilePath):
+ depositRootUrl = 'https://localhost/api/deposit/depositions'
+
+ accessToken = str(getConfigurationProperty(tr, 'accessToken'))
+ operationLog.info('accessToken: %s' % accessToken)
+
+ headers = {
+ # 'Content-Type': 'application/json',
+ 'Authorization': 'Bearer ' + accessToken,
+ }
+
+ httpClient = None
+ try:
+ httpClient = createHttpClient()
+
+ httpClient.setFollowRedirects(False)
+ httpClient.start()
+
+ depositionData = createDepositionResource(httpClient.newRequest(depositRootUrl), accessToken)
+
+ print('depositionData="%s"' % depositionData.toString())
+ print('depositionId=%d' % depositionData.get('id'))
+ depositionLinks = depositionData.get('links')
+ depositUrl = depositionLinks.get('files')
+
+ print('depositUrl=%s' % depositUrl)
+
+ submissionResult = submitFile(tempZipFilePath, accessToken, httpClient.newRequest(depositUrl))
+
+ print('submissionResult=%s' % submissionResult.toString())
+
+ result = depositionLinks.get('html')
+ print('result="%s"' % result)
+ return result
+ except Exception as e:
+ operationLog.error('Exception at: ' + traceback.format_exc())
+ operationLog.error('Exception: ' + str(e))
+ raise e
+ finally:
+ if httpClient is not None:
+ httpClient.stop()
+
+
+def submitFile(tempZipFilePath, accessToken, request):
+ multiPart = MultiPartContentProvider()
+ multiPart.addFilePart('file', 'content.zip', PathContentProvider(Paths.get(tempZipFilePath)), None)
+ multiPart.close()
+ addAuthenticationHeader(accessToken, request)
+ response = request.method(HttpMethod.POST).content(multiPart).send()
+ checkResponseStatus(response)
+ contentStr = response.getContentAsString()
+ return JSONObject(contentStr)
+
+
+def createDepositionResource(request, accessToken):
+ addAuthenticationHeader(accessToken, request)
+ response = request.method(HttpMethod.POST).content(StringContentProvider("{}"), "application/json").send()
+ checkResponseStatus(response)
+ contentStr = response.getContentAsString()
+ return JSONObject(contentStr)
+
+
+def addAuthenticationHeader(accessToken, request):
+ request.header('Authorization', 'Bearer ' + accessToken)
-def export(tr, params):
- pass
+def createHttpClient():
+ sslContextFactory = SslContextFactory()
+ sslContextFactory.setTrustAll(True)
+ return HttpClient(sslContextFactory)
--
GitLab