From f0cf1519938c659c998d4ec7a067885fc38f28aa Mon Sep 17 00:00:00 2001
From: felmer <felmer>
Date: Mon, 26 Jun 2017 10:31:45 +0000
Subject: [PATCH] SSDM-5284: get private token needed for getting latest git
hash also from file. core plugin 'entity-reg' moved from git repository to
here.
SVN: 38452
---
.../core-plugins/openbis1/.gitignore | 0
.../1/dss/drop-boxes/entity-reg/dropbox.py | 50 +++++++++++++++++++
.../drop-boxes/entity-reg/plugin.properties | 6 +++
integration-tests/test_openbis_sync.py | 27 ++++------
4 files changed, 67 insertions(+), 16 deletions(-)
delete mode 100644 integration-tests/templates/test_openbis_sync/core-plugins/openbis1/.gitignore
create mode 100644 integration-tests/templates/test_openbis_sync/core-plugins/openbis1/1/dss/drop-boxes/entity-reg/dropbox.py
create mode 100644 integration-tests/templates/test_openbis_sync/core-plugins/openbis1/1/dss/drop-boxes/entity-reg/plugin.properties
diff --git a/integration-tests/templates/test_openbis_sync/core-plugins/openbis1/.gitignore b/integration-tests/templates/test_openbis_sync/core-plugins/openbis1/.gitignore
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/integration-tests/templates/test_openbis_sync/core-plugins/openbis1/1/dss/drop-boxes/entity-reg/dropbox.py b/integration-tests/templates/test_openbis_sync/core-plugins/openbis1/1/dss/drop-boxes/entity-reg/dropbox.py
new file mode 100644
index 00000000000..d091a18799f
--- /dev/null
+++ b/integration-tests/templates/test_openbis_sync/core-plugins/openbis1/1/dss/drop-boxes/entity-reg/dropbox.py
@@ -0,0 +1,50 @@
+import glob
+import os
+from datetime import datetime
+from ch.systemsx.cisd.openbis.dss.generic.shared.utils import ExcelFileReader
+
+def process(transaction):
+ print "Starting..."
+ spaceCode = "SYNC"
+ p1 = transaction.createNewProject("/%s/P1" % spaceCode)
+ exp1 = transaction.createNewExperiment(p1.getProjectIdentifier() + "/E1", "UNKNOWN")
+ container_smp = transaction.createNewSample("/%s/S1" % spaceCode, "UNKNOWN")
+ container_smp.setExperiment(exp1)
+ component_samp = transaction.createNewSample("/%s/S2" % spaceCode, "UNKNOWN")
+ component_samp.setContainer(container_smp)
+ component_samp.setExperiment(exp1)
+
+ comp_ds = createDsWithPermId(transaction, component_samp, "COMPONENT_DS1", "UNKNOWN")
+ cont_ds = registerContainerDS(transaction, component_samp.getExperiment(), comp_ds.getDataSetCode())
+
+ '''A new sample of the type added by the master data initialization script'''
+ md_test_smp = transaction.createNewSample("/%s/S3" % spaceCode, "MD_TEST")
+ md_test_smp.setPropertyValue("GENDER", "MALE")
+
+ #===========================================================================
+ # sample = transaction.getSampleForUpdate("/DS1_SRC/S8")
+ # sample.setPropertyValue("resolution", "")
+ #===========================================================================
+
+ #===========================================================================
+ # exp2 = transaction.createNewExperiment(p1.getProjectIdentifier() + "/E2", "UNKNOWN")
+ # parent_smp = transaction.createNewSample("/TEST/S2", "UNKNOWN")
+ # parent_smp.setExperiment(exp2)
+ # parent_ds = createDsWithPermId(transaction, exp2, "PARENT_DS1", "UNKNOWN")
+ # comp_ds.setParentDatasets([parent_ds.getCode()])
+ # component_smp.setParentSampleIdentifiers(parent_smp.getSampleIdentifier())
+ #===========================================================================
+
+def createDsWithPermId(transaction, sample, code, ds_type):
+ ds = transaction.createNewDataSet(ds_type, code)
+ ds.setSample(sample)
+ transaction.moveFile(transaction.getIncoming().getPath(), ds)
+ return ds
+
+def registerContainerDS(transaction, experiment, comp_ds_code):
+ container = transaction.createNewDataSet("HCS_IMAGE_CONTAINER_RAW", "CONTAINER_DS1")
+ container.setExperiment(experiment)
+ container.setContainedDataSetCodes([comp_ds_code])
+
+ #raise Exception('stop')
+
diff --git a/integration-tests/templates/test_openbis_sync/core-plugins/openbis1/1/dss/drop-boxes/entity-reg/plugin.properties b/integration-tests/templates/test_openbis_sync/core-plugins/openbis1/1/dss/drop-boxes/entity-reg/plugin.properties
new file mode 100644
index 00000000000..ae69d3fab98
--- /dev/null
+++ b/integration-tests/templates/test_openbis_sync/core-plugins/openbis1/1/dss/drop-boxes/entity-reg/plugin.properties
@@ -0,0 +1,6 @@
+incoming-dir = ${root-dir}/openbis-sync-entity-reg
+incoming-dir-create = true
+incoming-data-completeness-condition = auto-detection
+top-level-data-set-handler = ch.systemsx.cisd.etlserver.registrator.api.v2.JythonTopLevelDataSetHandlerV2
+script-path = dropbox.py
+storage-processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor
diff --git a/integration-tests/test_openbis_sync.py b/integration-tests/test_openbis_sync.py
index b5eafcfed60..1240c1887c4 100755
--- a/integration-tests/test_openbis_sync.py
+++ b/integration-tests/test_openbis_sync.py
@@ -46,7 +46,6 @@ class TestCase(systemtest.testcase.TestCase):
'''Set openbis1 as the datasource'''
self.installDataSourcePlugin(openbis1, openbis1_dss_port)
- self.installEntityRegistrationPlugin(openbis1)
'''Copy master data script'''
filePath = "%s/servers/core-plugins/%s/1/as" % (openbis1.installPath, openbis1.instanceName)
@@ -115,7 +114,6 @@ class TestCase(systemtest.testcase.TestCase):
openbis1=self.createOpenbisController(instanceName = 'openbis1', dropDatabases=False)
openbis1.setDummyAuthentication()
self.installDataSourcePlugin(openbis1, openbis1_dss_port)
- self.installEntityRegistrationPlugin(openbis1)
'''Copy master data script'''
filePath = "%s/servers/core-plugins/%s/1/as" % (openbis1.installPath, openbis1.instanceName)
@@ -296,10 +294,11 @@ class TestCase(systemtest.testcase.TestCase):
def installPlugin(self, openbisController, plugin_name):
repository = GitLabArtifactRepository(self.artifactRepository.localRepositoryFolder)
- path = repository.getPathToArtifact('sis/OpenbisSync', 'archive.zip')
+ project = 'sis/OpenbisSync'
+ path = repository.getPathToArtifact(project, 'archive.zip')
util.printAndFlush("downloaded repository as : %s" % path)
destination = "%s/servers/core-plugins/%s/" % (openbisController.installPath, openbisController.instanceName)
- commit_id = self.getLatestCommitHashForCorePlugin('sissource.ethz.ch', 10)
+ commit_id = repository.getLatestCommitHash(project)
path_in_archive = "OpenbisSync-master-%s" % commit_id
util.printAndFlush("path to plugin in repository repository as : %s" % path_in_archive)
util.printAndFlush("Unzipping plugin % s into folder %s"% (plugin_name, destination))
@@ -324,9 +323,6 @@ class TestCase(systemtest.testcase.TestCase):
pluginProps['databaseKind'] = openbisController.databaseKind
util.writeProperties(plugin_properties_file, pluginProps)
- def installEntityRegistrationPlugin(self, openbisController):
- self.installPlugin(openbisController, "test")
-
def installHarvesterPlugin(self, openbisController):
self.installPlugin(openbisController, "harvester")
@@ -336,13 +332,6 @@ class TestCase(systemtest.testcase.TestCase):
def getMasterDataScriptFolder(self):
return systemtest.testcase.TEMPLATES + "/" + self.name + "/master_data"
- def getLatestCommitHashForCorePlugin(self, host, project):
- url = "https://%s/api/v4/projects/%s/repository/commits/master" % (host, project)
- request = Request(url, headers = {'PRIVATE-TOKEN' : 'Rz1DbhpVBXSUpRny5Dbr'})
- response = urllib2.urlopen(request)
- result = json.load(response)
- return result["id"]
-
class GitLabArtifactRepository(GitArtifactRepository):
"""
Artifact repository for a gitlab projects.
@@ -354,11 +343,17 @@ class GitLabArtifactRepository(GitArtifactRepository):
def downloadArtifact(self, project, pattern):
url = "https://%s/%s/repository/%s" % (self.host, project, pattern)
util.printAndFlush("Download %s to %s." % (url, self.localRepositoryFolder))
- private_token = self._read_private_token()
- request = Request(url, headers = {'PRIVATE-TOKEN' : private_token})
+ request = Request(url, headers = {'PRIVATE-TOKEN' : self._read_private_token()})
self._download(urllib2.urlopen(request), pattern)
return pattern
+ def getLatestCommitHash(self, project):
+ url = "https://%s/api/v4/projects/%s/repository/commits/master" % (self.host, project.replace('/','%2F'))
+ request = Request(url, headers = {'PRIVATE-TOKEN' : self._read_private_token()})
+ response = urllib2.urlopen(request)
+ result = json.load(response)
+ return result["id"]
+
def _read_private_token(self):
with open('targets/sissource_private-token.txt', 'r') as f:
return f.readline().strip()
--
GitLab