diff --git a/api-openbis-python3-pybis/src/python/tests/systemtest/__init__.py b/api-openbis-python3-pybis/src/python/tests/systemtest/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..6b5d1ce929b60094a402f1ff6e10dc47f67f1e8d
--- /dev/null
+++ b/api-openbis-python3-pybis/src/python/tests/systemtest/__init__.py
@@ -0,0 +1,14 @@
+# Copyright ETH 2007 - 2023 Zürich, Scientific IT Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
diff --git a/api-openbis-python3-pybis/src/python/tests/systemtest/artifactrepository.py b/api-openbis-python3-pybis/src/python/tests/systemtest/artifactrepository.py
new file mode 100644
index 0000000000000000000000000000000000000000..a9cf5ab03040c52ec1994541ab9c8227528e21d6
--- /dev/null
+++ b/api-openbis-python3-pybis/src/python/tests/systemtest/artifactrepository.py
@@ -0,0 +1,143 @@
+# Copyright ETH 2013 - 2023 Zürich, Scientific IT Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#import os
+import os.path
+import re
+import xml.dom.minidom
+
+from urllib.request import urlopen
+from util import printAndFlush
+# import os
+import os.path
+import re
+import xml.dom.minidom
+from urllib.request import urlopen
+
+from util import printAndFlush
+
+
+class ArtifactRepository():
+ """
+ Abstract artifact repository which keeps artifacts in a local repository folder.
+ The main method is getPathToArtifact() which returns the path to the requested artifact in the repository.
+ Concrete subclasses have to implement downloadArtifact().
+ """
+ def __init__(self, localRepositoryFolder):
+ """
+ Creates a new instance for the specified folder. The folder will be created if it does not exist.
+ """
+ self.localRepositoryFolder = localRepositoryFolder
+ if not os.path.exists(localRepositoryFolder):
+ os.makedirs(localRepositoryFolder)
+ printAndFlush("Artifact repository: %s" % localRepositoryFolder)
+
+ def clear(self):
+ """
+ Removes all artifacts in the local repository folder.
+ """
+ for f in os.listdir(self.localRepositoryFolder):
+ path = "%s/%s" % (self.localRepositoryFolder, f)
+ if os.path.isfile(path):
+ os.remove(path)
+ printAndFlush("Artifact repository cleared.")
+
+ def getPathToArtifact(self, project, pattern='.*'):
+ """
+ Returns the path to artifact requested by the specified pattern and project.
+ The pattern is a regular expression which has to match the beginning of the artifact file name.
+ The project specifies the project on CI server to download the artifact.
+
+ An Exception is raised if non or more than one artifact matches the pattern.
+ """
+ files = [f for f in os.listdir(self.localRepositoryFolder) if re.match(pattern, f)]
+ if len(files) > 1:
+ raise Exception("More than one artifact in '%s' matches the pattern '%s': %s"
+ % (self.localRepositoryFolder, pattern, files))
+ if len(files) == 0:
+ f = self.downloadArtifact(project, pattern)
+ else:
+ f = files[0]
+ return "%s/%s" % (self.localRepositoryFolder, f)
+
+ def downloadArtifact(self, project, pattern):
+ """
+ Abstract method which needs to be implemented by subclasses.
+ """
+ pass
+
+ def _download(self, readHandle, fileName):
+ filePath = "%s/%s" % (self.localRepositoryFolder, fileName)
+ writeHandle = open(filePath, 'wb')
+ try:
+ blockSize = 8192
+ while True:
+ dataBlock = readHandle.read(blockSize)
+ if not dataBlock:
+ break
+ writeHandle.write(dataBlock)
+ finally:
+ writeHandle.close()
+
+class JenkinsArtifactRepository(ArtifactRepository):
+ """
+ Artifact repository for a CI server based on Jenkins.
+ """
+ def __init__(self, baseUrl, localRepositoryFolder):
+ """
+ Creates a new instance for the specified server URL and local repository.
+ """
+ ArtifactRepository.__init__(self, localRepositoryFolder)
+ self.baseUrl = baseUrl
+
+ def downloadArtifact(self, project, pattern):
+ """
+ Downloads the requested artifact from Jenkins. It uses the Jenkins API.
+ """
+ projectUrl = "%s/job/%s" % (self.baseUrl, project)
+ apiUrl = "%s/lastSuccessfulBuild/api/xml?xpath=//artifact&wrapper=bag" % projectUrl
+ printAndFlush("Get artifact info from %s" % apiUrl)
+ handle = urlopen(apiUrl) # urllib.urlopen(apiUrl)
+ url = None
+ fileName = None
+ dom = xml.dom.minidom.parseString(handle.read())
+ for element in dom.getElementsByTagName('artifact'):
+ elementFileName = element.getElementsByTagName('fileName')[0].firstChild.nodeValue
+ if re.match(pattern, elementFileName):
+ if fileName != None:
+ raise Exception("Pattern '%s' matches at least two artifacts in project '%s': %s and %s"
+ % (pattern, project, fileName, elementFileName))
+ fileName = elementFileName
+ relativePath = element.getElementsByTagName('relativePath')[0].firstChild.nodeValue
+ url = "%s/lastSuccessfulBuild/artifact/%s" % (projectUrl, relativePath)
+ if url == None:
+ raise Exception("For pattern '%s' no artifact found in project '%s'." % (pattern, project))
+ printAndFlush("Download %s to %s." % (url, self.localRepositoryFolder))
+ self._download(urlopen(url), fileName)
+ return fileName
+
+class GitArtifactRepository(ArtifactRepository):
+ """
+ Artifact repository for a git projects.
+ """
+ def __init__(self, localRepositoryFolder, host = 'github.com'):
+ ArtifactRepository.__init__(self, localRepositoryFolder)
+ self.host = host
+
+ def downloadArtifact(self, project, pattern):
+ url = "https://%s/%s/archive/%s" % (self.host, project, pattern)
+ printAndFlush("Download %s to %s." % (url, self.localRepositoryFolder))
+ self._download(urlopen(url), pattern)
+ return pattern
diff --git a/api-openbis-python3-pybis/src/python/tests/systemtest/settings.py b/api-openbis-python3-pybis/src/python/tests/systemtest/settings.py
new file mode 100644
index 0000000000000000000000000000000000000000..8086cd3b3a0e21df0bc245f277032433974f4b5d
--- /dev/null
+++ b/api-openbis-python3-pybis/src/python/tests/systemtest/settings.py
@@ -0,0 +1,56 @@
+# Copyright ETH 2013 - 2023 Zürich, Scientific IT Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+Setup infrastructure common for all tests.
+"""
+import sys
+import os.path
+import time
+
+# Default base URL of the CI server which hosts the artifacts.
+ci_base_url = 'http://bs-ci01.ethz.ch:8090'
+
+reuseRepository = False
+devMode = False
+cmd = sys.argv[0]
+if len(sys.argv) > 1:
+ firstArgument = sys.argv[1]
+ if firstArgument == '-r':
+ reuseRepository = True
+ elif firstArgument == '-dr' or firstArgument == '-rd':
+ reuseRepository = True
+ devMode = True
+ elif firstArgument == '-d':
+ devMode = True
+ elif firstArgument == '-s':
+ ci_base_url = sys.argv[2]
+ elif firstArgument == '-h':
+ print(("Usage: %s [-h|-r|-d|-rd|-s <ci server>]\n-h: prints this help\n-r: reuses artifact repository\n"
+ + "-d: developing mode\n-rd: both options\n"
+ + "-s <ci server>: option for CI server base URL") % os.path.basename(cmd))
+ exit(1)
+ else:
+ print("Unknown option: %s. Use option '-h' to see usage." % firstArgument)
+ exit(1)
+
+dirname = os.path.dirname(os.path.abspath(__file__))
+sys.path.append("%s/source" % dirname)
+sys.path.append("%s/sourceTest" % dirname)
+
+from artifactrepository import JenkinsArtifactRepository
+
+REPOSITORY = JenkinsArtifactRepository(ci_base_url, "%s/targets/artifact-repository" % dirname)
+if not reuseRepository:
+ REPOSITORY.clear()
diff --git a/api-openbis-python3-pybis/src/python/tests/systemtest/test.py b/api-openbis-python3-pybis/src/python/tests/systemtest/test.py
new file mode 100644
index 0000000000000000000000000000000000000000..27fc51f239724cc9260922f0fcfc51af2c55cd98
--- /dev/null
+++ b/api-openbis-python3-pybis/src/python/tests/systemtest/test.py
@@ -0,0 +1,37 @@
+# Copyright ETH 2023 Zürich, Scientific IT Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+
+import settings
+import testcase
+
+
+class TestCase(testcase.TestCase):
+
+ def execute(self):
+
+ self.installOpenbis()
+ # self.installPybis()
+ self.openbisController = self.createOpenbisController()
+ self.openbisController.createTestDatabase("openbis")
+ self.openbisController.allUp()
+
+ os.system('pytest --junitxml=test_results_pybis.xml $WORKSPACE/api-openbis-python3-pybis/src/python/tests')
+
+
+
+
+TestCase(settings, __file__).runTest()
\ No newline at end of file
diff --git a/api-openbis-python3-pybis/src/python/tests/systemtest/testcase.py b/api-openbis-python3-pybis/src/python/tests/systemtest/testcase.py
new file mode 100644
index 0000000000000000000000000000000000000000..a84d6a55bcce98567da0c408105d847fb6182603
--- /dev/null
+++ b/api-openbis-python3-pybis/src/python/tests/systemtest/testcase.py
@@ -0,0 +1,871 @@
+# Copyright ETH 2013 - 2023 Zürich, Scientific IT Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import difflib
+import os
+import os.path
+import re
+import shutil
+import time
+import traceback
+
+import util as util
+
+INSTALLER_PROJECT = 'app-openbis-installer'
+OPENBIS_STANDARD_TECHNOLOGIES_PROJECT = 'core-plugin-openbis'
+DATAMOVER_PROJECT = 'datamover'
+
+PSQL_EXE = 'psql'
+
+PLAYGROUND = 'targets/playground'
+TEMPLATES = 'templates'
+TEST_DATA = 'testData'
+
+DEFAULT_TIME_OUT_IN_MINUTES = 5
+
+
+class TestCase(object):
+ """
+ Abstract superclass of a test case.
+ Subclasses have to override execute() and optionally executeInDevMode().
+ The test case is run by invoking runTest().
+ Here is a skeleton of a test case:
+
+ #!/usr/bin/python
+ import settings
+ import systemtest.testcase
+
+ class TestCase(systemtest.testcase.TestCase):
+
+ def execute(self):
+ ....
+
+ def executeInDevMode(self):
+ ....
+
+ TestCase(settings, __file__).runTest()
+
+ There are two execution modes (controlled by command line option -d and -rd):
+
+ Normal mode:
+ 1. Cleans up playground: Kills running servers and deletes playground folder of this test case.
+ 2. Invokes execute() method.
+ 3. Release resources: Shuts down running servers.
+
+ Developing mode:
+ 1. Invokes executeInDevMode() method.
+
+ The developing mode allows to reuse already installed servers.
+ Servers might be restarted. This mode leads to fast development
+ of test code by doing incremental development. Working code
+ can be moved from executeInDevMode() to execute().
+ """
+
+ def __init__(self, settings, filePath):
+ self.artifactRepository = settings.REPOSITORY
+ self.project = None
+ fileName = os.path.basename(filePath)
+ self.name = fileName[0:fileName.rfind('.')]
+ self.playgroundFolder = "%s/%s" % (PLAYGROUND, self.name)
+ self.numberOfFailures = 0
+ self.devMode = settings.devMode
+ self.runningInstances = []
+
+ def runTest(self):
+ """
+ Runs this test case. This is a final method. It should not be overwritten.
+ """
+ startTime = time.time()
+ util.printAndFlush("\n/''''''''''''''''''' %s started at %s %s ''''''''''"
+ % (self.name, time.strftime('%Y-%m-%d %H:%M:%S'),
+ 'in DEV MODE' if self.devMode else ''))
+ try:
+ if not self.devMode:
+ if os.path.exists(self.playgroundFolder):
+ self._cleanUpPlayground()
+ os.makedirs(self.playgroundFolder)
+ self.execute()
+ else:
+ self.executeInDevMode()
+ success = self.numberOfFailures == 0
+ except:
+ util.printAndFlush(traceback.format_exc())
+ success = False
+ finally:
+ duration = util.renderDuration(time.time() - startTime)
+ if not self.devMode:
+ self.releaseResources()
+ if success:
+ util.printAndFlush(
+ "\...........SUCCESS: %s executed in %s .........." % (self.name, duration))
+ else:
+ util.printAndFlush(
+ "\............FAILED: %s executed in %s .........." % (self.name, duration))
+ raise Exception("%s failed" % self.name)
+
+ def execute(self):
+ """
+ Executes this test case in normal mode.
+ This is an abstract method which has to be overwritten in subclasses.
+ """
+ pass
+
+ def executeInDevMode(self):
+ """
+ Executes this test case in developing mode.
+ This method can be overwritten in subclasses.
+ """
+ pass
+
+ def releaseResources(self):
+ """
+ Releases resources. It shuts down all running servers.
+ This method can be overwritten in subclasses.
+ Note, this method can be invoked in subclasses as follows:
+
+ super(type(self), self).releaseResources()
+
+ """
+ self._shutdownSevers()
+
+ def assertPatternInLog(self, log, pattern):
+ if not re.search(pattern, log):
+ self.fail("Pattern doesn't match: %s" % pattern)
+
+ def assertSmaller(self, itemName, expectedUpperLimit, actualValue, verbose=True):
+ """
+ Asserts that actualValue <= expectedUpperLimit. If not the test will be continued but counted as failed.
+ Returns False if assertion fails otherwise True.
+ """
+ if actualValue > expectedUpperLimit:
+ self.fail("%s\n actual value <%s> exceeds the expected upper limit <%s>" % (
+ itemName, actualValue, expectedUpperLimit))
+ return False
+ elif verbose:
+ util.printAndFlush("%s actual value <%s> is below the expected upper limit <%s>" % (
+ itemName, actualValue, expectedUpperLimit))
+ return True
+
+ def assertEquals(self, itemName, expected, actual, verbose=True):
+ """
+ Asserts that expected == actual. If not the test will be continued but counted as failed.
+ Returns False if assertion fails otherwise True.
+ """
+ rendered_expected = self._render(expected)
+ if expected != actual:
+ rendered_actual = self._render(actual)
+ diff = difflib.ndiff(rendered_expected.splitlines(), rendered_actual.splitlines())
+ self.fail("%s\n Differences:\n%s" % (itemName, '\n'.join(diff)))
+ return False
+ elif verbose:
+ util.printAndFlush("%s as expected: <%s>" % (itemName, rendered_expected))
+ return True
+
+ def assertType(self, variableName, expectedType, variable):
+ self.assertEquals("Type of %s" % variableName, expectedType, type(variable))
+
+ def assertIn(self, itemsName, items, item):
+ if item not in items:
+ self.fail("Item %s not in %s" % (item, itemsName))
+ util.printAndFlush("%s as expected: contains <%s>" % (itemsName, item))
+
+ def assertNone(self, itemName, item):
+ self.assertEquals(itemName, None, item)
+
+ def assertNotNone(self, itemName, item):
+ if item is None:
+ self.fail("Item %s is None" % itemName)
+ util.printAndFlush("%s as expected: not None" % itemName)
+
+ def assertTrue(self, itemName, item):
+ self.assertEquals(itemName, True, item)
+
+ def assertFalse(self, itemName, item):
+ self.assertEquals(itemName, False, item)
+
+ def assertLength(self, itemsName, length, items):
+ self.assertEquals("Length of %s" % itemsName, length, len(items))
+
+ def assertEmpty(self, itemsName, items):
+ self.assertLength(itemsName, 0, items)
+
+ def assertNotEmpty(self, itemsName, items):
+ if len(items) == 0:
+ self.fail("%s is empty" % itemsName)
+ util.printAndFlush("%s as expected: not empty" % itemsName)
+
+ def _render(self, item):
+ if not isinstance(item, list):
+ return str(item)
+ result = ""
+ for e in item:
+ if len(result) > 0:
+ result += "\n"
+ result += str(e)
+ return result
+
+ def fail(self, errorMessage):
+ """
+ Prints specified error message and mark test case as failed.
+ """
+ self.numberOfFailures += 1
+ util.printWhoAmI(levels=10, template="ERROR found (caller chain: %s)")
+ util.printAndFlush("ERROR causing test failure: %s" % errorMessage)
+
+ def installScriptBasedServer(self, templateName, instanceName,
+ startCommand=['./start.sh'], stopCommand=['./stop.sh']):
+ installPath = self._getInstallPath(instanceName)
+ if os.path.exists(installPath):
+ shutil.rmtree(installPath)
+ shutil.copytree("%s/%s" % (self.getTemplatesFolder(), templateName), installPath)
+ return ScriptBasedServerController(self, self.name, installPath, instanceName, startCommand,
+ stopCommand)
+
+ def createScriptBasedServerController(self, instanceName, startCommand=['./start.sh'],
+ stopCommand=['./stop.sh']):
+ return ScriptBasedServerController(self, self.name, self._getInstallPath(instanceName),
+ instanceName,
+ startCommand, stopCommand)
+
+ def installDatamover(self, instanceName='datamover'):
+ zipFile = self.artifactRepository.getPathToArtifact(DATAMOVER_PROJECT, 'datamover')
+ installPath = self._getInstallPath(instanceName)
+ util.unzip(zipFile, self.playgroundFolder)
+ os.rename("%s/datamover" % (self.playgroundFolder), installPath)
+ return DatamoverController(self, self.name, installPath, instanceName)
+
+ def createDatamoverController(self, instanceName='datamover'):
+ return DatamoverController(self, self.name, self._getInstallPath(instanceName),
+ instanceName)
+
+ def installOpenbis(self, instanceName='openbis', technologies=[]):
+ """
+ Installs openBIS from the installer.
+
+ The instanceName specifies the subfolder in the playground folder
+ where the instance will be installed.
+ In addition it is also part of the database names.
+ The technologies are an array of enabled technologies.
+ """
+ installerPath = self.artifactRepository.getPathToArtifact(INSTALLER_PROJECT,
+ 'openBIS-installation')
+ installerFileName = os.path.basename(installerPath).rpartition('.tar')[0]
+ util.executeCommand(['tar', '-zxf', installerPath, '-C', self.playgroundFolder],
+ "Couldn't untar openBIS installer.")
+ consolePropertiesFile = "%s/%s/console.properties" % (
+ self.playgroundFolder, installerFileName)
+ consoleProperties = util.readProperties(consolePropertiesFile)
+ installPath = self._getInstallPath(instanceName)
+ consoleProperties['INSTALL_PATH'] = installPath
+ consoleProperties['DSS_ROOT_DIR'] = "%s/data" % installPath
+ for technology in technologies:
+ consoleProperties[technology.upper()] = True
+ util.writeProperties(consolePropertiesFile, consoleProperties)
+ util.executeCommand("%s/%s/run-console.sh" % (self.playgroundFolder, installerFileName),
+ "Couldn't install openBIS", consoleInput='admin\nadmin')
+ shutil.rmtree("%s/%s" % (self.playgroundFolder, installerFileName))
+
+ def cloneOpenbisInstance(self, nameOfInstanceToBeCloned, nameOfNewInstance,
+ dataStoreServerOnly=False):
+ """ Clones an openBIS instance. """
+
+ oldInstanceInstallPath = "%s/%s" % (self.playgroundFolder, nameOfInstanceToBeCloned)
+ newInstanceInstallPath = "%s/%s" % (self.playgroundFolder, nameOfNewInstance)
+ paths = ['bin', 'data', 'servers/core-plugins', 'servers/datastore_server']
+ if not dataStoreServerOnly:
+ paths.append('servers/openBIS-server')
+ for path in paths:
+ util.copyFromTo(oldInstanceInstallPath, newInstanceInstallPath, path)
+ dssPropsFile = "%s/servers/datastore_server/etc/service.properties" % newInstanceInstallPath
+ dssProps = util.readProperties(dssPropsFile)
+ dssProps['root-dir'] = dssProps['root-dir'].replace(nameOfInstanceToBeCloned,
+ nameOfNewInstance)
+ util.writeProperties(dssPropsFile, dssProps)
+
+ def createOpenbisController(self, instanceName='openbis', port='8443', dropDatabases=True,
+ databasesToDrop=[]):
+ """
+ Creates an openBIS controller object assuming that an openBIS instance for the specified name is installed.
+ """
+ return OpenbisController(self, self.name, self._getInstallPath(instanceName), instanceName,
+ port,
+ dropDatabases, databasesToDrop)
+
+ def installScreeningTestClient(self):
+ """ Installs the screening test client and returns an instance of ScreeningTestClient. """
+ zipFile = self.artifactRepository.getPathToArtifact(OPENBIS_STANDARD_TECHNOLOGIES_PROJECT,
+ 'openBIS-screening-API')
+ installPath = "%s/screeningAPI" % self.playgroundFolder
+ util.unzip(zipFile, installPath)
+ return ScreeningTestClient(self, installPath)
+
+ def installPybis(self):
+ # install the local pybis in editable-mode (-e)
+ util.executeCommand(['pip', 'install', '-e', '../api-openbis-python3-pybis/src/python'],
+ "Installation of pybis failed.")
+
+ def installObis(self):
+ # install the local obis in editable-mode (-e)
+ util.executeCommand(['pip', 'install', '-e', '../app-openbis-command-line/src/python'],
+ "Installation of obis failed.")
+
+ def getTemplatesFolder(self):
+ return "%s/%s" % (TEMPLATES, self.name)
+
+ def _getInstallPath(self, instanceName):
+ return os.path.abspath("%s/%s" % (self.playgroundFolder, instanceName))
+
+ def _cleanUpPlayground(self):
+ for f in os.listdir(self.playgroundFolder):
+ path = "%s/%s" % (self.playgroundFolder, f)
+ if not os.path.isdir(path):
+ continue
+ util.printAndFlush("clean up %s" % path)
+ util.killProcess("%s/servers/datastore_server/datastore_server.pid" % path)
+ util.killProcess("%s/servers/openBIS-server/jetty/openbis.pid" % path)
+ util.killProcess("%s/datamover.pid" % path)
+ util.deleteFolder(self.playgroundFolder)
+
+ def _addToRunningInstances(self, controller):
+ self.runningInstances.append(controller)
+
+ def _removeFromRunningInstances(self, controller):
+ if controller in self.runningInstances:
+ self.runningInstances.remove(controller)
+
+ def _shutdownSevers(self):
+ for instance in reversed(self.runningInstances):
+ instance.stop()
+
+
+class _Controller(object):
+ def __init__(self, testCase, testName, installPath, instanceName):
+ self.testCase = testCase
+ self.testName = testName
+ self.instanceName = instanceName
+ self.installPath = installPath
+ util.printAndFlush("Controller created for instance '%s'. Installation path: %s" % (
+ instanceName, installPath))
+
+ def createFolder(self, folderPath):
+ """
+ Creates a folder with specified path relative to installation directory.
+ """
+ path = "%s/%s" % (self.installPath, folderPath)
+ os.makedirs(path)
+
+ def assertEmptyFolder(self, pathRelativeToInstallPath):
+ """
+ Asserts that the specified path (relative to the installation path) is an empty folder.
+ """
+ relativePath = "%s/%s" % (self.installPath, pathRelativeToInstallPath)
+ files = self._getFiles(relativePath)
+ if len(files) == 0:
+ util.printAndFlush("Empty folder as expected: %s" % relativePath)
+ else:
+ self.testCase.fail(
+ "%s isn't empty. It contains the following files:\n %s" % (relativePath, files))
+
+ def assertFiles(self, folderPathRelativeToInstallPath, expectedFiles):
+ """
+ Asserts that the specified path (relative to the installation path) contains the specified files.
+ """
+ relativePath = "%s/%s" % (self.installPath, folderPathRelativeToInstallPath)
+ files = self._getFiles(relativePath)
+ self.testCase.assertEquals("Files in %s" % relativePath, expectedFiles, sorted(files))
+
+ def _getFiles(self, relativePath):
+ if not os.path.isdir(relativePath):
+ self.testCase.fail("Doesn't exist or isn't a folder: %s" % relativePath)
+ files = os.listdir(relativePath)
+ return files
+
+
+class ScriptBasedServerController(_Controller):
+ def __init__(self, testCase, testName, installPath, instanceName, startCommand, stopCommand):
+ super(ScriptBasedServerController, self).__init__(testCase, testName, installPath,
+ instanceName)
+ self.startCommand = startCommand
+ self.stopCommand = stopCommand
+
+ def start(self):
+ self.testCase._addToRunningInstances(self)
+ util.executeCommand(self.startCommand, "Couldn't start server '%s'" % self.instanceName,
+ workingDir=self.installPath)
+
+ def stop(self):
+ self.testCase._removeFromRunningInstances(self)
+ util.executeCommand(self.stopCommand, "Couldn't stop server '%s'" % self.instanceName,
+ workingDir=self.installPath)
+
+
+class DatamoverController(_Controller):
+ def __init__(self, testCase, testName, installPath, instanceName):
+ super(DatamoverController, self).__init__(testCase, testName, installPath, instanceName)
+ self.servicePropertiesFile = "%s/etc/service.properties" % self.installPath
+ self.serviceProperties = util.readProperties(self.servicePropertiesFile)
+ self.serviceProperties['check-interval'] = 2
+ self.serviceProperties['quiet-period'] = 5
+ self.serviceProperties['inactivity-period'] = 15
+ dataCompletedScript = "%s/%s/data-completed.sh" % (
+ testCase.getTemplatesFolder(), instanceName)
+ if os.path.exists(dataCompletedScript):
+ self.serviceProperties['data-completed-script'] = "../../../../%s" % dataCompletedScript
+
+ def setPrefixForIncoming(self, prefix):
+ """ Set service property 'prefix-for-incoming'. """
+ self.serviceProperties['prefix-for-incoming'] = prefix
+
+ def setTreatIncomingAsRemote(self, flag):
+ """ Set service property 'treat-incoming-as-remote'. """
+ self.serviceProperties['treat-incoming-as-remote'] = flag
+
+ def setOutgoingTarget(self, path):
+ """
+ Set service property 'outgoing-target'.
+ This has to be a path relative to installation path of the datamover.
+ """
+ self.serviceProperties['outgoing-target'] = path
+
+ def setExtraCopyDir(self, path):
+ """
+ Set service property 'extra-copy-dir'.
+ This has to be a path relative to installation path of the datamover.
+ """
+ self.serviceProperties['extra-copy-dir'] = path
+
+ def start(self):
+ """ Starts up datamover server. """
+ util.writeProperties(self.servicePropertiesFile, self.serviceProperties)
+ self.testCase._addToRunningInstances(self)
+ output = util.executeCommand(["%s/datamover.sh" % self.installPath, 'start'],
+ suppressStdOut=True)
+ joinedOutput = '\n'.join(output)
+ if 'FAILED' in joinedOutput:
+ util.printAndFlush(
+ "Start up of datamover %s failed:\n%s" % (self.instanceName, joinedOutput))
+ raise Exception("Couldn't start up datamover '%s'." % self.instanceName)
+
+ def stop(self):
+ """ Stops datamover server. """
+ self.testCase._removeFromRunningInstances(self)
+ util.executeCommand(["%s/datamover.sh" % self.installPath, 'stop'],
+ "Couldn't shut down datamover '%s'." % self.instanceName)
+
+ def drop(self, testDataSetName):
+ """ Drops the specified test data set into incoming folder. """
+ util.copyFromTo("%s/%s" % (TEST_DATA, self.testName), "%s/data/incoming" % self.installPath,
+ testDataSetName)
+
+
+class ScreeningTestClient(object):
+ """
+ Class representing the screeing test client.
+ """
+
+ def __init__(self, testCase, installPath):
+ self.testCase = testCase
+ self.installPath = installPath
+
+ def run(self):
+ """ Runs the test client and returns the console output as a list of strings. """
+ output = util.executeCommand(['java',
+ '-Djavax.net.ssl.trustStore=../openbis/servers/openBIS-server/jetty/etc/openBIS.keystore',
+ '-jar', 'openbis_screening_api.jar', 'admin', 'admin',
+ 'https://localhost:8443'], suppressStdOut=True,
+ workingDir=self.installPath)
+ with open("%s/log.txt" % self.installPath, 'w') as log:
+ for line in output:
+ log.write("%s\n" % line)
+ return output
+
+
+class DataSet(object):
+ def __init__(self, resultSetRow):
+ self.id = resultSetRow[0]
+ self.dataStore = resultSetRow[1]
+ self.experimentCode = resultSetRow[2]
+ self.code = resultSetRow[3]
+ self.type = resultSetRow[4]
+ self.location = resultSetRow[5]
+ self.status = resultSetRow[6]
+ self.presentInArchive = resultSetRow[7]
+ self.producer = resultSetRow[8]
+ self.productionTimeStamp = resultSetRow[9]
+ self.parents = []
+ self.children = []
+
+ def __str__(self):
+ parents = [d.id for d in self.parents]
+ children = [d.id for d in self.children]
+ return "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s" % (
+ self.id, self.dataStore, self.code, self.type,
+ self.location, self.status, self.presentInArchive,
+ parents, children, self.experimentCode,
+ self.producer, self.productionTimeStamp)
+
+
+class OpenbisController(_Controller):
+ """
+ Class to control AS and DSS of an installed openBIS instance.
+ """
+
+ def __init__(self, testCase, testName, installPath, instanceName, port='8443',
+ dropDatabases=True, databasesToDrop=[]):
+ """
+ Creates a new instance for specifies test case with specified test and instance name, installation path.
+ """
+ super(OpenbisController, self).__init__(testCase, testName, installPath, instanceName)
+ self.templatesFolder = testCase.getTemplatesFolder()
+ self.binFolder = "%s/bin" % installPath
+ self.bisUpScript = "%s/bisup.sh" % self.binFolder
+ self.bisDownScript = "%s/bisdown.sh" % self.binFolder
+ self.dssUpScript = "%s/dssup.sh" % self.binFolder
+ self.dssDownScript = "%s/dssdown.sh" % self.binFolder
+ self.databaseKind = "%s_%s" % (testName, instanceName)
+ self.asServicePropertiesFile = "%s/servers/openBIS-server/jetty/etc/service.properties" % installPath
+ self.asProperties = None
+ if os.path.exists(self.asServicePropertiesFile):
+ self.asProperties = util.readProperties(self.asServicePropertiesFile)
+ self.asProperties['database.kind'] = self.databaseKind
+ self.asPropertiesModified = True
+ self.dssServicePropertiesFile = "%s/servers/datastore_server/etc/service.properties" % installPath
+ self.dssProperties = util.readProperties(self.dssServicePropertiesFile)
+ self.dssProperties['path-info-db.databaseKind'] = self.databaseKind
+ self.dssProperties['imaging-database.kind'] = self.databaseKind
+ self.dssPropertiesModified = True
+ self.passwdScript = "%s/servers/openBIS-server/jetty/bin/passwd.sh" % installPath
+ if port != '8443':
+ self.sslIniFile = "%s/servers/openBIS-server/jetty/start.d/ssl.ini" % installPath
+ if os.path.exists(self.sslIniFile):
+ self.sslIni = util.readProperties(self.sslIniFile)
+ self.sslIni['jetty.ssl.port'] = port
+ util.writeProperties(self.sslIniFile, self.sslIni)
+ if dropDatabases:
+ util.dropDatabase(PSQL_EXE, "openbis_%s" % self.databaseKind)
+ util.dropDatabase(PSQL_EXE, "pathinfo_%s" % self.databaseKind)
+ util.dropDatabase(PSQL_EXE, "imaging_%s" % self.databaseKind)
+ self._setUpStore()
+ self._setUpFileServer()
+ for databaseToDrop in databasesToDrop:
+ util.dropDatabase(PSQL_EXE, "%s_%s" % (databaseToDrop, self.databaseKind))
+ self._applyCorePlugins()
+
+ def setDummyAuthentication(self):
+ """ Disables authentication. """
+ self.asProperties['authentication-service'] = 'dummy-authentication-service'
+
+ def setOpenbisPortDataStoreServer(self, port):
+ as_url = self.dssProperties['server-url']
+ util.printAndFlush('as_url' + as_url)
+ parts = as_url.split(':')
+ s = ""
+ for idx, part in enumerate(parts):
+ if (idx < len(parts) - 1):
+ s = s + part + ":"
+ self.dssProperties['server-url'] = s + port
+
+ def setDataStoreServerCode(self, code):
+ """ Sets the code of the Data Store Server. """
+ self.dssProperties['data-store-server-code'] = code
+
+ def getDataStoreServerCode(self):
+ return self.dssProperties['data-store-server-code']
+
+ def setDataStoreServerPort(self, port):
+ """ Sets the port of the Data Store Server. """
+ self.dssProperties['port'] = port
+
+ def setDataStoreServerUsername(self, username):
+ """ Sets the username of the Data Store Server. """
+ self.dssProperties['username'] = username
+
+ def setDataStoreServerProperty(self, prop, val):
+ """ Can be used to set the value of any property in DSS service.properties """
+ self.dssProperties[prop] = val
+
+ def setAsMaxHeapSize(self, maxHeapSize):
+ self._setMaxHeapSize("openBIS-server/jetty/etc/openbis.conf", maxHeapSize)
+
+ def setDssMaxHeapSize(self, maxHeapSize):
+ self._setMaxHeapSize("datastore_server/etc/datastore_server.conf", maxHeapSize)
+
+ def enableProjectSamples(self):
+ self.asProperties['project-samples-enabled'] = "true"
+
+ def assertFileExist(self, pathRelativeToInstallPath):
+ """
+ Asserts that the specified path (relative to the installation path) exists.
+ """
+ relativePath = "%s/%s" % (self.installPath, pathRelativeToInstallPath)
+ if os.path.exists(relativePath):
+ util.printAndFlush("Path exists as expected: %s" % relativePath)
+ else:
+ self.testCase.fail("Path doesn't exist: %s" % relativePath)
+
+ def assertDataSetContent(self, pathToOriginal, dataSet):
+ path = "%s/data/store/1/%s/original" % (self.installPath, dataSet.location)
+ path = "%s/%s" % (path, os.listdir(path)[0])
+ numberOfDifferences = util.getNumberOfDifferences(pathToOriginal, path)
+ if numberOfDifferences > 0:
+ self.testCase.fail("%s differences found." % numberOfDifferences)
+
+ def assertNumberOfDataSets(self, expectedNumberOfDataSets, dataSets):
+ """
+ Asserts that the specified number of data sets from the specified list of DataSet instances
+ are in the data store.
+ """
+ count = 0
+ for dataSet in dataSets:
+ if dataSet.dataStore != self.getDataStoreServerCode() or dataSet.location == '':
+ continue
+ count += 1
+ self.assertFileExist("data/store/1/%s" % dataSet.location)
+ self.testCase.assertEquals(
+ "Number of data sets in data store %s" % self.getDataStoreServerCode(),
+ expectedNumberOfDataSets, count)
+
+ def storeDirectory(self):
+ """
+ Return the path to the data/store directory
+ """
+ return "data/store"
+
+ def getDataSets(self):
+ """
+ Returns all data sets as a list (ordered by data set ids) of instances of class DataSet.
+ """
+ resultSet = self.queryDatabase('openbis',
+ "select data.id,ds.code,e.code,data.code,t.code,location,status,present_in_archive,"
+ + " data.data_producer_code,data.production_timestamp from data"
+ + " left join external_data as ed on ed.id = data.id"
+ + " join data_set_types as t on data.dsty_id = t.id"
+ + " join experiments as e on data.expe_id = e.id"
+ + " join data_stores as ds on data.dast_id = ds.id order by data.id")
+ dataSets = []
+ dataSetsById = {}
+ for row in resultSet:
+ dataSet = DataSet(row)
+ dataSets.append(dataSet)
+ dataSetsById[dataSet.id] = dataSet
+ relationships = self.queryDatabase('openbis',
+ "select data_id_parent, data_id_child from data_set_relationships"
+ + " order by data_id_parent, data_id_child")
+ for parent_id, child_id in relationships:
+ parent = dataSetsById[parent_id]
+ child = dataSetsById[child_id]
+ parent.children.append(child)
+ child.parents.append(parent)
+ util.printAndFlush(
+ "All data sets:\nid,dataStore,code,type,location,status,presentInArchive,parents,children,experiment,producer,productionTimeStamp")
+ for dataSet in dataSets:
+ util.printAndFlush(dataSet)
+ return dataSets
+
+ def createTestDatabase(self, databaseType):
+ """
+ Creates a test database for the specified database type.
+ """
+ database = "%s_%s" % (databaseType, self.databaseKind)
+ scriptPath = "%s/%s.sql" % (self.templatesFolder, database)
+ util.createDatabase(PSQL_EXE, database, scriptPath)
+
+ def dropDatabase(self, databaseType):
+ """
+ Drops the database for the specified database type.
+ """
+ util.dropDatabase(PSQL_EXE, "%s_%s" % (databaseType, self.databaseKind))
+
+ def queryDatabase(self, databaseType, queryStatement, showHeaders=False):
+ """
+ Executes the specified SQL statement for the specified database type. Result set is returned
+ as a list of lists.
+ """
+ database = "%s_%s" % (databaseType, self.databaseKind)
+ return util.queryDatabase(PSQL_EXE, database, queryStatement, showHeaders)
+
+ def allUp(self):
+ """ Starts up AS and DSS if not running. """
+ if not util.isAlive("%s/servers/openBIS-server/jetty/openbis.pid" % self.installPath,
+ "openBIS.keystore"):
+ self._saveAsPropertiesIfModified()
+ util.executeCommand([self.bisUpScript],
+ "Starting up openBIS AS '%s' failed." % self.instanceName)
+ self.dssUp()
+
+ def stop(self):
+ self.allDown()
+
+ def allDown(self):
+ """ Shuts down AS and DSS. """
+ self.testCase._removeFromRunningInstances(self)
+ util.executeCommand([self.dssDownScript],
+ "Shutting down openBIS DSS '%s' failed." % self.instanceName)
+ if self.asProperties:
+ util.executeCommand([self.bisDownScript],
+ "Shutting down openBIS AS '%s' failed." % self.instanceName)
+
+ def dssUp(self):
+ """ Starts up DSS if not running. """
+ if not util.isAlive("%s/servers/datastore_server/datastore_server.pid" % self.installPath,
+ "openBIS.keystore"):
+ self._saveDssPropertiesIfModified()
+ self.testCase._addToRunningInstances(self)
+ util.executeCommand([self.dssUpScript],
+ "Starting up openBIS DSS '%s' failed." % self.instanceName)
+
+ def dssDown(self):
+ """ Shuts down DSS. """
+ self.testCase._removeFromRunningInstances(self)
+ util.executeCommand([self.dssDownScript],
+ "Shutting down openBIS DSS '%s' failed." % self.instanceName)
+
+ def dropAndWait(self, dataName, dropBoxName, numberOfDataSets=1,
+ timeOutInMinutes=DEFAULT_TIME_OUT_IN_MINUTES):
+ """
+ Drops the specified data into the specified drop box. The data is either a folder or a ZIP file
+ in TEST_DATA/<test name>. A ZIP file will be unpacked in the drop box. After dropping the method waits
+ until the specified number of data sets have been registered.
+ """
+ self.drop(dataName, dropBoxName)
+ self.waitUntilDataSetRegistrationFinished(numberOfDataSets=numberOfDataSets,
+ timeOutInMinutes=timeOutInMinutes)
+
+ def dataFile(self, dataName):
+ """
+ Returns the path to the given test data
+ """
+ return "%s/%s/%s" % (TEST_DATA, self.testName, dataName)
+
+ def drop(self, dataName, dropBoxName):
+ """
+ Drops the specified test data into the specified drop box. The test data is either a folder or a ZIP file
+ in TEST_DATA/<test name>. A ZIP file will be unpacked in the drop box.
+ """
+ destination = "%s/data/%s" % (self.installPath, dropBoxName)
+ self.dropIntoDestination(dataName, destination)
+
+ def dropIntoDestination(self, dataName, destination):
+ """
+ Drops the specified test data into the destination. The test data is either a folder or a ZIP file
+ in TEST_DATA/<test name>. A ZIP file will be unpacked in the drop box.
+ """
+ testDataFolder = "%s/%s" % (TEST_DATA, self.testName)
+ if dataName.endswith('.zip'):
+ util.unzip("%s/%s" % (testDataFolder, dataName), destination)
+ else:
+ util.copyFromTo(testDataFolder, destination, dataName)
+
+ def waitUntilDataSetRegistrationFinished(self, numberOfDataSets=1,
+ timeOutInMinutes=DEFAULT_TIME_OUT_IN_MINUTES):
+ """ Waits until the specified number of data sets have been registrated. """
+ monitor = self.createLogMonior(timeOutInMinutes)
+ monitor.addNotificationCondition(util.RegexCondition('Incoming Data Monitor'))
+ monitor.addNotificationCondition(util.RegexCondition('post-registration'))
+ numberOfRegisteredDataSets = 0
+ while numberOfRegisteredDataSets < numberOfDataSets:
+ condition1 = util.RegexCondition('Post registration of (\\d*). of \\1 data sets')
+ condition2 = util.RegexCondition(
+ 'Paths inside data set .* successfully added to database')
+ elements = monitor.waitUntilEvent(util.ConditionSequence([condition1, condition2]))
+ numberOfRegisteredDataSets += int(elements[0][0])
+ util.printAndFlush(
+ "%d of %d data sets registered" % (numberOfRegisteredDataSets, numberOfDataSets))
+
+ def waitUntilDataSetRegistrationFailed(self, timeOutInMinutes=DEFAULT_TIME_OUT_IN_MINUTES):
+ """ Waits until data set registration failed. """
+ self.waitUntilConditionMatched(util.EventTypeCondition('ERROR'), timeOutInMinutes)
+ util.printAndFlush("Data set registration failed as expected.")
+
+ def waitUntilConditionMatched(self, condition, timeOutInMinutes=DEFAULT_TIME_OUT_IN_MINUTES):
+ """
+ Waits until specified condition has been detected in DSS log.
+ """
+ monitor = self.createLogMonior(timeOutInMinutes)
+ monitor.addNotificationCondition(util.RegexCondition('Incoming Data Monitor'))
+ monitor.addNotificationCondition(util.RegexCondition('post-registration'))
+ monitor.waitUntilEvent(condition)
+
+ def createLogMonior(self, timeOutInMinutes=DEFAULT_TIME_OUT_IN_MINUTES):
+ logFilePath = "%s/servers/datastore_server/log/datastore_server_log.txt" % self.installPath
+ return util.LogMonitor("%s.DSS" % self.instanceName, logFilePath, timeOutInMinutes)
+
+ def assertFeatureVectorLabel(self, featureCode, expectedFeatureLabel):
+ data = self.queryDatabase('imaging',
+ "select distinct label from feature_defs where code = '%s'" % featureCode)
+ self.testCase.assertEquals("label of feature %s" % featureCode, [[expectedFeatureLabel]],
+ data)
+
+ def _applyCorePlugins(self):
+ source = "%s/core-plugins/%s" % (self.templatesFolder, self.instanceName)
+ if os.path.exists(source):
+ corePluginsFolder = "%s/servers/core-plugins" % self.installPath
+ destination = "%s/%s" % (corePluginsFolder, self.instanceName)
+ shutil.rmtree(destination, ignore_errors=True)
+ shutil.copytree(source, destination)
+ self.enableCorePlugin(self.instanceName)
+
+ def enableCorePlugin(self, pluginName):
+ corePluginsFolder = "%s/servers/core-plugins" % self.installPath
+ corePluginsPropertiesFile = "%s/core-plugins.properties" % corePluginsFolder
+ corePluginsProperties = util.readProperties(corePluginsPropertiesFile)
+ enabledModules = corePluginsProperties['enabled-modules']
+ enabledModules = "%s, %s" % (enabledModules, pluginName) if len(
+ enabledModules) > 0 else pluginName
+ corePluginsProperties['enabled-modules'] = enabledModules
+ util.writeProperties(corePluginsPropertiesFile, corePluginsProperties)
+
+ def addUser(self, name, password):
+ util.executeCommand([self.passwdScript, 'add', name, '-p', password],
+ "Could not add user '%s' to instance '%s'." % (name, self.instanceName))
+
+ def _setUpStore(self):
+ templateStore = "%s/stores/%s" % (self.templatesFolder, self.instanceName)
+ if os.path.isdir(templateStore):
+ storeFolder = "%s/data/store" % self.installPath
+ util.printAndFlush("Set up initial data store by copying content of %s to %s" % (
+ templateStore, storeFolder))
+ shutil.rmtree(storeFolder, ignore_errors=True)
+ shutil.copytree(templateStore, storeFolder)
+
+ def _setUpFileServer(self):
+ templateFileServer = "%s/file-servers/%s" % (self.templatesFolder, self.instanceName)
+ if os.path.isdir(templateFileServer):
+ fileServiceFolder = "%s/data/file-server" % self.installPath
+ util.printAndFlush("Set up initial file server by copying content of %s to %s" % (
+ templateFileServer, fileServiceFolder))
+ shutil.rmtree(fileServiceFolder, ignore_errors=True)
+ shutil.copytree(templateFileServer, fileServiceFolder)
+
+ def _saveAsPropertiesIfModified(self):
+ if self.asPropertiesModified:
+ util.writeProperties(self.asServicePropertiesFile, self.asProperties)
+ self.asPropertiesModified = False
+
+ def _saveDssPropertiesIfModified(self):
+ if self.dssPropertiesModified:
+ util.writeProperties(self.dssServicePropertiesFile, self.dssProperties)
+ self.dssPropertiesModified = False
+
+ def _setMaxHeapSize(self, configFile, maxHeapSize):
+ path = "%s/servers/%s" % (self.installPath, configFile)
+ lines = []
+ for line in util.getContent(path):
+ if line.strip().startswith('JAVA_MEM_OPTS'):
+ line = re.sub(r'(.*)-Xmx[^ ]+(.*)', r"\1-Xmx%s\2" % maxHeapSize, line)
+ lines.append(line)
+ with open(path, "w") as f:
+ for line in lines:
+ f.write("%s\n" % line)
diff --git a/api-openbis-python3-pybis/src/python/tests/systemtest/util.py b/api-openbis-python3-pybis/src/python/tests/systemtest/util.py
new file mode 100644
index 0000000000000000000000000000000000000000..d1e0c4305ceea13a839f109be27c2251a9d94278
--- /dev/null
+++ b/api-openbis-python3-pybis/src/python/tests/systemtest/util.py
@@ -0,0 +1,412 @@
+# Copyright ETH 2013 - 2023 Zürich, Scientific IT Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import filecmp
+import inspect
+import os
+import os.path
+import re
+import shutil
+import subprocess
+import sys
+import time
+import zipfile
+
+USER=os.environ['USER']
+DEFAULT_WHO_AM_I_TEMPLATE="""
+
+/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\
+\\/\\/\/ %s
+/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\/\\
+
+"""
+
+def printWhoAmI(levels = 1, template = DEFAULT_WHO_AM_I_TEMPLATE):
+ """
+ Prints the names of the functions in the caller chain of this function up to the specified number of levels.
+ """
+ stack = inspect.stack()
+ chain = ''
+ for i in range(1, min(levels + 1, len(stack))):
+ stack_entry = stack[i]
+ location = "%s:%s" % (stack_entry[3], stack_entry[2])
+ chain = "%s > %s" % (location, chain) if chain != '' else location
+ printAndFlush(template % chain)
+
+def printAndFlush(data):
+ """
+ Prints argument onto the standard console and flushes output.
+ This is necessary to get Python output and bash output in sync on CI server.
+ """
+ print(data)
+ sys.stdout.flush()
+
+
+def readProperties(propertiesFile):
+ """
+ Reads a Java properties file and returns the key-value pairs as a dictionary.
+ """
+ with open(propertiesFile, "r") as f:
+ result = {}
+ for line in f.readlines():
+ trimmedLine = line.strip()
+ if len(trimmedLine) > 0 and not trimmedLine.startswith('#'):
+ splittedLine = line.split('=', 1)
+ key = splittedLine[0].strip()
+ value = splittedLine[1].strip()
+ result[key] = value
+ return result
+
+def writeProperties(propertiesFile, dictionary):
+ """
+ Saves the specified dictionary as a Java properties file.
+ """
+ with open(propertiesFile, "w") as f:
+ for key in sorted(dictionary):
+ f.write("%s=%s\n" % (key, dictionary[key]))
+
+def executeCommand(commandWithArguments, failingMessage = None, consoleInput = None, suppressStdOut = False,
+ workingDir = None):
+ """
+ Executes specified command with arguments.
+ If the exit value of the command is not zero and a failing message has been specified
+ an exception with the failing message will be thrown.
+ Optionally a string for console input can be specified.
+ If flag suppressStdOut is set standard output will be suppressed but returned as a list of output lines.
+ If workingDir is specified a change to workingDir is done for execution.
+ """
+ printAndFlush("\n------- START: %s" % commandWithArguments)
+ currentDir = None
+ if workingDir != None:
+ printAndFlush("change to working directory '%s'" % workingDir)
+ currentDir = os.getcwd()
+ os.chdir(workingDir)
+ try:
+ processIn = subprocess.PIPE if consoleInput != None else None
+ processOut = subprocess.PIPE if suppressStdOut else None
+ # Setting the time zone is needed for sprint server otherwise Java log files have wrong time zone
+ os.environ['TZ'] = time.tzname[0]
+ p = subprocess.Popen(commandWithArguments, stdin = processIn, stdout = processOut, encoding='utf8')
+ if consoleInput != None:
+ p.communicate(consoleInput)
+ lines = []
+ if suppressStdOut:
+ for line in iter(p.stdout.readline,''):
+ lines.append(line.strip())
+ exitValue = p.wait()
+ if currentDir != None:
+ printAndFlush("change back to previous working directory '%s'" % currentDir)
+ if exitValue != 0 and failingMessage != None:
+ printAndFlush("---- FAILED %d: %s" % (exitValue, commandWithArguments))
+ raise Exception(failingMessage)
+ printAndFlush("---- FINISHED: %s" % commandWithArguments)
+ return lines
+ finally:
+ if currentDir != None:
+ os.chdir(currentDir)
+
+
+
+def killProcess(pidFile):
+ """
+ Kills the process in specified PID file. Does nothing if PID file doesn't exist.
+ """
+ pid = getPid(pidFile)
+ if pid is None:
+ return
+ executeCommand(['kill', pid])
+
+def isAlive(pidFile, pattern):
+ """
+ Checks if the process with PID in specified file is alive. The specified regex
+ is used to check that the process of expected PID is the process expected.
+ """
+ pid = getPid(pidFile)
+ if pid is None:
+ return False
+ lines = executeCommand(['ps', '-p', pid], suppressStdOut=True)
+ if len(lines) < 2:
+ return False
+ return re.compile(pattern).search(lines[1]) is not None
+
+
+def getPid(pidFile):
+ if not os.path.exists(pidFile):
+ return None
+ return readFirstLine(pidFile)
+
+def readFirstLine(textFile):
+ """
+ Returns the first line of the specified textFile.
+ """
+ with open(textFile, 'r') as handle:
+ return handle.readline().rstrip()
+
+def unzip(zipFile, destination):
+ """
+ Unzips specified ZIP file at specified destination.
+ """
+ executeCommand(['unzip', '-q', '-o', zipFile, '-d', destination], "Couldn't unzip %s at %s" % (zipFile, destination))
+
+def unzipSubfolder(zipFile, subfolder, destination):
+ """
+ Unzips the specified subtree from the specified ZIP file into the specified destination
+ """
+ zf = zipfile.ZipFile(zipFile)
+ parent, name = os.path.split(subfolder)
+ if name == '':
+ parent = os.path.dirname(parent)
+ for entry in zf.namelist():
+ if entry.startswith(subfolder):
+ newPath = entry.replace(parent, destination)
+ newPathParent = os.path.dirname(newPath)
+ if not os.path.exists(newPathParent):
+ os.makedirs(newPathParent)
+ if not newPath.endswith('/'):
+ data = zf.read(entry)
+ with open(newPath, 'wb') as out:
+ out.write(data)
+
+def deleteFolder(folderPath):
+ """
+ Deletes the specified folder.
+ Raises an exception in case of error.
+ """
+ printAndFlush("Delete '%s'" % folderPath)
+ def errorHandler(*args):
+ _, path, _ = args
+ raise Exception("Couldn't delete '%s'" % path)
+ shutil.rmtree(folderPath, onerror = errorHandler)
+
+def copyFromTo(sourceFolder, destinationFolder, relativePathInSourceFolder):
+ source = "%s/%s" % (sourceFolder, relativePathInSourceFolder)
+ destination = "%s/%s" % (destinationFolder, relativePathInSourceFolder)
+ if os.path.isfile(source):
+ shutil.copyfile(source, destination)
+ else:
+ shutil.copytree(source,
+ destination, ignore = shutil.ignore_patterns(".*"))
+ printAndFlush("'%s' copied from '%s' to '%s'" % (relativePathInSourceFolder, sourceFolder, destinationFolder))
+
+def getDatabaseHost():
+ host = os.environ.get('FORCE_OPENBIS_POSTGRES_HOST')
+ if (host is None):
+ host = "localhost"
+ return host
+
+def dropDatabase(psqlExe, database):
+ """
+ Drops the specified database by using the specified path to psql.
+ """
+ executeCommand([psqlExe,
+ '-h', getDatabaseHost(),
+ '-U', 'postgres', '-c' , "drop database if exists %s" % database],
+ "Couldn't drop database %s" % database)
+
+def createDatabase(psqlExe, database, scriptPath = None):
+ """
+ Creates specified database and run (if defined) the specified SQL script.
+ """
+ executeCommand([psqlExe,
+ '-h', getDatabaseHost(),
+ '-U', 'postgres', '-c' , "create database %s with owner %s" % (database, USER)],
+ "Couldn't create database %s" % database)
+ if scriptPath == None:
+ return
+ executeCommand([psqlExe,
+ '-h', getDatabaseHost(),
+ '-q', '-U', USER, '-d', database, '-f', scriptPath], suppressStdOut=True,
+ failingMessage="Couldn't execute script %s for database %s" % (scriptPath, database))
+
+def queryDatabase(psqlExe, database, queryStatement, showHeaders = False):
+ """
+ Queries specified database by applying specified SQL statement and returns the result set as a list
+ where each row is a list, too.
+ """
+ printingOption = '-A' if showHeaders else '-tA'
+ lines = executeCommand([psqlExe,
+ '-h', getDatabaseHost(),
+ '-U', 'postgres', printingOption, '-d', database, '-c', queryStatement],
+ "Couldn't execute query: %s" % queryStatement, suppressStdOut = True)
+ result = []
+ for line in lines:
+ result.append(line.split('|'))
+ return result
+
+def printResultSet(resultSet):
+ """
+ Prints the specified result set.
+ """
+ for row in resultSet:
+ printAndFlush(row)
+
+def getNumberOfDifferences(fileOrFolder1, fileOrFolder2):
+ """
+ Gets and reports differences in file system structures between both arguments.
+ """
+ result = filecmp.dircmp(fileOrFolder1, fileOrFolder2, ignore=['.svn'])
+ result.report()
+ return len(result.left_only) + len(result.right_only) + len(result.diff_files)
+
+def getContent(path):
+ """
+ Returns the content at specified path as an array of lines. Trailing white spaces (including new line)
+ has been stripped off.
+ """
+ with open(path, "r") as f:
+ return [ l.rstrip() for l in f.readlines()]
+
+def renderDuration(duration):
+ renderedDuration = renderNumber(duration, 'second')
+ if duration > 80:
+ minutes = duration / 60
+ seconds = duration % 60
+ if seconds > 0:
+ renderedDuration = "%s and %s" % (renderNumber(minutes, 'minute'), renderNumber(seconds, 'second'))
+ else:
+ renderedDuration = renderNumber(minutes, 'minute')
+ return renderedDuration
+
+def renderNumber(number, unit):
+ return ("1 %s" % unit) if number == 1 else ("%d %ss" % (number, unit))
+
+
+class LogMonitor():
+ """
+ Monitor of a log file. Conditions can be specified for printing a notification and waiting.
+
+ A condition has to be a class with method 'match' which has two string arguments:
+ Event type and log message. It returns 'None' in case of no match and
+ a tuple with zero or more matching elements found in log message.
+ """
+ def __init__(self, logName, logFilePath, timeOutInMinutes = 5):
+ """
+ Creates an instance with specified log name (used in notification), log file, and time out.
+ """
+ self.logName = logName
+ self.logFilePath = logFilePath
+ self.timeOutInMinutes = timeOutInMinutes
+ self.conditions = []
+ self.timeProvider = time
+ class SystemPrinter:
+ def printMsg(self, msg):
+ printAndFlush(msg)
+ self.printer = SystemPrinter()
+
+ def addNotificationCondition(self, condition):
+ """
+ Adds a notification condition
+ """
+ self.conditions.append(condition)
+
+ def getFormattedTime(self, timeSec):
+ if timeSec is None:
+ self.printer.printMsg("Error: Provided time is None!")
+ return time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(timeSec))
+
+
+ def waitUntilEvent(self, condition, startTime = None, delay = 0):
+ """
+ Waits until an event matches the specified condition.
+ Returns tuple with zero or more elements of matching log message.
+ """
+ startTime = self.timeProvider.time() if startTime is None else startTime
+ self.conditions.append(condition)
+ renderedStartTime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(startTime))
+ self.printer.printMsg("\n>>>>> Start monitoring %s log at %s >>>>>>>>>>>>>>>>>>>>"
+ % (self.logName, renderedStartTime))
+ finalTime = startTime + self.timeOutInMinutes * 60
+ if delay > 0:
+ time.sleep(delay)
+ try:
+ alreadyPrintedLines = set()
+ while True:
+ log = open(self.logFilePath, 'r')
+ while True:
+ actualTime = self.timeProvider.time()
+ if actualTime > finalTime:
+ self.printer.printMsg(f"Time out detected! start time: {renderedStartTime}, calculated end time: {self.getFormattedTime(finalTime)}, current time: {self.getFormattedTime(actualTime)}, timeout (min): {self.timeOutInMinutes}")
+ raise Exception("Time out after %d minutes for monitoring %s log."
+ % (self.timeOutInMinutes, self.logName))
+ line = log.readline()
+ if line == '':
+ break
+ match = re.match('(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}),(\d{3}) (.{6})(.*)', line)
+ if match == None:
+ continue
+ timestamp = match.group(1)
+ milliseconds = int(match.group(2))
+ eventType = match.group(3).strip()
+ message = match.group(4)
+ eventTime = time.mktime(time.strptime(timestamp, '%Y-%m-%d %H:%M:%S')) + 0.001 * milliseconds
+ if eventTime < startTime:
+ continue
+ for c in self.conditions:
+ if c.match(eventType, message) != None and not line in alreadyPrintedLines:
+ alreadyPrintedLines.add(line)
+ self.printer.printMsg(">> %s" % line.strip())
+ break
+ elements = condition.match(eventType, message)
+ if elements != None:
+ return elements
+ log.seek(0, os.SEEK_CUR)
+ time.sleep(2)
+ finally:
+ self.printer.printMsg(">>>>> Finished monitoring %s log >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
+ % self.logName)
+
+class EventTypeCondition():
+ """ A condition which matches in case of specified event type. """
+ def __init__(self, eventType):
+ self.eventType = eventType
+
+ def match(self, eventType, message):
+ return () if self.eventType == eventType else None
+
+class StartsWithCondition():
+ """
+ A condition which matches if the message starts with a specified string.
+ """
+ def __init__(self, startsWithString):
+ self.startsWithString = startsWithString
+
+ def match(self, eventType, message):
+ return () if message.startswith(self.startsWithString) else None
+
+class RegexCondition():
+ """
+ A condition which matches if the message matches a specified regular expression.
+ """
+ def __init__(self, regex):
+ self.regex = regex
+
+ def match(self, eventType, message):
+ match = re.search(self.regex, message)
+ return match.groups() if match else None
+
+class ConditionSequence():
+ def __init__(self, conditions):
+ self.conditions = conditions
+ self.matches = []
+
+ def match(self, eventType, message):
+ match_count = len(self.matches)
+ if match_count == len(self.conditions):
+ return self.matches
+ result = self.conditions[match_count].match(eventType, message)
+ if result is not None:
+ self.matches.append(result)
+ if len(self.matches) == len(self.conditions):
+ return self.matches
+ return None
\ No newline at end of file