testcase.py

import re
import os
import os.path
import shutil
import time
import traceback

import util

INSTALLER_PROJECT = 'installation'
OPENBIS_STANDARD_TECHNOLOGIES_PROJECT = 'openbis_standard_technologies'
DATAMOVER_PROJECT = 'datamover'

PSQL_EXE = 'psql'

PLAYGROUND = 'targets/playground'
TEMPLATES = 'templates'
TEST_DATA = 'testData'

DEFAULT_TIME_OUT_IN_MINUTES = 5

class TestCase(object):
    """
    Abstract superclass of a test case. 
    Subclasses have to override execute() and optionally executeInDevMode(). 
    The test case is run by invoking runTest(). 
    Here is a skeleton of a test case:
    
    #!/usr/bin/python
    import settings
    import systemtest.testcase

    class TestCase(systemtest.testcase.TestCase):
    
        def execute(self):
            ....
            
        def executeInDevMode(self):
            ....
            
    TestCase(settings, __file__).runTest()

    There are two execution modes (controlled by command line option -d and -rd):
    
    Normal mode: 
        1. Cleans up playground: Kills running servers and deletes playground folder of this test case.
        2. Invokes execute() method.
        3. Release resources: Shuts down running servers.
        
    Developing mode:
        1. Invokes executeInDevMode() method.
        
    The developing mode allows to reuse already installed servers. 
    Servers might be restarted. This mode leads to fast development 
    of test code by doing incremental development. Working code
    can be moved from executeInDevMode() to execute(). 
    """
    def __init__(self, settings, filePath):
        self.artifactRepository = settings.REPOSITORY
        self.project = None
        fileName = os.path.basename(filePath)
        self.name = fileName[0:fileName.rfind('.')]
        self.playgroundFolder = "%s/%s" % (PLAYGROUND, self.name)
        self.numberOfFailures = 0
        self.devMode = settings.devMode
        self.runningInstances = []

    def runTest(self):
        """
        Runs this test case. This is a final method. It should not be overwritten.
        """
        startTime = time.time()
        util.printAndFlush("\n/''''''''''''''''''' %s started at %s %s ''''''''''" 
                           % (self.name, time.strftime('%Y-%m-%d %H:%M:%S'), 'in DEV MODE' if self.devMode else ''))
        try:
            if not self.devMode:
                if os.path.exists(self.playgroundFolder):
                    self._cleanUpPlayground()
                os.makedirs(self.playgroundFolder)
                self.execute()
            else:
                self.executeInDevMode()
            success = self.numberOfFailures == 0
        except:
            traceback.print_exc()
            success = False
        finally:
            duration = util.renderDuration(time.time() - startTime)
            if not self.devMode:
                self.releaseResources()
            if success:
                util.printAndFlush("\...........SUCCESS: %s executed in %s .........." % (self.name, duration))
            else:
                util.printAndFlush("\............FAILED: %s executed in %s .........." % (self.name, duration))
                raise Exception("%s failed" % self.name)
        
    def execute(self):
        """
        Executes this test case in normal mode. 
        This is an abstract method which has to be overwritten in subclasses.
        """
        pass
    
    def executeInDevMode(self):
        """
        Executes this test case in developing mode. 
        This method can be overwritten in subclasses.
        """
        pass
    
    def releaseResources(self):
        """
        Releases resources. It shuts down all running servers.
        This method can be overwritten in subclasses. 
        Note, this method can be invoked in subclasses as follows:
        
                super(type(self), self).releaseResources()
        
        """
        self._shutdownSevers()
        
    def assertPatternInLog(self, log, pattern):
        if not re.search(pattern, log):
            self.fail("Pattern doesn't match: %s" % pattern)

    def assertSmaller(self, itemName, expectedUpperLimit, actualValue, verbose=True):
        """
        Asserts that actualValue <= expectedUpperLimit. If not the test will be continued but counted as failed.
        Returns False if assertion fails otherwise True.
        """
        if actualValue > expectedUpperLimit:
            self.fail("%s\n  actual value <%s> exceeds the expected upper limit <%s>" % (itemName, actualValue, expectedUpperLimit))
            return False
        elif verbose:
            util.printAndFlush("%s actual value <%s> is below the expected upper limit <%s>" % (itemName, actualValue, expectedUpperLimit))
        return True

    def assertEquals(self, itemName, expected, actual, verbose=True):
        """
        Asserts that expected == actual. If not the test will be continued but counted as failed.
        Returns False if assertion fails otherwise True.
        """
        if expected != actual:
            self.fail("%s\n  expected: <%s>\n   but was: <%s>" % (itemName, expected, actual))
            return False
        elif verbose:
            util.printAndFlush("%s as expected: <%s>" % (itemName, expected))
        return True
    
    def fail(self, errorMessage):
        """
        Prints specified error message and mark test case as failed.
        """
        self.numberOfFailures += 1
        util.printWhoAmI(levels = 10, template = "ERROR found (caller chain: %s)")
        util.printAndFlush("ERROR causing test failure: %s" % errorMessage)
        
    def installScriptBasedServer(self, templateName, instanceName, 
                                 startCommand = ['./start.sh'], stopCommand = ['./stop.sh']):
        installPath = self._getInstallPath(instanceName)
        if os.path.exists(installPath):
            shutil.rmtree(installPath)
        shutil.copytree("%s/%s" % (self.getTemplatesFolder(), templateName), installPath)
        return ScriptBasedServerController(self, self.name, installPath, instanceName, startCommand, stopCommand)
        
    def createScriptBasedServerController(self, instanceName, startCommand = ['./start.sh'], stopCommand = ['./stop.sh']):
        return ScriptBasedServerController(self, self.name, self._getInstallPath(instanceName), instanceName, 
                                           startCommand, stopCommand)
        
    def installDatamover(self, instanceName = 'datamover'):
        zipFile = self.artifactRepository.getPathToArtifact(DATAMOVER_PROJECT, 'datamover')
        installPath = self._getInstallPath(instanceName)
        util.unzip(zipFile, self.playgroundFolder)
        os.rename("%s/datamover" % (self.playgroundFolder), installPath)
        return DatamoverController(self, self.name, installPath, instanceName)
    
    def createDatamoverController(self, instanceName = 'datamover'):
        return DatamoverController(self, self.name, self._getInstallPath(instanceName), instanceName)

    def installOpenbis(self, instanceName = 'openbis', technologies = []):
        """
        Installs openBIS from the installer. 
        
        The instanceName specifies the subfolder in the playground folder 
        where the instance will be installed. 
        In addition it is also part of the database names.
        The technologies are an array of enabled technologies.
        """
        installerPath = self.artifactRepository.getPathToArtifact(INSTALLER_PROJECT, 'openBIS-installation')
        installerFileName = os.path.basename(installerPath).rpartition('.tar')[0]
        util.executeCommand(['tar', '-zxf', installerPath, '-C', self.playgroundFolder], 
                            "Couldn't untar openBIS installer.")
        consolePropertiesFile = "%s/%s/console.properties" % (self.playgroundFolder, installerFileName)
        consoleProperties = util.readProperties(consolePropertiesFile)
        installPath = self._getInstallPath(instanceName)
        consoleProperties['INSTALL_PATH'] = installPath
        consoleProperties['DSS_ROOT_DIR'] = "%s/data" % installPath
        for technology in technologies:
            consoleProperties[technology.upper()] = True
        util.writeProperties(consolePropertiesFile, consoleProperties)
        util.executeCommand("%s/%s/run-console.sh" % (self.playgroundFolder, installerFileName), 
                            "Couldn't install openBIS", consoleInput='admin\nadmin')
        shutil.rmtree("%s/%s" % (self.playgroundFolder, installerFileName))
        
    def cloneOpenbisInstance(self, nameOfInstanceToBeCloned, nameOfNewInstance, dataStoreServerOnly = False):
        """ Clones an openBIS instance. """
        
        oldInstanceInstallPath = "%s/%s" % (self.playgroundFolder, nameOfInstanceToBeCloned)
        newInstanceInstallPath = "%s/%s" % (self.playgroundFolder, nameOfNewInstance)
        paths = ['bin', 'data', 'servers/core-plugins', 'servers/datastore_server']
        if not dataStoreServerOnly:
            paths.append('servers/openBIS-server')
        for path in paths:
            util.copyFromTo(oldInstanceInstallPath, newInstanceInstallPath, path)
        dssPropsFile = "%s/servers/datastore_server/etc/service.properties" % newInstanceInstallPath
        dssProps = util.readProperties(dssPropsFile)
        dssProps['root-dir'] = dssProps['root-dir'].replace(nameOfInstanceToBeCloned, nameOfNewInstance)
        util.writeProperties(dssPropsFile, dssProps)
        
    
    def createOpenbisController(self, instanceName = 'openbis', dropDatabases = True, databasesToDrop = []):
        """
        Creates an openBIS controller object assuming that an openBIS instance for the specified name is installed.
        """
        return OpenbisController(self, self.name, self._getInstallPath(instanceName), instanceName, 
                                 dropDatabases, databasesToDrop)
    
    def installScreeningTestClient(self):
        """ Installs the screening test client and returns an instance of ScreeningTestClient. """
        zipFile = self.artifactRepository.getPathToArtifact(OPENBIS_STANDARD_TECHNOLOGIES_PROJECT, 'openBIS-screening-API')
        installPath = "%s/screeningAPI" % self.playgroundFolder
        util.unzip(zipFile, installPath)
        return ScreeningTestClient(self, installPath)
    
    def getTemplatesFolder(self):
        return "%s/%s" % (TEMPLATES, self.name)
    
    def _getInstallPath(self, instanceName):
        return os.path.abspath("%s/%s" % (self.playgroundFolder, instanceName))

    def _cleanUpPlayground(self):
        for f in os.listdir(self.playgroundFolder):
            path = "%s/%s" % (self.playgroundFolder, f)
            if not os.path.isdir(path):
                continue
            util.printAndFlush("clean up %s" % path)
            util.killProcess("%s/servers/datastore_server/datastore_server.pid" % path)
            util.killProcess("%s/servers/openBIS-server/jetty/openbis.pid" % path)
            util.killProcess("%s/datamover.pid" % path)
        util.deleteFolder(self.playgroundFolder)

    def _addToRunningInstances(self, controller):
        self.runningInstances.append(controller)
        
    def _removeFromRunningInstances(self, controller):
        if controller in self.runningInstances:
            self.runningInstances.remove(controller)
    
    def _shutdownSevers(self):
        for instance in reversed(self.runningInstances):
            instance.stop()
            
class _Controller(object):
    def __init__(self, testCase, testName, installPath, instanceName):
        self.testCase = testCase
        self.testName = testName
        self.instanceName = instanceName
        self.installPath = installPath
        util.printAndFlush("Controller created for instance '%s'. Installation path: %s" % (instanceName, installPath))

    def createFolder(self, folderPath):
        """
        Creates a folder with specified path relative to installation directory.
        """
        path = "%s/%s" % (self.installPath, folderPath)
        os.makedirs(path)
    

    def assertEmptyFolder(self, pathRelativeToInstallPath):
        """
        Asserts that the specified path (relative to the installation path) is an empty folder.
        """
        relativePath = "%s/%s" % (self.installPath, pathRelativeToInstallPath)
        files = self._getFiles(relativePath)
        if len(files) == 0:
            util.printAndFlush("Empty folder as expected: %s" % relativePath)
        else:
            self.testCase.fail("%s isn't empty. It contains the following files:\n  %s" % (relativePath, files))
            
    def assertFiles(self, folderPathRelativeToInstallPath, expectedFiles):
        """
        Asserts that the specified path (relative to the installation path) contains the specified files.
        """
        relativePath = "%s/%s" % (self.installPath, folderPathRelativeToInstallPath)
        files = self._getFiles(relativePath)
        self.testCase.assertEquals("Files in %s" % relativePath, expectedFiles, sorted(files))
        
    def _getFiles(self, relativePath):
        if not os.path.isdir(relativePath):
            self.testCase.fail("Doesn't exist or isn't a folder: %s" % relativePath)
        files = os.listdir(relativePath)
        return files

            
class ScriptBasedServerController(_Controller):
    def __init__(self, testCase, testName, installPath, instanceName, startCommand, stopCommand):
        super(ScriptBasedServerController, self).__init__(testCase, testName, installPath, instanceName)
        self.startCommand = startCommand
        self.stopCommand = stopCommand
        
    def start(self):
        self.testCase._addToRunningInstances(self)
        util.executeCommand(self.startCommand, "Couldn't start server '%s'" % self.instanceName, 
                            workingDir = self.installPath)
        
    def stop(self):
        self.testCase._removeFromRunningInstances(self)
        util.executeCommand(self.stopCommand, "Couldn't stop server '%s'" % self.instanceName, 
                            workingDir = self.installPath)
        
            
class DatamoverController(_Controller):
    def __init__(self, testCase, testName, installPath, instanceName):
        super(DatamoverController, self).__init__(testCase, testName, installPath, instanceName)
        self.servicePropertiesFile = "%s/etc/service.properties" % self.installPath
        self.serviceProperties = util.readProperties(self.servicePropertiesFile)
        self.serviceProperties['check-interval'] = 2
        self.serviceProperties['quiet-period'] = 5
        self.serviceProperties['inactivity-period'] = 15
        dataCompletedScript = "%s/%s/data-completed.sh" % (testCase.getTemplatesFolder(), instanceName)
        if os.path.exists(dataCompletedScript):
            self.serviceProperties['data-completed-script'] = "../../../../%s" % dataCompletedScript
        
    def setPrefixForIncoming(self, prefix):
        """ Set service property 'prefix-for-incoming'. """
        self.serviceProperties['prefix-for-incoming'] = prefix
        
    def setTreatIncomingAsRemote(self, flag):
        """ Set service property 'treat-incoming-as-remote'. """
        self.serviceProperties['treat-incoming-as-remote'] = flag
        
    def setOutgoingTarget(self, path):
        """ 
        Set service property 'outgoing-target'. 
        This has to be a path relative to installation path of the datamover. 
        """
        self.serviceProperties['outgoing-target'] = path
        
    def setExtraCopyDir(self, path):
        """ 
        Set service property 'extra-copy-dir'. 
        This has to be a path relative to installation path of the datamover. 
        """
        self.serviceProperties['extra-copy-dir'] = path
        
    def start(self):
        """ Starts up datamover server. """
        util.writeProperties(self.servicePropertiesFile, self.serviceProperties)
        self.testCase._addToRunningInstances(self)
        output = util.executeCommand(["%s/datamover.sh" % self.installPath, 'start'], suppressStdOut=True)
        joinedOutput = '\n'.join(output)
        if 'FAILED' in joinedOutput:
            util.printAndFlush("Start up of datamover %s failed:\n%s" % (self.instanceName, joinedOutput))
            raise Exception("Couldn't start up datamover '%s'." % self.instanceName)
            
    def stop(self):
        """ Stops datamover server. """
        self.testCase._removeFromRunningInstances(self)
        util.executeCommand(["%s/datamover.sh" % self.installPath, 'stop'],
                            "Couldn't shut down datamover '%s'." % self.instanceName)
        
    def drop(self, testDataSetName):
        """ Drops the specified test data set into incoming folder. """
        util.copyFromTo("%s/%s" % (TEST_DATA, self.testName), "%s/data/incoming" % self.installPath, testDataSetName)

            
class ScreeningTestClient(object):
    """
    Class representing the screeing test client.
    """
    def __init__(self, testCase, installPath):
        self.testCase = testCase
        self.installPath = installPath
        
    def run(self):
        """ Runs the test client and returns the console output as a list of strings. """
        output = util.executeCommand(['java', '-Djavax.net.ssl.trustStore=openBIS.keystore',
                                      '-jar', 'openbis_screening_api.jar', 'admin', 'admin', 
                                      'https://localhost:8443'], suppressStdOut = True,
                                      workingDir=self.installPath)
        with open("%s/log.txt" % self.installPath, 'w') as log:
            for line in output:
                log.write("%s\n" % line)
        return output
    
class DataSet(object):
        def __init__(self, resultSetRow):
            self.id = resultSetRow[0]
            self.dataStore = resultSetRow[1]
            self.experimentCode = resultSetRow[2]
            self.code = resultSetRow[3]
            self.type = resultSetRow[4]
            self.location = resultSetRow[5]
            self.status = resultSetRow[6]
            self.presentInArchive = resultSetRow[7]
            self.producer = resultSetRow[8]
            self.productionTimeStamp = resultSetRow[9]
            self.parents = []
            self.children = []
        
        def __str__(self):
            parents = [d.id for d in self.parents]  
            children = [d.id for d in self.children]
            return "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s" % (self.id, self.dataStore, self.code, self.type, 
                                                            self.location, self.status, self.presentInArchive,
                                                            parents, children, self.experimentCode, 
                                                            self.producer, self.productionTimeStamp) 
    
class OpenbisController(_Controller):
    """
    Class to control AS and DSS of an installed openBIS instance.
    """
    def __init__(self, testCase, testName, installPath, instanceName, dropDatabases = True, databasesToDrop = []):
        """
        Creates a new instance for specifies test case with specified test and instance name, installation path.
        """
        super(OpenbisController, self).__init__(testCase, testName, installPath, instanceName)
        self.templatesFolder = testCase.getTemplatesFolder()
        self.binFolder = "%s/bin" % installPath
        self.bisUpScript = "%s/bisup.sh" % self.binFolder
        self.bisDownScript = "%s/bisdown.sh" % self.binFolder
        self.dssUpScript = "%s/dssup.sh" % self.binFolder
        self.dssDownScript = "%s/dssdown.sh" % self.binFolder
        self.databaseKind = "%s_%s" % (testName, instanceName)
        self.asServicePropertiesFile = "%s/servers/openBIS-server/jetty/etc/service.properties" % installPath
        self.asProperties = None
        if os.path.exists(self.asServicePropertiesFile):
            self.asProperties = util.readProperties(self.asServicePropertiesFile)
            self.asProperties['database.kind'] = self.databaseKind
            self.asProperties['proteomics-database-kind'] = self.databaseKind
            self.asPropertiesModified = True
        self.dssServicePropertiesFile = "%s/servers/datastore_server/etc/service.properties" % installPath
        self.dssProperties = util.readProperties(self.dssServicePropertiesFile)
        self.dssProperties['path-info-db.databaseKind'] = self.databaseKind
        self.dssProperties['imaging-database.kind'] = self.databaseKind
        self.dssProperties['proteomics-database-kind'] = self.databaseKind
        self.dssPropertiesModified = True
        if dropDatabases:
            util.dropDatabase(PSQL_EXE, "openbis_%s" % self.databaseKind)
            util.dropDatabase(PSQL_EXE, "pathinfo_%s" % self.databaseKind)
            util.dropDatabase(PSQL_EXE, "imaging_%s" % self.databaseKind)
            util.dropDatabase(PSQL_EXE, "proteomics_%s" % self.databaseKind)
        for databaseToDrop in databasesToDrop:
            util.dropDatabase(PSQL_EXE, "%s_%s" % (databaseToDrop, self.databaseKind))
        self._applyCorePlugins()
        
    def setDummyAuthentication(self):
        """ Disables authentication. """
        self.asProperties['authentication-service'] = 'dummy-authentication-service'
        
    def setDataStoreServerCode(self, code):
        """ Sets the code of the Data Store Server. """
        self.dssProperties['data-store-server-code'] = code
        
    def getDataStoreServerCode(self):
        return self.dssProperties['data-store-server-code']
        
    def setDataStoreServerPort(self, port):
        """ Sets the port of the Data Store Server. """
        self.dssProperties['port'] = port
        
    def setDataStoreServerUsername(self, username):
        """ Sets the username of the Data Store Server. """
        self.dssProperties['username'] = username
        
    def setAsMaxHeapSize(self, maxHeapSize):
        self._setMaxHeapSize("openBIS-server/jetty/etc/openbis.conf", maxHeapSize)
        
    def setDssMaxHeapSize(self, maxHeapSize):
        self._setMaxHeapSize("datastore_server/etc/datastore_server.conf", maxHeapSize)
        
    def assertFileExist(self, pathRelativeToInstallPath):
        """
        Asserts that the specified path (relative to the installation path) exists.
        """
        relativePath = "%s/%s" % (self.installPath, pathRelativeToInstallPath)
        if os.path.exists(relativePath):
            util.printAndFlush("Path exists as expected: %s" % relativePath)
        else:
            self.testCase.fail("Path doesn't exist: %s" % relativePath)
            
    def assertDataSetContent(self, pathToOriginal, dataSet):
        path = "%s/data/store/1/%s/original" % (self.installPath, dataSet.location)
        path = "%s/%s" % (path, os.listdir(path)[0])
        numberOfDifferences = util.getNumberOfDifferences(pathToOriginal, path)
        if numberOfDifferences > 0:
            self.testCase.fail("%s differences found." % numberOfDifferences)
            
    def assertNumberOfDataSets(self, expectedNumberOfDataSets, dataSets):
        """
        Asserts that the specified number of data sets from the specified list of DataSet instances 
        are in the data store. 
        """
        count = 0
        for dataSet in dataSets:
            if dataSet.dataStore != self.getDataStoreServerCode() or dataSet.location == '':
                continue
            count += 1
            self.assertFileExist("data/store/1/%s" % dataSet.location)
        self.testCase.assertEquals("Number of data sets in data store %s" % self.getDataStoreServerCode(), 
                                   expectedNumberOfDataSets, count)
    
    def storeDirectory(self):
        """
        Return the path to the data/store directory
        """
        return "data/store"
    
    def getDataSets(self):
        """
        Returns all data sets as a list (ordered by data set ids) of instances of class DataSet.
        """
        resultSet = self.queryDatabase('openbis', 
                                       "select data.id,ds.code,e.code,data.code,t.code,location,status,present_in_archive,"
                                       + "    data.data_producer_code,data.production_timestamp from data"
                                       + " left join external_data as ed on ed.data_id = data.id" 
                                       + " join data_set_types as t on data.dsty_id = t.id"
                                       + " join experiments as e on data.expe_id = e.id"
                                       + " join data_stores as ds on data.dast_id = ds.id order by data.id")
        dataSets = []
        dataSetsById = {}
        for row in resultSet:
            dataSet = DataSet(row)
            dataSets.append(dataSet)
            dataSetsById[dataSet.id] = dataSet
        relationships = self.queryDatabase('openbis', 
                                           "select data_id_parent, data_id_child from data_set_relationships"
                                           + " order by data_id_parent, data_id_child")
        for parent_id, child_id in relationships:
            parent = dataSetsById[parent_id]
            child = dataSetsById[child_id]
            parent.children.append(child)
            child.parents.append(parent)
        util.printAndFlush("All data sets:\nid,dataStore,code,type,location,status,presentInArchive,parents,children,experiment,producer,productionTimeStamp")
        for dataSet in dataSets:
            util.printAndFlush(dataSet)
        return dataSets 
        
    def createTestDatabase(self, databaseType):
        """
        Creates a test database for the specified database type.
        """
        database = "%s_%s" % (databaseType, self.databaseKind)
        scriptPath = "%s/%s.sql" % (self.templatesFolder, database)
        util.createDatabase(PSQL_EXE, database, scriptPath)
        
    def dropDatabase(self, databaseType):
        """
        Drops the database for the specified database type.
        """
        util.dropDatabase(PSQL_EXE, "%s_%s" % (databaseType, self.databaseKind))
        
    def queryDatabase(self, databaseType, queryStatement, showHeaders = False):
        """
        Executes the specified SQL statement for the specified database type. Result set is returned
        as a list of lists.
        """
        database = "%s_%s" % (databaseType, self.databaseKind)
        return util.queryDatabase(PSQL_EXE, database, queryStatement, showHeaders)
    
    def allUp(self):
        """ Starts up AS and DSS if not running. """
        if not util.isAlive("%s/servers/openBIS-server/jetty/openbis.pid" % self.installPath, "openBIS.keystore"):
            self._saveAsPropertiesIfModified()
            util.executeCommand([self.bisUpScript], "Starting up openBIS AS '%s' failed." % self.instanceName)
        self.dssUp()
        
        
    def stop(self):
        self.allDown()
        
    def allDown(self):
        """ Shuts down AS and DSS. """
        self.testCase._removeFromRunningInstances(self)
        util.executeCommand([self.dssDownScript], "Shutting down openBIS DSS '%s' failed." % self.instanceName)
        if self.asProperties:
            util.executeCommand([self.bisDownScript], "Shutting down openBIS AS '%s' failed." % self.instanceName)
        
    def dssUp(self):
        """ Starts up DSS if not running. """
        if not util.isAlive("%s/servers/datastore_server/datastore_server.pid" % self.installPath, "openBIS.keystore"):
            self._saveDssPropertiesIfModified()
            self.testCase._addToRunningInstances(self)
            util.executeCommand([self.dssUpScript], "Starting up openBIS DSS '%s' failed." % self.instanceName)
        
    def dssDown(self):
        """ Shuts down DSS. """
        self.testCase._removeFromRunningInstances(self)
        util.executeCommand([self.dssDownScript], "Shutting down openBIS DSS '%s' failed." % self.instanceName)
        

    def dropAndWait(self, dataName, dropBoxName, numberOfDataSets = 1, timeOutInMinutes = DEFAULT_TIME_OUT_IN_MINUTES):
        """
        Drops the specified data into the specified drop box. The data is either a folder or a ZIP file
        in TEST_DATA/<test name>. A ZIP file will be unpacked in the drop box. After dropping the method waits
        until the specified number of data sets have been registered.
        """
        self.drop(dataName, dropBoxName)
        self.waitUntilDataSetRegistrationFinished(numberOfDataSets = numberOfDataSets, timeOutInMinutes = timeOutInMinutes)
    
    def dataFile(self, dataName):
        """
        Returns the path to the given test data
        """
        return "%s/%s/%s" % (TEST_DATA, self.testName, dataName)
    

    def drop(self, dataName, dropBoxName):
        """
        Drops the specified test data into the specified drop box. The test data is either a folder or a ZIP file
        in TEST_DATA/<test name>. A ZIP file will be unpacked in the drop box. 
        """
        destination = "%s/data/%s" % (self.installPath, dropBoxName)
        self.dropIntoDestination(dataName, destination)

    def dropIntoDestination(self, dataName, destination):
        """
        Drops the specified test data into the destination. The test data is either a folder or a ZIP file
        in TEST_DATA/<test name>. A ZIP file will be unpacked in the drop box. 
        """
        testDataFolder = "%s/%s" % (TEST_DATA, self.testName)
        if dataName.endswith('.zip'):
            util.unzip("%s/%s" % (testDataFolder, dataName), destination)
        else:
            util.copyFromTo(testDataFolder, destination, dataName)

    def waitUntilDataSetRegistrationFinished(self, numberOfDataSets = 1, timeOutInMinutes = DEFAULT_TIME_OUT_IN_MINUTES):
        """ Waits until the specified number of data sets have been registrated. """
        monitor = self.createLogMonior(timeOutInMinutes)
        monitor.addNotificationCondition(util.RegexCondition('Incoming Data Monitor'))
        monitor.addNotificationCondition(util.RegexCondition('post-registration'))
        numberOfRegisteredDataSets = 0
        while numberOfRegisteredDataSets < numberOfDataSets:
            elements = monitor.waitUntilEvent(util.RegexCondition('Post registration of (\\d*). of \\1 data sets'))
            numberOfRegisteredDataSets += int(elements[0])
            util.printAndFlush("%d of %d data sets registered" % (numberOfRegisteredDataSets, numberOfDataSets))

    def waitUntilDataSetRegistrationFailed(self, timeOutInMinutes = DEFAULT_TIME_OUT_IN_MINUTES):
        """ Waits until data set registration failed. """
        self.waitUntilConditionMatched(util.EventTypeCondition('ERROR'), timeOutInMinutes)
        util.printAndFlush("Data set registration failed as expected.")

    def waitUntilConditionMatched(self, condition, timeOutInMinutes = DEFAULT_TIME_OUT_IN_MINUTES):
        """
        Waits until specified condition has been detected in DSS log.
        """
        monitor = self.createLogMonior(timeOutInMinutes)
        monitor.addNotificationCondition(util.RegexCondition('Incoming Data Monitor'))
        monitor.addNotificationCondition(util.RegexCondition('post-registration'))
        monitor.waitUntilEvent(condition)
        
    def createLogMonior(self, timeOutInMinutes = DEFAULT_TIME_OUT_IN_MINUTES):
        logFilePath = "%s/servers/datastore_server/log/datastore_server_log.txt" % self.installPath
        return util.LogMonitor("%s.DSS" % self.instanceName, logFilePath, timeOutInMinutes)
        
    def assertFeatureVectorLabel(self, featureCode, expectedFeatureLabel):
        data = self.queryDatabase('imaging',  "select distinct label from feature_defs where code = '%s'" % featureCode);
        self.testCase.assertEquals("label of feature %s" % featureCode, [[expectedFeatureLabel]], data)
        
    def _applyCorePlugins(self):
        corePluginsFolder = "%s/servers/core-plugins" % self.installPath
        destination = "%s/%s" % (corePluginsFolder, self.instanceName)
        shutil.rmtree(destination, ignore_errors=True)
        shutil.copytree("%s/core-plugins/%s" % (self.templatesFolder, self.instanceName), destination)
        corePluginsPropertiesFile = "%s/core-plugins.properties" % corePluginsFolder
        corePluginsProperties = util.readProperties(corePluginsPropertiesFile)
        enabledModules = corePluginsProperties['enabled-modules']
        enabledModules = "%s, %s" % (enabledModules, self.instanceName) if len(enabledModules) > 0 else self.instanceName
        corePluginsProperties['enabled-modules'] = enabledModules
        util.writeProperties(corePluginsPropertiesFile, corePluginsProperties)
        
    def _saveAsPropertiesIfModified(self):
        if self.asPropertiesModified:
            util.writeProperties(self.asServicePropertiesFile, self.asProperties)
            self.asPropertiesModified = False
    
    def _saveDssPropertiesIfModified(self):
        if self.dssPropertiesModified:
            util.writeProperties(self.dssServicePropertiesFile, self.dssProperties)
            self.dssPropertiesModified = False
    
    def _setMaxHeapSize(self, configFile, maxHeapSize):
        path = "%s/servers/%s" % (self.installPath, configFile)
        lines = []
        for line in util.getContent(path):
            if line.strip().startswith('JAVA_MEM_OPTS'):
                line = re.sub(r'(.*)-Xmx[^ ]+(.*)', r"\1-Xmx%s\2" % maxHeapSize, line)
            lines.append(line)
        with open(path, "w") as f:
            for line in lines:
                f.write("%s\n" % line)