Skip to content
Snippets Groups Projects
Commit b8ab01f8 authored by gakin's avatar gakin
Browse files

SSDMSSDM-5023: test_openbis_sync add dry run to the test 

SVN: 38146
parent 65f072fb
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
integration-tests/test_openbis_sync.py
+ 111
− 69
View file @ b8ab01f8
......@@ -29,7 +29,6 @@ def sslwrap(func):
ssl.wrap_socket = sslwrap(ssl.wrap_socket)
class TestCase(systemtest.testcase.TestCase):
def execute(self):
openbis1_dss_port = '8444'
openbis2_port = '8445'
......@@ -48,17 +47,16 @@ class TestCase(systemtest.testcase.TestCase):
'''Set openbis1 as the datasource'''
self.installDataSourcePlugin(openbis1, openbis1_dss_port)
self.installEntityRegistrationPlugin(openbis1)
'''Copy master data script'''
'''Copy master data script'''
filePath = "%s/servers/core-plugins/%s/1/as" % (openbis1.installPath, openbis1.instanceName)
os.makedirs(filePath)
util.printAndFlush("Copying master data script from %s to %s" %(self.getMasterDataScriptFolder(), filePath ))
util.printAndFlush("Copying master data script from %s to %s" % (self.getMasterDataScriptFolder(), filePath))
util.copyFromTo(self.getMasterDataScriptFolder(), filePath, "initialize-master-data.py")
corePluginsPropertiesFile = "%s/servers/core-plugins/core-plugins.properties" % openbis1.installPath
util.printAndFlush(corePluginsPropertiesFile)
openbis1.allUp()
'''Drop the folder to register some test entities in space SYNC'''
openbis1.dropAndWait("ENTITY_REGISTRATION", "openbis-sync-entity-reg")
......@@ -74,7 +72,7 @@ class TestCase(systemtest.testcase.TestCase):
openbis2.createTestDatabase('pathinfo')
openbis2.createTestDatabase('imaging')
openbis2.createTestDatabase('proteomics')
'''set openbis2 as harvester'''
self.installHarvesterPlugin(openbis2)
source = self.getHarvesterConfigFolder()
......@@ -90,111 +88,142 @@ class TestCase(systemtest.testcase.TestCase):
'''datasource plugin is installed on harvester side as well in order to get the resource list during testing'''
self.installDataSourcePlugin(openbis2, openbis2_dss_port)
openbis2.allUp()
monitor = util.LogMonitor("%s syncronization.log" % openbis2.instanceName,
"%s/syncronization.log" % openbis2.installPath) # "%s/servers/datastore_server/log/datastore_server_log.txt" % openbis2.installPath
monitor.addNotificationCondition(util.RegexCondition('OPERATION.HarvesterMaintenanceTask'))
monitor.waitUntilEvent(util.RegexCondition('OPERATION.HarvesterMaintenanceTask - Saving the timestamp of sync start to file'))
'''read entity graph from datasource'''
datasource_graph_response = self.getResourceListForComparison('8444', 'harvester1', '123')
file1 = os.path.join(destination, "datasource_graph.txt")
self.writeResponseToFile(datasource_graph_response, file1)
content1 = self.readLinesFromFile(file1)
content1.sort()
'''read entity graph from harvester'''
harvester_graph_response = self.getResourceListForComparison(openbis2_dss_port, 'testuser1', '123')
file2 = os.path.join(destination, "harvester_graph.txt")
self.writeResponseToFile(harvester_graph_response, file2)
content2 = self.removePrefixFromLines(self.readLinesFromFile(file2), data_source_alias)
content2.sort()
source = self.getHarvesterConfigFolder()
harvester_config_file_name= "harvester-config.txt"
harvester_config_file_path = os.path.join(source, harvester_config_file_name)
harvester_config_lines = self.readLinesFromFile(harvester_config_file_path)
data_source_alias = self.extractDataSourceAlias(harvester_config_lines)
util.printAndFlush("Data source alias is: %s" % data_source_alias)
destination = openbis2.installPath
util.printAndFlush("Copying harvester configuration file from %s to %s"% (source, destination))
util.copyFromTo(source, destination, harvester_config_file_name)
'''first do a dry run'''
self.dry_run(openbis2_dss_port, openbis2, harvester_config_file_name, harvester_config_lines)
'''compare the two. If the only difference is in space labels then we are good.'''
diff_array = self.diff(set(content1), set(content2))
same = True
with open(os.path.join(destination, "diff.txt"), 'wb') as output:
for item in diff_array:
output.write("%s\n" % item)
if item.startswith("label") == False:
same = False
self.assertEquals("The entity graphs on datasource and harvester are equal", True, same)
'''then do a real run'''
self.realRun(openbis1, openbis2_dss_port, openbis2, source, harvester_config_file_name, data_source_alias)
if same == False:
self.fail("The entity graphs on datasource and harvester are not equal.See %s for details" % os.path.join(destination, "diff.txt"))
def executeInDevMode(self):
openbis1_dss_port = '8444'
openbis1 = self.createOpenbisController(instanceName = 'openbis1', dropDatabases=False)
openbis2_port = '8445'
openbis2_dss_port = '8446'
openbis1=self.createOpenbisController(instanceName = 'openbis1', dropDatabases=False)
openbis1.setDummyAuthentication()
self.installDataSourcePlugin(openbis1, openbis1_dss_port)
self.installEntityRegistrationPlugin(openbis1)
'''Copy master data script'''
filePath = "%s/servers/core-plugins/%s/1/as" % (openbis1.installPath, openbis1.instanceName)
os.makedirs(filePath)
util.printAndFlush("Copying master data script from %s to %s" %(self.getMasterDataScriptFolder(), filePath ))
util.copyFromTo(self.getMasterDataScriptFolder(), filePath, "initialize-master-data.py")
corePluginsPropertiesFile = "%s/servers/core-plugins/core-plugins.properties" % openbis1.installPath
util.printAndFlush(corePluginsPropertiesFile)
#util.writeProperties(corePluginsPropertiesFile)
openbis1.setDataStoreServerUsername('etlserver1')
openbis1.setDataStoreServerProperty("host-address", "https://localhost")
openbis1.allUp()
'''uncomment the following if we have not run the test once in non-dev mode before (otherwise we already have ENTITY_REGISTRATION and get an error'''
# '''Drop the folder to register some test entities in space SYNC'''
# openbis1.dropAndWait("ENTITY_REGISTRATION", "openbis-sync-entity-reg")
openbis2_port = '8445'
openbis2_dss_port = '8446'
openbis2 = self.createOpenbisController(instanceName = 'openbis2', port = openbis2_port, dropDatabases=False)
openbis2.setDataStoreServerPort(openbis2_dss_port)
openbis2.setOpenbisPortDataStoreServer(openbis2_port)
openbis2.setDataStoreServerProperty("host-address", "https://localhost")
self.installHarvesterPlugin(openbis2)
openbis2.setDummyAuthentication()
'''datasource plugin is installed on harvester side as well just for testing'''
self.installDataSourcePlugin(openbis2, openbis2_dss_port)
openbis2.setDataStoreServerUsername('etlserver2')
#=======================================================================
source = self.getHarvesterConfigFolder()
destination = openbis2.installPath
harvester_config_file_name= "harvester-config.txt"
data_source_alias = self.extractDataSourceAlias(os.path.join(source, harvester_config_file_name))
util.printAndFlush("data source alias is: %s" % data_source_alias)
harvester_config_file_path = os.path.join(source, harvester_config_file_name)
harvester_config_lines = self.readLinesFromFile(harvester_config_file_path)
data_source_alias = self.extractDataSourceAlias(harvester_config_lines)
util.printAndFlush("Data source alias is: %s" % data_source_alias)
util.printAndFlush("Copying harvester configuration file from %s to %s"% (source, destination))
util.copyFromTo(source, destination, harvester_config_file_name)
openbis2.allUp()
monitor = util.LogMonitor("%s syncronization.log" % openbis2.instanceName,
"%s/syncronization.log" % openbis2.installPath) # "%s/servers/datastore_server/log/datastore_server_log.txt" % openbis2.installPath
monitor.addNotificationCondition(util.RegexCondition('OPERATION.HarvesterMaintenanceTask'))
monitor.waitUntilEvent(util.RegexCondition('OPERATION.HarvesterMaintenanceTask - Saving the timestamp of sync start to file'))
'''first do a dry run'''
output = self.dry_run(openbis2_dss_port, openbis2, harvester_config_file_name, harvester_config_lines)
'''then do a real run'''
self.realRun(openbis1, openbis2_dss_port, openbis2, source, harvester_config_file_name, data_source_alias)
def getEntityGraph(self, openbis2_dss_port, openbis_instance, user, password):
harvester_graph_response = self.getResourceListForComparison(openbis2_dss_port, user, password)
file = os.path.join(openbis_instance.installPath, "%s_graph.txt" % openbis_instance.instanceName)
self.writeResponseToFile(harvester_graph_response, file)
graph_lines = self.readLinesFromFile(file)
return graph_lines
def setDryRunToTrue(self, harvester_config_file_path, harvester_config_lines):
with open(harvester_config_file_path, 'wb') as output:
for line in harvester_config_lines:
if line.startswith("dry-run") == True:
output.write("dry-run = true")
else:
output.write(line)
def dry_run(self, openbis2_dss_port, openbis2, harvester_config_file_name, harvester_config_lines):
util.printAndFlush("Setting dry-run = true in modifying harvester-config.txt file")
self.setDryRunToTrue(os.path.join(openbis2.installPath, harvester_config_file_name), harvester_config_lines)
'''disable harvester maintenance task'''
harvester_plugin_folder = os.path.join(openbis2.installPath, 'servers/core-plugins/openbis2/1/dss/maintenance-tasks/harvester')
util.printAndFlush("Disabling harvester plugin until the initial entity graph is read")
open(os.path.join(harvester_plugin_folder, 'disabled'), 'a')
openbis2.allUp()
'''read entity graph from harvester before starting dry run by removing disabled file from plugin folder'''
harvester_graph_lines_before_dry_run = self.getEntityGraph(openbis2_dss_port, openbis2, 'testuser1', '123')
openbis2.dssDown()
util.printAndFlush("Enabling harvester plugin for the dry run")
os.remove(os.path.join(harvester_plugin_folder, 'disabled'))
openbis2.dssUp()
monitor = util.LogMonitor("%s syncronization.log" % openbis2.instanceName,
"%s/syncronization.log" % openbis2.installPath) # "%s/servers/datastore_server/log/datastore_server_log.txt" % openbis2.installPath
monitor.addNotificationCondition(util.RegexCondition('OPERATION.HarvesterMaintenanceTask'))
monitor.waitUntilEvent(util.RegexCondition('OPERATION.HarvesterMaintenanceTask - Dry run finished'))
'''read entity graph from harvester after dry run finished'''
harvester_graph_lines_after_dry_run = self.getEntityGraph(openbis2_dss_port, openbis2, 'testuser1', '123')
harvester_graph_lines_before_dry_run.sort()
harvester_graph_lines_after_dry_run.sort()
diff_for_dry_run = self.diff(set(harvester_graph_lines_before_dry_run), set(harvester_graph_lines_after_dry_run))
self.assertEquals("The entity graph on harvester stays the same after a dry run", True, len(diff_for_dry_run) == 0)
def realRun(self, openbis1, openbis2_dss_port, openbis2, source, harvester_config_file_name, data_source_alias):
destination = openbis2.installPath
openbis2.dssDown()
util.printAndFlush("Copying harvester configuration file from %s to %s" % (source, destination))
util.copyFromTo(source, destination, harvester_config_file_name)
openbis2.dssUp()
'''read entity graph from datasource'''
datasource_graph_response = self.getResourceListForComparison('8444', 'harvester1', '123')
file1 = os.path.join(destination, "datasource_graph.txt")
self.writeResponseToFile(datasource_graph_response, file1)
content1 = self.readLinesFromFile(file1)
content1 = self.getEntityGraph('8444', openbis1, 'harvester1', '123')
content1.sort()
# for line in content1:
# util.printAndFlush(line)
'''read entity graph from harvester
the entities might be translated using a prefix specified in the harvester_config
remove the prefix'''
harvester_graph_response = self.getResourceListForComparison(openbis2_dss_port, 'testuser1', '123')
file2 = os.path.join(destination, "harvester_graph.txt")
self.writeResponseToFile(harvester_graph_response, file2)
content2 = self.removePrefixFromLines(self.readLinesFromFile(file2), data_source_alias)
monitor = util.LogMonitor("%s syncronization.log" % openbis2.instanceName,
"%s/syncronization.log" % openbis2.installPath) # "%s/servers/datastore_server/log/datastore_server_log.txt" % openbis2.installPath
monitor.addNotificationCondition(util.RegexCondition('OPERATION.HarvesterMaintenanceTask'))
monitor.waitUntilEvent(util.RegexCondition('OPERATION.HarvesterMaintenanceTask - Saving the timestamp of sync start to file'))
graph_lines = self.getEntityGraph(openbis2_dss_port, openbis2, 'testuser1', '123')
content2 = self.removePrefixFromLines(graph_lines, data_source_alias)
content2.sort()
'''compare the two. If the only difference is in space labels then we are good.'''
diff_array = self.diff(set(content1), set(content2))
same = True
......@@ -203,14 +232,27 @@ class TestCase(systemtest.testcase.TestCase):
output.write("%s\n" % item)
if item.startswith("label") == False:
same = False
self.assertEquals("The entity graphs on datasource and harvester are equal", True, same)
if same == False:
self.fail("The entity graphs on datasource and harvester are not equal.See %s for details" % os.path.join(destination, "diff.txt"))
def extractDryRunProperty(self, lines):
for line in lines:
match = re.search("dry-run=\s(.*)", line)
if match:
val = match.group(1)
print "val:" + val
if(val.lower() == "true") :
return True
elif (val.lower() == "false"):
return False
else:
self.fail("Dry-run value can either be true or false")
return False
def extractDataSourceAlias(self, harvester_config_file):
lines = self.readLinesFromFile(harvester_config_file)
def extractDataSourceAlias(self, lines):
for line in lines:
match = re.search("data-source-alias =\s(.*)", line)
if match:
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment