Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
O
openbis
Manage
Activity
Members
Labels
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Model registry
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
sispub
openbis
Commits
b8ab01f8
Commit
b8ab01f8
authored
7 years ago
by
gakin
Browse files
Options
Downloads
Patches
Plain Diff
SSDMSSDM-5023: test_openbis_sync add dry run to the test
SVN: 38146
parent
65f072fb
No related branches found
Branches containing commit
No related tags found
Tags containing commit
No related merge requests found
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
integration-tests/test_openbis_sync.py
+111
-69
111 additions, 69 deletions
integration-tests/test_openbis_sync.py
with
111 additions
and
69 deletions
integration-tests/test_openbis_sync.py
+
111
−
69
View file @
b8ab01f8
...
...
@@ -29,7 +29,6 @@ def sslwrap(func):
ssl
.
wrap_socket
=
sslwrap
(
ssl
.
wrap_socket
)
class
TestCase
(
systemtest
.
testcase
.
TestCase
):
def
execute
(
self
):
openbis1_dss_port
=
'
8444
'
openbis2_port
=
'
8445
'
...
...
@@ -48,17 +47,16 @@ class TestCase(systemtest.testcase.TestCase):
'''
Set openbis1 as the datasource
'''
self
.
installDataSourcePlugin
(
openbis1
,
openbis1_dss_port
)
self
.
installEntityRegistrationPlugin
(
openbis1
)
'''
Copy master data script
'''
'''
Copy master data script
'''
filePath
=
"
%s/servers/core-plugins/%s/1/as
"
%
(
openbis1
.
installPath
,
openbis1
.
instanceName
)
os
.
makedirs
(
filePath
)
util
.
printAndFlush
(
"
Copying master data script from %s to %s
"
%
(
self
.
getMasterDataScriptFolder
(),
filePath
))
util
.
printAndFlush
(
"
Copying master data script from %s to %s
"
%
(
self
.
getMasterDataScriptFolder
(),
filePath
))
util
.
copyFromTo
(
self
.
getMasterDataScriptFolder
(),
filePath
,
"
initialize-master-data.py
"
)
corePluginsPropertiesFile
=
"
%s/servers/core-plugins/core-plugins.properties
"
%
openbis1
.
installPath
util
.
printAndFlush
(
corePluginsPropertiesFile
)
openbis1
.
allUp
()
'''
Drop the folder to register some test entities in space SYNC
'''
openbis1
.
dropAndWait
(
"
ENTITY_REGISTRATION
"
,
"
openbis-sync-entity-reg
"
)
...
...
@@ -74,7 +72,7 @@ class TestCase(systemtest.testcase.TestCase):
openbis2
.
createTestDatabase
(
'
pathinfo
'
)
openbis2
.
createTestDatabase
(
'
imaging
'
)
openbis2
.
createTestDatabase
(
'
proteomics
'
)
'''
set openbis2 as harvester
'''
self
.
installHarvesterPlugin
(
openbis2
)
source
=
self
.
getHarvesterConfigFolder
()
...
...
@@ -90,111 +88,142 @@ class TestCase(systemtest.testcase.TestCase):
'''
datasource plugin is installed on harvester side as well in order to get the resource list during testing
'''
self
.
installDataSourcePlugin
(
openbis2
,
openbis2_dss_port
)
openbis2
.
allUp
()
monitor
=
util
.
LogMonitor
(
"
%s syncronization.log
"
%
openbis2
.
instanceName
,
"
%s/syncronization.log
"
%
openbis2
.
installPath
)
# "%s/servers/datastore_server/log/datastore_server_log.txt" % openbis2.installPath
monitor
.
addNotificationCondition
(
util
.
RegexCondition
(
'
OPERATION.HarvesterMaintenanceTask
'
))
monitor
.
waitUntilEvent
(
util
.
RegexCondition
(
'
OPERATION.HarvesterMaintenanceTask - Saving the timestamp of sync start to file
'
))
'''
read entity graph from datasource
'''
datasource_graph_response
=
self
.
getResourceListForComparison
(
'
8444
'
,
'
harvester1
'
,
'
123
'
)
file1
=
os
.
path
.
join
(
destination
,
"
datasource_graph.txt
"
)
self
.
writeResponseToFile
(
datasource_graph_response
,
file1
)
content1
=
self
.
readLinesFromFile
(
file1
)
content1
.
sort
()
'''
read entity graph from harvester
'''
harvester_graph_response
=
self
.
getResourceListForComparison
(
openbis2_dss_port
,
'
testuser1
'
,
'
123
'
)
file2
=
os
.
path
.
join
(
destination
,
"
harvester_graph.txt
"
)
self
.
writeResponseToFile
(
harvester_graph_response
,
file2
)
content2
=
self
.
removePrefixFromLines
(
self
.
readLinesFromFile
(
file2
),
data_source_alias
)
content2
.
sort
()
source
=
self
.
getHarvesterConfigFolder
()
harvester_config_file_name
=
"
harvester-config.txt
"
harvester_config_file_path
=
os
.
path
.
join
(
source
,
harvester_config_file_name
)
harvester_config_lines
=
self
.
readLinesFromFile
(
harvester_config_file_path
)
data_source_alias
=
self
.
extractDataSourceAlias
(
harvester_config_lines
)
util
.
printAndFlush
(
"
Data source alias is: %s
"
%
data_source_alias
)
destination
=
openbis2
.
installPath
util
.
printAndFlush
(
"
Copying harvester configuration file from %s to %s
"
%
(
source
,
destination
))
util
.
copyFromTo
(
source
,
destination
,
harvester_config_file_name
)
'''
first do a dry run
'''
self
.
dry_run
(
openbis2_dss_port
,
openbis2
,
harvester_config_file_name
,
harvester_config_lines
)
'''
compare the two. If the only difference is in space labels then we are good.
'''
diff_array
=
self
.
diff
(
set
(
content1
),
set
(
content2
))
same
=
True
with
open
(
os
.
path
.
join
(
destination
,
"
diff.txt
"
),
'
wb
'
)
as
output
:
for
item
in
diff_array
:
output
.
write
(
"
%s
\n
"
%
item
)
if
item
.
startswith
(
"
label
"
)
==
False
:
same
=
False
self
.
assertEquals
(
"
The entity graphs on datasource and harvester are equal
"
,
True
,
same
)
'''
then do a real run
'''
self
.
realRun
(
openbis1
,
openbis2_dss_port
,
openbis2
,
source
,
harvester_config_file_name
,
data_source_alias
)
if
same
==
False
:
self
.
fail
(
"
The entity graphs on datasource and harvester are not equal.See %s for details
"
%
os
.
path
.
join
(
destination
,
"
diff.txt
"
))
def
executeInDevMode
(
self
):
openbis1_dss_port
=
'
8444
'
openbis1
=
self
.
createOpenbisController
(
instanceName
=
'
openbis1
'
,
dropDatabases
=
False
)
openbis2_port
=
'
8445
'
openbis2_dss_port
=
'
8446
'
openbis1
=
self
.
createOpenbisController
(
instanceName
=
'
openbis1
'
,
dropDatabases
=
False
)
openbis1
.
setDummyAuthentication
()
self
.
installDataSourcePlugin
(
openbis1
,
openbis1_dss_port
)
self
.
installEntityRegistrationPlugin
(
openbis1
)
'''
Copy master data script
'''
filePath
=
"
%s/servers/core-plugins/%s/1/as
"
%
(
openbis1
.
installPath
,
openbis1
.
instanceName
)
os
.
makedirs
(
filePath
)
util
.
printAndFlush
(
"
Copying master data script from %s to %s
"
%
(
self
.
getMasterDataScriptFolder
(),
filePath
))
util
.
copyFromTo
(
self
.
getMasterDataScriptFolder
(),
filePath
,
"
initialize-master-data.py
"
)
corePluginsPropertiesFile
=
"
%s/servers/core-plugins/core-plugins.properties
"
%
openbis1
.
installPath
util
.
printAndFlush
(
corePluginsPropertiesFile
)
#util.writeProperties(corePluginsPropertiesFile)
openbis1
.
setDataStoreServerUsername
(
'
etlserver1
'
)
openbis1
.
setDataStoreServerProperty
(
"
host-address
"
,
"
https://localhost
"
)
openbis1
.
allUp
()
'''
uncomment the following if we have not run the test once in non-dev mode before (otherwise we already have ENTITY_REGISTRATION and get an error
'''
# '''Drop the folder to register some test entities in space SYNC'''
# openbis1.dropAndWait("ENTITY_REGISTRATION", "openbis-sync-entity-reg")
openbis2_port
=
'
8445
'
openbis2_dss_port
=
'
8446
'
openbis2
=
self
.
createOpenbisController
(
instanceName
=
'
openbis2
'
,
port
=
openbis2_port
,
dropDatabases
=
False
)
openbis2
.
setDataStoreServerPort
(
openbis2_dss_port
)
openbis2
.
setOpenbisPortDataStoreServer
(
openbis2_port
)
openbis2
.
setDataStoreServerProperty
(
"
host-address
"
,
"
https://localhost
"
)
self
.
installHarvesterPlugin
(
openbis2
)
openbis2
.
setDummyAuthentication
()
'''
datasource plugin is installed on harvester side as well just for testing
'''
self
.
installDataSourcePlugin
(
openbis2
,
openbis2_dss_port
)
openbis2
.
setDataStoreServerUsername
(
'
etlserver2
'
)
#=======================================================================
source
=
self
.
getHarvesterConfigFolder
()
destination
=
openbis2
.
installPath
harvester_config_file_name
=
"
harvester-config.txt
"
data_source_alias
=
self
.
extractDataSourceAlias
(
os
.
path
.
join
(
source
,
harvester_config_file_name
))
util
.
printAndFlush
(
"
data source alias is: %s
"
%
data_source_alias
)
harvester_config_file_path
=
os
.
path
.
join
(
source
,
harvester_config_file_name
)
harvester_config_lines
=
self
.
readLinesFromFile
(
harvester_config_file_path
)
data_source_alias
=
self
.
extractDataSourceAlias
(
harvester_config_lines
)
util
.
printAndFlush
(
"
Data source alias is: %s
"
%
data_source_alias
)
util
.
printAndFlush
(
"
Copying harvester configuration file from %s to %s
"
%
(
source
,
destination
))
util
.
copyFromTo
(
source
,
destination
,
harvester_config_file_name
)
openbis2
.
allUp
()
monitor
=
util
.
LogMonitor
(
"
%s syncronization.log
"
%
openbis2
.
instanceName
,
"
%s/syncronization.log
"
%
openbis2
.
installPath
)
# "%s/servers/datastore_server/log/datastore_server_log.txt" % openbis2.installPath
monitor
.
addNotificationCondition
(
util
.
RegexCondition
(
'
OPERATION.HarvesterMaintenanceTask
'
))
monitor
.
waitUntilEvent
(
util
.
RegexCondition
(
'
OPERATION.HarvesterMaintenanceTask - Saving the timestamp of sync start to file
'
))
'''
first do a dry run
'''
output
=
self
.
dry_run
(
openbis2_dss_port
,
openbis2
,
harvester_config_file_name
,
harvester_config_lines
)
'''
then do a real run
'''
self
.
realRun
(
openbis1
,
openbis2_dss_port
,
openbis2
,
source
,
harvester_config_file_name
,
data_source_alias
)
def
getEntityGraph
(
self
,
openbis2_dss_port
,
openbis_instance
,
user
,
password
):
harvester_graph_response
=
self
.
getResourceListForComparison
(
openbis2_dss_port
,
user
,
password
)
file
=
os
.
path
.
join
(
openbis_instance
.
installPath
,
"
%s_graph.txt
"
%
openbis_instance
.
instanceName
)
self
.
writeResponseToFile
(
harvester_graph_response
,
file
)
graph_lines
=
self
.
readLinesFromFile
(
file
)
return
graph_lines
def
setDryRunToTrue
(
self
,
harvester_config_file_path
,
harvester_config_lines
):
with
open
(
harvester_config_file_path
,
'
wb
'
)
as
output
:
for
line
in
harvester_config_lines
:
if
line
.
startswith
(
"
dry-run
"
)
==
True
:
output
.
write
(
"
dry-run = true
"
)
else
:
output
.
write
(
line
)
def
dry_run
(
self
,
openbis2_dss_port
,
openbis2
,
harvester_config_file_name
,
harvester_config_lines
):
util
.
printAndFlush
(
"
Setting dry-run = true in modifying harvester-config.txt file
"
)
self
.
setDryRunToTrue
(
os
.
path
.
join
(
openbis2
.
installPath
,
harvester_config_file_name
),
harvester_config_lines
)
'''
disable harvester maintenance task
'''
harvester_plugin_folder
=
os
.
path
.
join
(
openbis2
.
installPath
,
'
servers/core-plugins/openbis2/1/dss/maintenance-tasks/harvester
'
)
util
.
printAndFlush
(
"
Disabling harvester plugin until the initial entity graph is read
"
)
open
(
os
.
path
.
join
(
harvester_plugin_folder
,
'
disabled
'
),
'
a
'
)
openbis2
.
allUp
()
'''
read entity graph from harvester before starting dry run by removing disabled file from plugin folder
'''
harvester_graph_lines_before_dry_run
=
self
.
getEntityGraph
(
openbis2_dss_port
,
openbis2
,
'
testuser1
'
,
'
123
'
)
openbis2
.
dssDown
()
util
.
printAndFlush
(
"
Enabling harvester plugin for the dry run
"
)
os
.
remove
(
os
.
path
.
join
(
harvester_plugin_folder
,
'
disabled
'
))
openbis2
.
dssUp
()
monitor
=
util
.
LogMonitor
(
"
%s syncronization.log
"
%
openbis2
.
instanceName
,
"
%s/syncronization.log
"
%
openbis2
.
installPath
)
# "%s/servers/datastore_server/log/datastore_server_log.txt" % openbis2.installPath
monitor
.
addNotificationCondition
(
util
.
RegexCondition
(
'
OPERATION.HarvesterMaintenanceTask
'
))
monitor
.
waitUntilEvent
(
util
.
RegexCondition
(
'
OPERATION.HarvesterMaintenanceTask - Dry run finished
'
))
'''
read entity graph from harvester after dry run finished
'''
harvester_graph_lines_after_dry_run
=
self
.
getEntityGraph
(
openbis2_dss_port
,
openbis2
,
'
testuser1
'
,
'
123
'
)
harvester_graph_lines_before_dry_run
.
sort
()
harvester_graph_lines_after_dry_run
.
sort
()
diff_for_dry_run
=
self
.
diff
(
set
(
harvester_graph_lines_before_dry_run
),
set
(
harvester_graph_lines_after_dry_run
))
self
.
assertEquals
(
"
The entity graph on harvester stays the same after a dry run
"
,
True
,
len
(
diff_for_dry_run
)
==
0
)
def
realRun
(
self
,
openbis1
,
openbis2_dss_port
,
openbis2
,
source
,
harvester_config_file_name
,
data_source_alias
):
destination
=
openbis2
.
installPath
openbis2
.
dssDown
()
util
.
printAndFlush
(
"
Copying harvester configuration file from %s to %s
"
%
(
source
,
destination
))
util
.
copyFromTo
(
source
,
destination
,
harvester_config_file_name
)
openbis2
.
dssUp
()
'''
read entity graph from datasource
'''
datasource_graph_response
=
self
.
getResourceListForComparison
(
'
8444
'
,
'
harvester1
'
,
'
123
'
)
file1
=
os
.
path
.
join
(
destination
,
"
datasource_graph.txt
"
)
self
.
writeResponseToFile
(
datasource_graph_response
,
file1
)
content1
=
self
.
readLinesFromFile
(
file1
)
content1
=
self
.
getEntityGraph
(
'
8444
'
,
openbis1
,
'
harvester1
'
,
'
123
'
)
content1
.
sort
()
# for line in content1:
# util.printAndFlush(line)
'''
read entity graph from harvester
the entities might be translated using a prefix specified in the harvester_config
remove the prefix
'''
harvester_graph_response
=
self
.
getResourceListForComparison
(
openbis2_dss_port
,
'
testuser1
'
,
'
123
'
)
file2
=
os
.
path
.
join
(
destination
,
"
harvester_graph.txt
"
)
self
.
writeResponseToFile
(
harvester_graph_response
,
file2
)
content2
=
self
.
removePrefixFromLines
(
self
.
readLinesFromFile
(
file2
),
data_source_alias
)
monitor
=
util
.
LogMonitor
(
"
%s syncronization.log
"
%
openbis2
.
instanceName
,
"
%s/syncronization.log
"
%
openbis2
.
installPath
)
# "%s/servers/datastore_server/log/datastore_server_log.txt" % openbis2.installPath
monitor
.
addNotificationCondition
(
util
.
RegexCondition
(
'
OPERATION.HarvesterMaintenanceTask
'
))
monitor
.
waitUntilEvent
(
util
.
RegexCondition
(
'
OPERATION.HarvesterMaintenanceTask - Saving the timestamp of sync start to file
'
))
graph_lines
=
self
.
getEntityGraph
(
openbis2_dss_port
,
openbis2
,
'
testuser1
'
,
'
123
'
)
content2
=
self
.
removePrefixFromLines
(
graph_lines
,
data_source_alias
)
content2
.
sort
()
'''
compare the two. If the only difference is in space labels then we are good.
'''
diff_array
=
self
.
diff
(
set
(
content1
),
set
(
content2
))
same
=
True
...
...
@@ -203,14 +232,27 @@ class TestCase(systemtest.testcase.TestCase):
output
.
write
(
"
%s
\n
"
%
item
)
if
item
.
startswith
(
"
label
"
)
==
False
:
same
=
False
self
.
assertEquals
(
"
The entity graphs on datasource and harvester are equal
"
,
True
,
same
)
if
same
==
False
:
self
.
fail
(
"
The entity graphs on datasource and harvester are not equal.See %s for details
"
%
os
.
path
.
join
(
destination
,
"
diff.txt
"
))
def
extractDryRunProperty
(
self
,
lines
):
for
line
in
lines
:
match
=
re
.
search
(
"
dry-run=\s(.*)
"
,
line
)
if
match
:
val
=
match
.
group
(
1
)
print
"
val:
"
+
val
if
(
val
.
lower
()
==
"
true
"
)
:
return
True
elif
(
val
.
lower
()
==
"
false
"
):
return
False
else
:
self
.
fail
(
"
Dry-run value can either be true or false
"
)
return
False
def
extractDataSourceAlias
(
self
,
harvester_config_file
):
lines
=
self
.
readLinesFromFile
(
harvester_config_file
)
def
extractDataSourceAlias
(
self
,
lines
):
for
line
in
lines
:
match
=
re
.
search
(
"
data-source-alias =\s(.*)
"
,
line
)
if
match
:
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment