Add JUnit XML reports; refactor test reporting
* Add Junit output file support
* Differentiate between an expected failure test and an error in the
test, as described in JUnit.
* In case of an error/exception during test, record and attach it to the
Test object and continue running the tests, and show it at the end
during the trial report.
Change-Id: Iedf6d912b3cce3333a187a4ac6d5c6b70fe9d5c5
diff --git a/selftest/suite_test.ok b/selftest/suite_test.ok
index fda77dc..30c6915 100644
--- a/selftest/suite_test.ok
+++ b/selftest/suite_test.ok
@@ -59,15 +59,30 @@
tst hello_world.py:[LINENR]: two [test_suite↪hello_world.py:[LINENR]]
tst hello_world.py:[LINENR]: three [test_suite↪hello_world.py:[LINENR]]
tst hello_world.py:[LINENR] PASS [test_suite↪hello_world.py]
-pass: all 1 tests passed.
+tst test_suite: PASS
+pass: all 6 tests passed (5 skipped).
- a test with an error
tst test_suite: Suite run start [suite.py:[LINENR]]
tst test_error.py:[LINENR] START [test_suite↪test_error.py] [suite.py:[LINENR]]
tst test_error.py:[LINENR]: I am 'test_suite' / 'test_error.py:[LINENR]' [test_suite↪test_error.py:[LINENR]] [test_error.py:[LINENR]]
-tst test_error.py:[LINENR]: FAIL [test_suite↪test_error.py:[LINENR]] [suite.py:[LINENR]]
-tst test_error.py:[LINENR]: ERR: AssertionError: [test_suite↪test_error.py:[LINENR]] [test_error.py:[LINENR]: assert False]
-FAIL: 1 of 1 tests failed:
- test_error.py
+tst test_error.py:[LINENR]: ERR: AssertionError: [test_error.py:[LINENR]: assert False]
+tst test_error.py:[LINENR] FAIL (AssertionError) [test_suite↪test_error.py] [suite.py:[LINENR]]
+tst test_suite: FAIL [suite.py:[LINENR]]
+
+- a test with a failure
+tst test_suite: Suite run start [suite.py:[LINENR]]
+tst test_fail.py:[LINENR] START [test_suite↪test_fail.py] [suite.py:[LINENR]]
+tst test_fail.py:[LINENR]: I am 'test_suite' / 'test_fail.py:[LINENR]' [test_suite↪test_fail.py:[LINENR]] [test_fail.py:[LINENR]]
+tst test_fail.py:[LINENR] FAIL (EpicFail) [test_suite↪test_fail.py] [suite.py:[LINENR]]
+tst test_suite: FAIL [suite.py:[LINENR]]
+
+- a test with a raised failure
+tst test_suite: Suite run start [suite.py:[LINENR]]
+tst test_fail_raise.py:[LINENR] START [test_suite↪test_fail_raise.py] [suite.py:[LINENR]]
+tst test_fail_raise.py:[LINENR]: I am 'test_suite' / 'test_fail_raise.py:[LINENR]' [test_suite↪test_fail_raise.py:[LINENR]] [test_fail_raise.py:[LINENR]]
+tst test_fail_raise.py:[LINENR]: ERR: Failure: ('EpicFail', 'This failure is expected') [test_fail_raise.py:[LINENR]: raise Failure('EpicFail', 'This failure is expected')]
+tst test_fail_raise.py:[LINENR] FAIL (EpicFail) [test_suite↪test_fail_raise.py] [suite.py:[LINENR]]
+tst test_suite: FAIL [suite.py:[LINENR]]
- graceful exit.
diff --git a/selftest/suite_test.py b/selftest/suite_test.py
index 315c683..16342c5 100755
--- a/selftest/suite_test.py
+++ b/selftest/suite_test.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python3
import os
import _prep
-from osmo_gsm_tester import log, suite, config
+from osmo_gsm_tester import log, suite, config, report
config.ENV_CONF = './suite_test'
@@ -22,13 +22,33 @@
print('- run hello world test')
s = suite.SuiteRun(None, 'test_suite', s_def)
results = s.run_tests('hello_world.py')
-print(str(results))
+print(report.suite_to_text(s))
log.style_change(src=True)
#log.style_change(trace=True)
print('\n- a test with an error')
results = s.run_tests('test_error.py')
-print(str(results))
+output = report.suite_to_text(s)
+assert 'FAIL: [test_suite] 1 failed ' in output
+assert 'FAIL: [test_error.py]' in output
+assert "type:'AssertionError' message: AssertionError()" in output
+assert 'assert False' in output
+
+print('\n- a test with a failure')
+results = s.run_tests('test_fail.py')
+output = report.suite_to_text(s)
+assert 'FAIL: [test_suite] 1 failed ' in output
+assert 'FAIL: [test_fail.py]' in output
+assert "type:'EpicFail' message: This failure is expected" in output
+assert "test.set_fail('EpicFail', 'This failure is expected')" in output
+
+print('\n- a test with a raised failure')
+results = s.run_tests('test_fail_raise.py')
+output = report.suite_to_text(s)
+assert 'FAIL: [test_suite] 1 failed ' in output
+assert 'FAIL: [test_fail_raise.py]' in output
+assert "type:'EpicFail' message: This failure is expected" in output
+assert "raise Failure('EpicFail', 'This failure is expected')" in output
print('\n- graceful exit.')
# vim: expandtab tabstop=4 shiftwidth=4
diff --git a/selftest/suite_test/test_suite/test_fail.py b/selftest/suite_test/test_suite/test_fail.py
new file mode 100755
index 0000000..6880c81
--- /dev/null
+++ b/selftest/suite_test/test_suite/test_fail.py
@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+from osmo_gsm_tester.test import *
+
+print('I am %r / %r' % (suite.name(), test.name()))
+
+test.set_fail('EpicFail', 'This failure is expected')
diff --git a/selftest/suite_test/test_suite/test_fail_raise.py b/selftest/suite_test/test_suite/test_fail_raise.py
new file mode 100755
index 0000000..a7b0b61
--- /dev/null
+++ b/selftest/suite_test/test_suite/test_fail_raise.py
@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+from osmo_gsm_tester.test import *
+
+print('I am %r / %r' % (suite.name(), test.name()))
+
+raise Failure('EpicFail', 'This failure is expected')
diff --git a/selftest/trial_test.ok b/selftest/trial_test.ok
index 6ad39a9..7434a10 100644
--- a/selftest/trial_test.ok
+++ b/selftest/trial_test.ok
@@ -4,7 +4,7 @@
[TMP]/third
- fetch trial dirs in order
first
-['taken']
+['last_run', 'run.[TIMESTAMP]', 'taken']
second
third
- no more trial dirs left