add test.report_fragment()
Allow enriching the junit output with arbitrary subtasks within a test.
The current aim is, for handover tests, to not just show that a test
failed, but to show exactly which steps worked and which didn't, e.g.:
handover.py/01_bts0_started PASSED
handover.py/02.1_ms0_attach PASSED
handover.py/02.2_ms1_attach PASSED
handover.py/02.3_subscribed_in_msc PASSED
handover.py/03_call_established PASSED
handover.py/04.1_bts1_started FAILED
In this case it is immediately obvious from looking at the jenkins
results analyzer that bts1 is the cause of the test failure, and it is
visible which parts of the test are flaky, over time.
First user Will be the upcoming handover_2G suite, in
I0b2671304165a1aaae2b386af46fbd8b098e3bd8.
Change-Id: I4ca9100b6f8db24d1f7e0a09b3b7ba88b8ae3b59
diff --git a/src/osmo_gsm_tester/core/report.py b/src/osmo_gsm_tester/core/report.py
index c3390fe..c5e185f 100644
--- a/src/osmo_gsm_tester/core/report.py
+++ b/src/osmo_gsm_tester/core/report.py
@@ -132,14 +132,37 @@
testsuite.set('time', str(math.ceil(suite.duration)))
testsuite.set('tests', str(len(suite.tests)))
passed, skipped, failed, errors = suite.count_test_results()
- testsuite.set('errors', str(errors))
- testsuite.set('failures', str(failed))
- testsuite.set('skipped', str(skipped))
- testsuite.set('disabled', str(skipped))
for suite_test in suite.tests:
testcase = test_to_junit(suite_test)
testcase.set('classname', suite.name())
testsuite.append(testcase)
+
+ for report_fragment in suite_test.report_fragments:
+ full_name = '%s/%s' % (suite_test.name(), report_fragment.name)
+ el = et.Element('testcase')
+ el.set('name', full_name)
+ el.set('time', str(math.ceil(report_fragment.duration)))
+ if report_fragment.result == test.Test.SKIP:
+ et.SubElement(el, 'skipped')
+ skipped += 1
+ elif report_fragment.result == test.Test.FAIL:
+ failure = et.SubElement(el, 'failure')
+ failure.set('type', suite_test.fail_type or 'failure')
+ failed += 1
+ elif report_fragment.result != test.Test.PASS:
+ error = et.SubElement(el, 'error')
+ error.text = 'could not run'
+ errors += 1
+
+ if report_fragment.output:
+ sout = et.SubElement(el, 'system-out')
+ sout.text = escape_xml_invalid_characters(report_fragment.output)
+ testsuite.append(el)
+
+ testsuite.set('errors', str(errors))
+ testsuite.set('failures', str(failed))
+ testsuite.set('skipped', str(skipped))
+ testsuite.set('disabled', str(skipped))
return testsuite
def test_to_junit(t):