Improve junit xml generated fields
* trial ('testsuites' node): Add fields 'tests', 'errors', 'failures',
'time'.
* testsuite: Add fields 'errors', 'failures', 'skipped',
'disabled'
* test: Add field 'classname' (empty).
Fix and improve suite.py and test.py to count errors, skipped, failures
properly.
Change-Id: Ie2d10cee88a9c0d829e4620553164cf3150e8e5c
diff --git a/src/osmo_gsm_tester/report.py b/src/osmo_gsm_tester/report.py
index 5d23c38..f781695 100644
--- a/src/osmo_gsm_tester/report.py
+++ b/src/osmo_gsm_tester/report.py
@@ -17,6 +17,8 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
+# junit xml format: https://llg.cubic.org/docs/junit/
+
import math
import sys
import re
@@ -50,9 +52,24 @@
def trial_to_junit(trial):
testsuites = et.Element('testsuites')
+ num_tests = 0
+ num_failures = 0
+ num_errors = 0
+ time = 0
+ id = 0
for suite in trial.suites:
testsuite = suite_to_junit(suite)
+ testsuite.set('id', str(id))
+ id += 1
testsuites.append(testsuite)
+ num_tests += int(testsuite.get('tests'))
+ num_failures += int(testsuite.get('failures'))
+ num_errors += int(testsuite.get('errors'))
+ time += suite.duration
+ testsuites.set('tests', str(num_tests))
+ testsuites.set('errors', str(num_errors))
+ testsuites.set('failures', str(num_failures))
+ testsuites.set('time', str(math.ceil(time)))
return testsuites
def suite_to_junit(suite):
@@ -63,9 +80,14 @@
testsuite.set('timestamp', datetime.fromtimestamp(round(suite.start_timestamp)).isoformat())
testsuite.set('time', str(math.ceil(suite.duration)))
testsuite.set('tests', str(len(suite.tests)))
- testsuite.set('failures', str(suite.count_test_results()[2]))
+ passed, skipped, failed, errors = suite.count_test_results()
+ testsuite.set('errors', str(errors))
+ testsuite.set('failures', str(failed))
+ testsuite.set('skipped', str(skipped))
+ testsuite.set('disabled', str(skipped))
for suite_test in suite.tests:
testcase = test_to_junit(suite_test)
+ testcase.set('classname', suite.name())
testsuite.append(testcase)
return testsuite
@@ -113,10 +135,12 @@
if not suite.tests:
return 'no tests were run.'
- passed, skipped, failed = suite.count_test_results()
+ passed, skipped, failed, errors = suite.count_test_results()
details = []
if failed:
details.append('fail: %d' % failed)
+ if errors:
+ details.append('errors: %d' % errors)
if passed:
details.append('pass: %d' % passed)
if skipped:
diff --git a/src/osmo_gsm_tester/suite.py b/src/osmo_gsm_tester/suite.py
index c2faa36..8c79d35 100644
--- a/src/osmo_gsm_tester/suite.py
+++ b/src/osmo_gsm_tester/suite.py
@@ -213,9 +213,9 @@
util.import_path_remove(suite_libdir)
self.duration = time.time() - self.start_timestamp
- passed, skipped, failed = self.count_test_results()
+ passed, skipped, failed, errors = self.count_test_results()
# if no tests ran, count it as failure
- if passed and not failed:
+ if passed and not failed and not errors:
self.status = SuiteRun.PASS
else:
self.status = SuiteRun.FAIL
@@ -229,14 +229,17 @@
passed = 0
skipped = 0
failed = 0
+ errors = 0
for t in self.tests:
- if t.status == test.Test.PASS:
+ if t.status == test.Test.SKIP:
+ skipped += 1
+ elif t.status == test.Test.PASS:
passed += 1
elif t.status == test.Test.FAIL:
failed += 1
- else:
- skipped += 1
- return (passed, skipped, failed)
+ else: # error, could not run
+ errors += 1
+ return (passed, skipped, failed, errors)
def remember_to_stop(self, process, respawn=False):
'''Ask suite to monitor and manage lifecycle of the Process object. If a
diff --git a/src/osmo_gsm_tester/test.py b/src/osmo_gsm_tester/test.py
index be6e8da..0bbff41 100644
--- a/src/osmo_gsm_tester/test.py
+++ b/src/osmo_gsm_tester/test.py
@@ -26,7 +26,7 @@
from . import log, util, resource
class Test(log.Origin):
- UNKNOWN = 'UNKNOWN'
+ UNKNOWN = 'UNKNOWN' # matches junit 'error'
SKIP = 'skip'
PASS = 'pass'
FAIL = 'FAIL'