Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 1 | # osmo_gsm_tester: report: directory of binaries to be tested |
| 2 | # |
| 3 | # Copyright (C) 2016-2017 by sysmocom - s.f.m.c. GmbH |
| 4 | # |
| 5 | # Author: Pau Espin Pedrol <pespin@sysmocom.de> |
| 6 | # |
| 7 | # This program is free software: you can redistribute it and/or modify |
Harald Welte | 2720534 | 2017-06-03 09:51:45 +0200 | [diff] [blame] | 8 | # it under the terms of the GNU General Public License as |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 9 | # published by the Free Software Foundation, either version 3 of the |
| 10 | # License, or (at your option) any later version. |
| 11 | # |
| 12 | # This program is distributed in the hope that it will be useful, |
| 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
Harald Welte | 2720534 | 2017-06-03 09:51:45 +0200 | [diff] [blame] | 15 | # GNU General Public License for more details. |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 16 | # |
Harald Welte | 2720534 | 2017-06-03 09:51:45 +0200 | [diff] [blame] | 17 | # You should have received a copy of the GNU General Public License |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 18 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 19 | |
Pau Espin Pedrol | 02e8a8d | 2020-03-05 17:22:40 +0100 | [diff] [blame] | 20 | # junit xml format: https://llg.cubic.org/docs/junit/ |
| 21 | |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 22 | import math |
Pau Espin Pedrol | 3f088da | 2020-03-02 10:44:09 +0100 | [diff] [blame] | 23 | import sys |
| 24 | import re |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 25 | from datetime import datetime |
| 26 | import xml.etree.ElementTree as et |
Pau Espin Pedrol | 5bbdab8 | 2020-02-24 18:19:10 +0100 | [diff] [blame] | 27 | from xml.sax.saxutils import escape |
Holger Hans Peter Freyther | d03acdf | 2018-09-23 18:06:48 +0100 | [diff] [blame] | 28 | from . import test |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 29 | |
Pau Espin Pedrol | 3f088da | 2020-03-02 10:44:09 +0100 | [diff] [blame] | 30 | invalid_xml_char_ranges = [(0x00, 0x08), (0x0B, 0x0C), (0x0E, 0x1F), (0x7F, 0x84), |
| 31 | (0x86, 0x9F), (0xFDD0, 0xFDDF), (0xFFFE, 0xFFFF)] |
| 32 | if sys.maxunicode >= 0x10000: # not narrow build |
| 33 | invalid_xml_char_ranges.extend([(0x1FFFE, 0x1FFFF), (0x2FFFE, 0x2FFFF), |
| 34 | (0x3FFFE, 0x3FFFF), (0x4FFFE, 0x4FFFF), |
| 35 | (0x5FFFE, 0x5FFFF), (0x6FFFE, 0x6FFFF), |
| 36 | (0x7FFFE, 0x7FFFF), (0x8FFFE, 0x8FFFF), |
| 37 | (0x9FFFE, 0x9FFFF), (0xAFFFE, 0xAFFFF), |
| 38 | (0xBFFFE, 0xBFFFF), (0xCFFFE, 0xCFFFF), |
| 39 | (0xDFFFE, 0xDFFFF), (0xEFFFE, 0xEFFFF), |
| 40 | (0xFFFFE, 0xFFFFF), (0x10FFFE, 0x10FFFF)]) |
| 41 | invalid_xml_char_ranges_str = ['%s-%s' % (chr(low), chr(high)) |
| 42 | for (low, high) in invalid_xml_char_ranges] |
| 43 | invalid_xml_char_ranges_regex = re.compile('[%s]' % ''.join(invalid_xml_char_ranges_str)) |
Neels Hofmeyr | 16c8be4 | 2020-12-03 22:45:53 +0100 | [diff] [blame] | 44 | ansi_color_re = re.compile('\033[0-9;]{1,4}m') |
Pau Espin Pedrol | 3f088da | 2020-03-02 10:44:09 +0100 | [diff] [blame] | 45 | |
| 46 | def escape_xml_invalid_characters(str): |
| 47 | replacement_char = '\uFFFD' # Unicode replacement character |
| 48 | return invalid_xml_char_ranges_regex.sub(replacement_char, escape(str)) |
| 49 | |
Neels Hofmeyr | 16c8be4 | 2020-12-03 22:45:53 +0100 | [diff] [blame] | 50 | def strip_ansi_colors(text): |
| 51 | return ''.join(ansi_color_re.split(text)) |
| 52 | |
Pau Espin Pedrol | ef919c0 | 2020-06-05 13:19:02 +0200 | [diff] [blame] | 53 | def hash_info_to_junit(testsuite, hash_info): |
| 54 | properties = et.SubElement(testsuite, 'properties') |
| 55 | for key, val in hash_info.items(): |
| 56 | prop = et.SubElement(properties, 'property') |
| 57 | prop.set('name', 'ref:' + key) |
| 58 | prop.set('value', val) |
| 59 | |
Pau Espin Pedrol | e3d1b61 | 2020-06-15 14:27:50 +0200 | [diff] [blame] | 60 | def dict_to_junit(parent, d): |
| 61 | for key, val in d.items(): |
| 62 | if isinstance(val, dict): |
| 63 | node = et.SubElement(parent, 'kpi_node') |
| 64 | node.set('name', key) |
| 65 | dict_to_junit(node, val) |
| 66 | continue |
| 67 | if isinstance(val, (tuple, list)): |
| 68 | node = et.SubElement(parent, 'kpi_node') |
| 69 | node.set('name', key) |
| 70 | list_to_junit(node, val) |
| 71 | continue |
| 72 | # scalar: |
| 73 | node = et.SubElement(parent, 'property') |
| 74 | node.set('name', key) |
| 75 | node.set('value', str(val)) |
| 76 | |
| 77 | def list_to_junit(parent, li): |
| 78 | for i in range(len(li)): |
| 79 | if isinstance(li[i], dict): |
| 80 | node = et.SubElement(parent, 'kpi_node') |
| 81 | node.set('name', str(i)) |
| 82 | dict_to_junit(node, li[i]) |
| 83 | continue |
| 84 | if isinstance(val, (tuple, list)): |
| 85 | node = et.SubElement(parent, 'kpi_node') |
| 86 | node.set('name', str(i)) |
| 87 | list_to_junit(node, li[i]) |
| 88 | continue |
| 89 | # scalar: |
| 90 | node = et.SubElement(parent, 'property') |
| 91 | node.set('name', str(i)) |
| 92 | node.set('value', str(li[i])) |
| 93 | |
| 94 | def kpis_to_junit(parent, kpis): |
| 95 | if not kpis: |
| 96 | return |
| 97 | assert isinstance(kpis, dict) |
| 98 | knode = et.SubElement(parent, 'kpis') |
| 99 | dict_to_junit(knode, kpis) |
Pau Espin Pedrol | ef919c0 | 2020-06-05 13:19:02 +0200 | [diff] [blame] | 100 | |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 101 | def trial_to_junit_write(trial, junit_path): |
| 102 | elements = et.ElementTree(element=trial_to_junit(trial)) |
| 103 | elements.write(junit_path) |
| 104 | |
| 105 | def trial_to_junit(trial): |
| 106 | testsuites = et.Element('testsuites') |
Pau Espin Pedrol | 02e8a8d | 2020-03-05 17:22:40 +0100 | [diff] [blame] | 107 | num_tests = 0 |
| 108 | num_failures = 0 |
| 109 | num_errors = 0 |
| 110 | time = 0 |
| 111 | id = 0 |
Pau Espin Pedrol | ef919c0 | 2020-06-05 13:19:02 +0200 | [diff] [blame] | 112 | hash_info = trial.get_all_inst_hash_info() |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 113 | for suite in trial.suites: |
| 114 | testsuite = suite_to_junit(suite) |
Pau Espin Pedrol | ef919c0 | 2020-06-05 13:19:02 +0200 | [diff] [blame] | 115 | hash_info_to_junit(testsuite, hash_info) |
Pau Espin Pedrol | 02e8a8d | 2020-03-05 17:22:40 +0100 | [diff] [blame] | 116 | testsuite.set('id', str(id)) |
| 117 | id += 1 |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 118 | testsuites.append(testsuite) |
Pau Espin Pedrol | 02e8a8d | 2020-03-05 17:22:40 +0100 | [diff] [blame] | 119 | num_tests += int(testsuite.get('tests')) |
| 120 | num_failures += int(testsuite.get('failures')) |
| 121 | num_errors += int(testsuite.get('errors')) |
| 122 | time += suite.duration |
| 123 | testsuites.set('tests', str(num_tests)) |
| 124 | testsuites.set('errors', str(num_errors)) |
| 125 | testsuites.set('failures', str(num_failures)) |
| 126 | testsuites.set('time', str(math.ceil(time))) |
Pau Espin Pedrol | f3df1e4 | 2020-06-05 11:53:24 +0200 | [diff] [blame] | 127 | testsuites.set('name', trial.name()) |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 128 | return testsuites |
| 129 | |
| 130 | def suite_to_junit(suite): |
| 131 | testsuite = et.Element('testsuite') |
| 132 | testsuite.set('name', suite.name()) |
| 133 | testsuite.set('hostname', 'localhost') |
Neels Hofmeyr | f8e6186 | 2017-06-06 23:08:07 +0200 | [diff] [blame] | 134 | if suite.start_timestamp: |
| 135 | testsuite.set('timestamp', datetime.fromtimestamp(round(suite.start_timestamp)).isoformat()) |
| 136 | testsuite.set('time', str(math.ceil(suite.duration))) |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 137 | testsuite.set('tests', str(len(suite.tests))) |
Pau Espin Pedrol | 02e8a8d | 2020-03-05 17:22:40 +0100 | [diff] [blame] | 138 | passed, skipped, failed, errors = suite.count_test_results() |
Holger Hans Peter Freyther | f922a99 | 2019-02-27 04:35:17 +0000 | [diff] [blame] | 139 | for suite_test in suite.tests: |
| 140 | testcase = test_to_junit(suite_test) |
Pau Espin Pedrol | 02e8a8d | 2020-03-05 17:22:40 +0100 | [diff] [blame] | 141 | testcase.set('classname', suite.name()) |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 142 | testsuite.append(testcase) |
Neels Hofmeyr | 9596b21 | 2020-12-02 09:39:01 +0100 | [diff] [blame] | 143 | |
| 144 | for report_fragment in suite_test.report_fragments: |
| 145 | full_name = '%s/%s' % (suite_test.name(), report_fragment.name) |
| 146 | el = et.Element('testcase') |
| 147 | el.set('name', full_name) |
| 148 | el.set('time', str(math.ceil(report_fragment.duration))) |
| 149 | if report_fragment.result == test.Test.SKIP: |
| 150 | et.SubElement(el, 'skipped') |
| 151 | skipped += 1 |
| 152 | elif report_fragment.result == test.Test.FAIL: |
| 153 | failure = et.SubElement(el, 'failure') |
| 154 | failure.set('type', suite_test.fail_type or 'failure') |
| 155 | failed += 1 |
| 156 | elif report_fragment.result != test.Test.PASS: |
| 157 | error = et.SubElement(el, 'error') |
| 158 | error.text = 'could not run' |
| 159 | errors += 1 |
| 160 | |
| 161 | if report_fragment.output: |
| 162 | sout = et.SubElement(el, 'system-out') |
Neels Hofmeyr | 16c8be4 | 2020-12-03 22:45:53 +0100 | [diff] [blame] | 163 | sout.text = escape_xml_invalid_characters(strip_ansi_colors(report_fragment.output)) |
Neels Hofmeyr | 9596b21 | 2020-12-02 09:39:01 +0100 | [diff] [blame] | 164 | testsuite.append(el) |
| 165 | |
| 166 | testsuite.set('errors', str(errors)) |
| 167 | testsuite.set('failures', str(failed)) |
| 168 | testsuite.set('skipped', str(skipped)) |
| 169 | testsuite.set('disabled', str(skipped)) |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 170 | return testsuite |
| 171 | |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 172 | def test_to_junit(t): |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 173 | testcase = et.Element('testcase') |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 174 | testcase.set('name', t.name()) |
| 175 | testcase.set('time', str(math.ceil(t.duration))) |
| 176 | if t.status == test.Test.SKIP: |
Holger Hans Peter Freyther | d03acdf | 2018-09-23 18:06:48 +0100 | [diff] [blame] | 177 | et.SubElement(testcase, 'skipped') |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 178 | elif t.status == test.Test.FAIL: |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 179 | failure = et.SubElement(testcase, 'failure') |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 180 | failure.set('type', t.fail_type or 'failure') |
| 181 | failure.text = t.fail_message |
| 182 | if t.fail_tb: |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 183 | system_err = et.SubElement(testcase, 'system-err') |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 184 | system_err.text = t.fail_tb |
| 185 | elif t.status != test.Test.PASS: |
Neels Hofmeyr | f8e6186 | 2017-06-06 23:08:07 +0200 | [diff] [blame] | 186 | error = et.SubElement(testcase, 'error') |
| 187 | error.text = 'could not run' |
Pau Espin Pedrol | e3d1b61 | 2020-06-15 14:27:50 +0200 | [diff] [blame] | 188 | kpis_to_junit(testcase, t.kpis()) |
Pau Espin Pedrol | 5bbdab8 | 2020-02-24 18:19:10 +0100 | [diff] [blame] | 189 | sout = et.SubElement(testcase, 'system-out') |
Pau Espin Pedrol | 644cb41 | 2020-03-04 16:14:31 +0100 | [diff] [blame] | 190 | sout.text = escape_xml_invalid_characters(t.report_stdout()) |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 191 | return testcase |
| 192 | |
| 193 | def trial_to_text(trial): |
Neels Hofmeyr | 7d79ea4 | 2020-11-28 10:06:14 +0100 | [diff] [blame] | 194 | suite_passes = [] |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 195 | suite_failures = [] |
| 196 | count_fail = 0 |
| 197 | count_pass = 0 |
| 198 | for suite in trial.suites: |
| 199 | if suite.passed(): |
| 200 | count_pass += 1 |
Neels Hofmeyr | 7d79ea4 | 2020-11-28 10:06:14 +0100 | [diff] [blame] | 201 | suite_passes.append(suite_to_text(suite)) |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 202 | else: |
| 203 | count_fail += 1 |
| 204 | suite_failures.append(suite_to_text(suite)) |
| 205 | |
| 206 | summary = ['%s: %s' % (trial.name(), trial.status)] |
| 207 | if count_fail: |
| 208 | summary.append('%d suites failed' % count_fail) |
| 209 | if count_pass: |
| 210 | summary.append('%d suites passed' % count_pass) |
| 211 | msg = [', '.join(summary)] |
| 212 | msg.extend(suite_failures) |
Neels Hofmeyr | 7d79ea4 | 2020-11-28 10:06:14 +0100 | [diff] [blame] | 213 | msg.extend(suite_passes) |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 214 | return '\n'.join(msg) |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 215 | |
| 216 | def suite_to_text(suite): |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 217 | if not suite.tests: |
| 218 | return 'no tests were run.' |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 219 | |
Pau Espin Pedrol | 02e8a8d | 2020-03-05 17:22:40 +0100 | [diff] [blame] | 220 | passed, skipped, failed, errors = suite.count_test_results() |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 221 | details = [] |
| 222 | if failed: |
| 223 | details.append('fail: %d' % failed) |
Pau Espin Pedrol | 02e8a8d | 2020-03-05 17:22:40 +0100 | [diff] [blame] | 224 | if errors: |
| 225 | details.append('errors: %d' % errors) |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 226 | if passed: |
| 227 | details.append('pass: %d' % passed) |
| 228 | if skipped: |
| 229 | details.append('skip: %d' % skipped) |
| 230 | msgs = ['%s: %s (%s)' % (suite.status, suite.name(), ', '.join(details))] |
| 231 | msgs.extend([test_to_text(t) for t in suite.tests]) |
| 232 | return '\n '.join(msgs) |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 233 | |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 234 | def test_to_text(t): |
| 235 | msgs = ['%s: %s' % (t.status, t.name())] |
| 236 | if t.start_timestamp: |
| 237 | msgs.append('(%.1f sec)' % t.duration) |
| 238 | if t.status == test.Test.FAIL: |
| 239 | msgs.append('%s: %s' % (t.fail_type, t.fail_message)) |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 240 | return ' '.join(msgs) |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 241 | |
| 242 | # vim: expandtab tabstop=4 shiftwidth=4 |