Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 1 | # osmo_gsm_tester: test class |
| 2 | # |
| 3 | # Copyright (C) 2017 by sysmocom - s.f.m.c. GmbH |
| 4 | # |
| 5 | # Author: Pau Espin Pedrol <pespin@sysmocom.de> |
| 6 | # |
| 7 | # This program is free software: you can redistribute it and/or modify |
| 8 | # it under the terms of the GNU General Public License as |
| 9 | # published by the Free Software Foundation, either version 3 of the |
| 10 | # License, or (at your option) any later version. |
| 11 | # |
| 12 | # This program is distributed in the hope that it will be useful, |
| 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 15 | # GNU General Public License for more details. |
| 16 | # |
| 17 | # You should have received a copy of the GNU General Public License |
| 18 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 19 | |
| 20 | import os |
| 21 | import sys |
| 22 | import time |
| 23 | import traceback |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 24 | |
Pau Espin Pedrol | aa1cbdc | 2020-05-04 20:21:31 +0200 | [diff] [blame] | 25 | from . import log |
| 26 | from . import util |
| 27 | from . import resource |
| 28 | from .event_loop import MainLoop |
| 29 | |
| 30 | from .. import testenv |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 31 | |
| 32 | class Test(log.Origin): |
Pau Espin Pedrol | 02e8a8d | 2020-03-05 17:22:40 +0100 | [diff] [blame] | 33 | UNKNOWN = 'UNKNOWN' # matches junit 'error' |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 34 | SKIP = 'skip' |
| 35 | PASS = 'pass' |
| 36 | FAIL = 'FAIL' |
| 37 | |
Pau Espin Pedrol | c3cf682 | 2020-06-12 17:54:55 +0200 | [diff] [blame] | 38 | def __init__(self, suite_run, test_basename, config_test_specific): |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 39 | self.basename = test_basename |
| 40 | super().__init__(log.C_TST, self.basename) |
Pau Espin Pedrol | 5860367 | 2018-08-09 13:45:55 +0200 | [diff] [blame] | 41 | self._run_dir = None |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 42 | self.suite_run = suite_run |
Pau Espin Pedrol | c3cf682 | 2020-06-12 17:54:55 +0200 | [diff] [blame] | 43 | self._config_test_specific = config_test_specific |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 44 | self.path = os.path.join(self.suite_run.definition.suite_dir, self.basename) |
| 45 | self.status = Test.UNKNOWN |
Neels Hofmeyr | 9596b21 | 2020-12-02 09:39:01 +0100 | [diff] [blame] | 46 | self.report_fragments = [] |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 47 | self.start_timestamp = 0 |
| 48 | self.duration = 0 |
| 49 | self.fail_type = None |
| 50 | self.fail_message = None |
Pau Espin Pedrol | ec28572 | 2020-06-11 17:57:43 +0200 | [diff] [blame] | 51 | self.log_targets = [] |
Pau Espin Pedrol | 644cb41 | 2020-03-04 16:14:31 +0100 | [diff] [blame] | 52 | self._report_stdout = None |
Pau Espin Pedrol | e3d1b61 | 2020-06-15 14:27:50 +0200 | [diff] [blame] | 53 | self._kpis = None |
Pau Espin Pedrol | c3cf682 | 2020-06-12 17:54:55 +0200 | [diff] [blame] | 54 | self.timeout = int(config_test_specific['timeout']) if 'timeout' in config_test_specific else None |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 55 | |
Pau Espin Pedrol | 3063730 | 2020-05-06 21:11:02 +0200 | [diff] [blame] | 56 | def module_name(self): |
| 57 | 'Return test name without trailing .py' |
| 58 | assert self.basename.endswith('.py') |
| 59 | return self.basename[:-3] |
| 60 | |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 61 | def get_run_dir(self): |
| 62 | if self._run_dir is None: |
| 63 | self._run_dir = util.Dir(self.suite_run.get_run_dir().new_dir(self._name)) |
| 64 | return self._run_dir |
| 65 | |
| 66 | def run(self): |
Pau Espin Pedrol | aa1cbdc | 2020-05-04 20:21:31 +0200 | [diff] [blame] | 67 | testenv_obj = None |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 68 | try: |
Pau Espin Pedrol | ec28572 | 2020-06-11 17:57:43 +0200 | [diff] [blame] | 69 | self.log_targets = [log.FileLogTarget(self.get_run_dir().new_child(log.FILE_LOG)).set_all_levels(log.L_DBG).style_change(trace=True), |
| 70 | log.FileLogTarget(self.get_run_dir().new_child(log.FILE_LOG_BRIEF)).style_change(src=False, all_origins_on_levels=(log.L_ERR, log.L_TRACEBACK))] |
Pau Espin Pedrol | a442cb8 | 2020-05-05 12:54:37 +0200 | [diff] [blame] | 71 | log.large_separator(self.suite_run.trial().name(), self.suite_run.name(), self.name(), sublevel=3) |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 72 | self.status = Test.UNKNOWN |
| 73 | self.start_timestamp = time.time() |
Pau Espin Pedrol | aa1cbdc | 2020-05-04 20:21:31 +0200 | [diff] [blame] | 74 | testenv_obj = testenv.setup(self.suite_run, self) |
Neels Hofmeyr | 3b493f3 | 2020-12-01 03:51:27 +0100 | [diff] [blame] | 75 | util.run_python_file('%s.%s' % (self.suite_run.definition.name(), self.basename), |
| 76 | self.path) |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 77 | if self.status == Test.UNKNOWN: |
| 78 | self.set_pass() |
| 79 | except Exception as e: |
| 80 | if hasattr(e, 'msg'): |
| 81 | msg = e.msg |
| 82 | else: |
| 83 | msg = str(e) |
| 84 | if isinstance(e, AssertionError): |
| 85 | # AssertionError lacks further information on what was |
| 86 | # asserted. Find the line where the code asserted: |
| 87 | msg += log.get_src_from_exc_info(sys.exc_info()) |
| 88 | # add source file information to failure report |
| 89 | if hasattr(e, 'origins'): |
| 90 | msg += ' [%s]' % e.origins |
| 91 | tb_str = traceback.format_exc() |
| 92 | if isinstance(e, resource.NoResourceExn): |
| 93 | tb_str += self.suite_run.resource_status_str() |
| 94 | self.set_fail(type(e).__name__, msg, tb_str, log.get_src_from_exc_info()) |
| 95 | except BaseException as e: |
| 96 | # when the program is aborted by a signal (like Ctrl-C), escalate to abort all. |
| 97 | self.err('TEST RUN ABORTED: %s' % type(e).__name__) |
| 98 | raise |
Pau Espin Pedrol | 5eae2c5 | 2019-09-18 16:50:38 +0200 | [diff] [blame] | 99 | finally: |
Pau Espin Pedrol | aa1cbdc | 2020-05-04 20:21:31 +0200 | [diff] [blame] | 100 | if testenv_obj: |
| 101 | testenv_obj.stop() |
Pau Espin Pedrol | ec28572 | 2020-06-11 17:57:43 +0200 | [diff] [blame] | 102 | for log_tgt in self.log_targets: |
| 103 | log_tgt.remove() |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 104 | |
Neels Hofmeyr | 081e89f | 2020-12-04 17:25:23 +0100 | [diff] [blame] | 105 | def src(self): |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 106 | l = log.get_line_for_src(self.path) |
| 107 | if l is not None: |
Neels Hofmeyr | 081e89f | 2020-12-04 17:25:23 +0100 | [diff] [blame] | 108 | return '%s:%s' % (self.name(), l) |
| 109 | return self.name() |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 110 | |
Pau Espin Pedrol | 444129e | 2020-06-12 16:38:37 +0200 | [diff] [blame] | 111 | def elapsed_time(self): |
| 112 | 'time elapsed since test was started' |
| 113 | return time.time() - self.start_timestamp |
| 114 | |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 115 | def set_fail(self, fail_type, fail_message, tb_str=None, src=4): |
| 116 | self.status = Test.FAIL |
Pau Espin Pedrol | 444129e | 2020-06-12 16:38:37 +0200 | [diff] [blame] | 117 | self.duration = self.elapsed_time() |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 118 | self.fail_type = fail_type |
| 119 | self.fail_message = fail_message |
| 120 | |
| 121 | if tb_str is None: |
| 122 | # populate an exception-less call to set_fail() with traceback info |
| 123 | tb_str = ''.join(traceback.format_stack()[:-1]) |
| 124 | |
| 125 | self.fail_tb = tb_str |
| 126 | self.err('%s: %s' % (self.fail_type, self.fail_message), _src=src) |
| 127 | if self.fail_tb: |
| 128 | self.log(self.fail_tb, _level=log.L_TRACEBACK) |
| 129 | self.log('Test FAILED (%.1f sec)' % self.duration) |
| 130 | |
| 131 | def set_pass(self): |
| 132 | self.status = Test.PASS |
Pau Espin Pedrol | 444129e | 2020-06-12 16:38:37 +0200 | [diff] [blame] | 133 | self.duration = self.elapsed_time() |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 134 | self.log('Test passed (%.1f sec)' % self.duration) |
| 135 | |
| 136 | def set_skip(self): |
| 137 | self.status = Test.SKIP |
| 138 | self.duration = 0 |
| 139 | |
Pau Espin Pedrol | a75f85a | 2020-06-12 17:13:26 +0200 | [diff] [blame] | 140 | def config_test_specific(self): |
| 141 | return self._config_test_specific |
| 142 | |
Pau Espin Pedrol | e3d1b61 | 2020-06-15 14:27:50 +0200 | [diff] [blame] | 143 | def set_kpis(self, kpis): |
| 144 | if not isinstance(kpis, dict): |
| 145 | raise log.Error('Expected dictionary in toplevel kpis') |
Andre Puschmann | 8b27ded | 2020-06-22 22:49:24 +0200 | [diff] [blame] | 146 | if isinstance(self._kpis, dict): |
| 147 | self._kpis.update(kpis) |
| 148 | else: |
| 149 | self._kpis = kpis |
Pau Espin Pedrol | e3d1b61 | 2020-06-15 14:27:50 +0200 | [diff] [blame] | 150 | |
| 151 | def kpis(self): |
| 152 | return self._kpis |
| 153 | |
Pau Espin Pedrol | 644cb41 | 2020-03-04 16:14:31 +0100 | [diff] [blame] | 154 | def set_report_stdout(self, text): |
| 155 | 'Overwrite stdout text stored in report from inside a test' |
| 156 | self._report_stdout = text |
| 157 | |
| 158 | def report_stdout(self): |
| 159 | # If test overwrote the text, provide it: |
| 160 | if self._report_stdout is not None: |
| 161 | return self._report_stdout |
Pau Espin Pedrol | ec28572 | 2020-06-11 17:57:43 +0200 | [diff] [blame] | 162 | # Otherwise vy default provide the entire test brief log: |
| 163 | if len(self.log_targets) == 2 and self.log_targets[1].log_file_path() is not None: |
| 164 | with open(self.log_targets[1].log_file_path(), 'r') as myfile: |
Pau Espin Pedrol | 644cb41 | 2020-03-04 16:14:31 +0100 | [diff] [blame] | 165 | return myfile.read() |
| 166 | else: |
| 167 | return 'test log file not available' |
Pau Espin Pedrol | 5bbdab8 | 2020-02-24 18:19:10 +0100 | [diff] [blame] | 168 | |
Neels Hofmeyr | 12ed996 | 2020-12-02 17:38:55 +0100 | [diff] [blame] | 169 | def log_file(self): |
| 170 | for lt in self.log_targets: |
| 171 | if isinstance(lt, log.FileLogTarget): |
| 172 | return lt |
| 173 | return None |
| 174 | |
| 175 | def get_log_mark(self): |
| 176 | lt = self.log_file() |
| 177 | if lt is None: |
| 178 | return 0 |
| 179 | return lt.get_mark() |
| 180 | |
| 181 | def get_log_output(self, since_mark=0): |
| 182 | lt = self.log_file() |
| 183 | if lt is None: |
| 184 | return '' |
| 185 | return lt.get_output(since_mark) |
| 186 | |
Neels Hofmeyr | 9596b21 | 2020-12-02 09:39:01 +0100 | [diff] [blame] | 187 | def report_fragment(self, name, result=None, **kwargs): |
| 188 | return Test.ReportFragment(parent_test=self, name=name, result=result, **kwargs) |
| 189 | |
| 190 | class ReportFragment: |
| 191 | '''Add additional test results in junit XML. |
| 192 | Convenient method that includes a test log: |
| 193 | with test.report_fragment('foo'): |
| 194 | do_test_steps() |
| 195 | |
| 196 | Or manually add a report fragment directly: |
| 197 | test.report_fragment('foo', result = test.PASS if worked else test.FAIL) |
| 198 | ''' |
| 199 | |
| 200 | def __init__(self, parent_test, name, result=None, output=None, since_mark=None, start_time=0.0): |
| 201 | self.parent_test = parent_test |
| 202 | self.name = name |
| 203 | self.result = Test.UNKNOWN |
| 204 | self.duration = 0.0 |
| 205 | self.output = output |
| 206 | self.start_time = start_time |
| 207 | self.log_mark = since_mark |
| 208 | assert name not in (x.name for x in self.parent_test.report_fragments) |
| 209 | self.parent_test.report_fragments.append(self) |
| 210 | if result is not None: |
| 211 | self.got_result(result) |
| 212 | |
| 213 | def __str__(self): |
| 214 | return '%s/%s/%s: %s (%.1fs)' % (self.parent_test.suite_run.name(), |
| 215 | self.parent_test.name(), self.name, self.result, self.duration) |
| 216 | |
| 217 | def __enter__(self): |
| 218 | self.start_time = self.parent_test.elapsed_time() |
| 219 | self.log_mark = self.parent_test.get_log_mark() |
| 220 | |
| 221 | def __exit__(self, *exc_info): |
| 222 | self.got_result(self.parent_test.PASS if exc_info[0] is None else self.parent_test.FAIL, |
| 223 | exc_info=exc_info) |
| 224 | |
| 225 | def got_result(self, result, exc_info=None): |
| 226 | self.result = result |
| 227 | self.duration = self.parent_test.elapsed_time() - self.start_time |
| 228 | if self.log_mark is not None and self.output is None: |
| 229 | self.output = self.parent_test.get_log_output(since_mark=self.log_mark) |
| 230 | if exc_info is not None and exc_info[0] is not None: |
| 231 | o = [] |
| 232 | if self.output: |
| 233 | o.append(self.output) |
| 234 | o.extend(traceback.format_exception(*exc_info)) |
| 235 | self.output = '\n'.join(o) |
| 236 | self.parent_test.log('----- Report fragment:', self) |
| 237 | |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 238 | # vim: expandtab tabstop=4 shiftwidth=4 |