Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 1 | # osmo_gsm_tester: test class |
| 2 | # |
| 3 | # Copyright (C) 2017 by sysmocom - s.f.m.c. GmbH |
| 4 | # |
| 5 | # Author: Pau Espin Pedrol <pespin@sysmocom.de> |
| 6 | # |
| 7 | # This program is free software: you can redistribute it and/or modify |
| 8 | # it under the terms of the GNU General Public License as |
| 9 | # published by the Free Software Foundation, either version 3 of the |
| 10 | # License, or (at your option) any later version. |
| 11 | # |
| 12 | # This program is distributed in the hope that it will be useful, |
| 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 15 | # GNU General Public License for more details. |
| 16 | # |
| 17 | # You should have received a copy of the GNU General Public License |
| 18 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 19 | |
| 20 | import os |
| 21 | import sys |
| 22 | import time |
| 23 | import traceback |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 24 | |
Pau Espin Pedrol | aa1cbdc | 2020-05-04 20:21:31 +0200 | [diff] [blame] | 25 | from . import log |
| 26 | from . import util |
| 27 | from . import resource |
| 28 | from .event_loop import MainLoop |
| 29 | |
| 30 | from .. import testenv |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 31 | |
| 32 | class Test(log.Origin): |
Pau Espin Pedrol | 02e8a8d | 2020-03-05 17:22:40 +0100 | [diff] [blame] | 33 | UNKNOWN = 'UNKNOWN' # matches junit 'error' |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 34 | SKIP = 'skip' |
| 35 | PASS = 'pass' |
| 36 | FAIL = 'FAIL' |
| 37 | |
Pau Espin Pedrol | c3cf682 | 2020-06-12 17:54:55 +0200 | [diff] [blame] | 38 | def __init__(self, suite_run, test_basename, config_test_specific): |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 39 | self.basename = test_basename |
| 40 | super().__init__(log.C_TST, self.basename) |
Pau Espin Pedrol | 5860367 | 2018-08-09 13:45:55 +0200 | [diff] [blame] | 41 | self._run_dir = None |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 42 | self.suite_run = suite_run |
Pau Espin Pedrol | c3cf682 | 2020-06-12 17:54:55 +0200 | [diff] [blame] | 43 | self._config_test_specific = config_test_specific |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 44 | self.path = os.path.join(self.suite_run.definition.suite_dir, self.basename) |
| 45 | self.status = Test.UNKNOWN |
| 46 | self.start_timestamp = 0 |
| 47 | self.duration = 0 |
| 48 | self.fail_type = None |
| 49 | self.fail_message = None |
Pau Espin Pedrol | ec28572 | 2020-06-11 17:57:43 +0200 | [diff] [blame] | 50 | self.log_targets = [] |
Pau Espin Pedrol | 644cb41 | 2020-03-04 16:14:31 +0100 | [diff] [blame] | 51 | self._report_stdout = None |
Pau Espin Pedrol | c3cf682 | 2020-06-12 17:54:55 +0200 | [diff] [blame] | 52 | self.timeout = int(config_test_specific['timeout']) if 'timeout' in config_test_specific else None |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 53 | |
Pau Espin Pedrol | 3063730 | 2020-05-06 21:11:02 +0200 | [diff] [blame] | 54 | def module_name(self): |
| 55 | 'Return test name without trailing .py' |
| 56 | assert self.basename.endswith('.py') |
| 57 | return self.basename[:-3] |
| 58 | |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 59 | def get_run_dir(self): |
| 60 | if self._run_dir is None: |
| 61 | self._run_dir = util.Dir(self.suite_run.get_run_dir().new_dir(self._name)) |
| 62 | return self._run_dir |
| 63 | |
| 64 | def run(self): |
Pau Espin Pedrol | aa1cbdc | 2020-05-04 20:21:31 +0200 | [diff] [blame] | 65 | testenv_obj = None |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 66 | try: |
Pau Espin Pedrol | ec28572 | 2020-06-11 17:57:43 +0200 | [diff] [blame] | 67 | self.log_targets = [log.FileLogTarget(self.get_run_dir().new_child(log.FILE_LOG)).set_all_levels(log.L_DBG).style_change(trace=True), |
| 68 | log.FileLogTarget(self.get_run_dir().new_child(log.FILE_LOG_BRIEF)).style_change(src=False, all_origins_on_levels=(log.L_ERR, log.L_TRACEBACK))] |
Pau Espin Pedrol | a442cb8 | 2020-05-05 12:54:37 +0200 | [diff] [blame] | 69 | log.large_separator(self.suite_run.trial().name(), self.suite_run.name(), self.name(), sublevel=3) |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 70 | self.status = Test.UNKNOWN |
| 71 | self.start_timestamp = time.time() |
Pau Espin Pedrol | aa1cbdc | 2020-05-04 20:21:31 +0200 | [diff] [blame] | 72 | testenv_obj = testenv.setup(self.suite_run, self) |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 73 | with self.redirect_stdout(): |
| 74 | util.run_python_file('%s.%s' % (self.suite_run.definition.name(), self.basename), |
| 75 | self.path) |
| 76 | if self.status == Test.UNKNOWN: |
| 77 | self.set_pass() |
| 78 | except Exception as e: |
| 79 | if hasattr(e, 'msg'): |
| 80 | msg = e.msg |
| 81 | else: |
| 82 | msg = str(e) |
| 83 | if isinstance(e, AssertionError): |
| 84 | # AssertionError lacks further information on what was |
| 85 | # asserted. Find the line where the code asserted: |
| 86 | msg += log.get_src_from_exc_info(sys.exc_info()) |
| 87 | # add source file information to failure report |
| 88 | if hasattr(e, 'origins'): |
| 89 | msg += ' [%s]' % e.origins |
| 90 | tb_str = traceback.format_exc() |
| 91 | if isinstance(e, resource.NoResourceExn): |
| 92 | tb_str += self.suite_run.resource_status_str() |
| 93 | self.set_fail(type(e).__name__, msg, tb_str, log.get_src_from_exc_info()) |
| 94 | except BaseException as e: |
| 95 | # when the program is aborted by a signal (like Ctrl-C), escalate to abort all. |
| 96 | self.err('TEST RUN ABORTED: %s' % type(e).__name__) |
| 97 | raise |
Pau Espin Pedrol | 5eae2c5 | 2019-09-18 16:50:38 +0200 | [diff] [blame] | 98 | finally: |
Pau Espin Pedrol | aa1cbdc | 2020-05-04 20:21:31 +0200 | [diff] [blame] | 99 | if testenv_obj: |
| 100 | testenv_obj.stop() |
Pau Espin Pedrol | ec28572 | 2020-06-11 17:57:43 +0200 | [diff] [blame] | 101 | for log_tgt in self.log_targets: |
| 102 | log_tgt.remove() |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 103 | |
| 104 | def name(self): |
| 105 | l = log.get_line_for_src(self.path) |
| 106 | if l is not None: |
| 107 | return '%s:%s' % (self._name, l) |
| 108 | return super().name() |
| 109 | |
Pau Espin Pedrol | 444129e | 2020-06-12 16:38:37 +0200 | [diff] [blame] | 110 | def elapsed_time(self): |
| 111 | 'time elapsed since test was started' |
| 112 | return time.time() - self.start_timestamp |
| 113 | |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 114 | def set_fail(self, fail_type, fail_message, tb_str=None, src=4): |
| 115 | self.status = Test.FAIL |
Pau Espin Pedrol | 444129e | 2020-06-12 16:38:37 +0200 | [diff] [blame] | 116 | self.duration = self.elapsed_time() |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 117 | self.fail_type = fail_type |
| 118 | self.fail_message = fail_message |
| 119 | |
| 120 | if tb_str is None: |
| 121 | # populate an exception-less call to set_fail() with traceback info |
| 122 | tb_str = ''.join(traceback.format_stack()[:-1]) |
| 123 | |
| 124 | self.fail_tb = tb_str |
| 125 | self.err('%s: %s' % (self.fail_type, self.fail_message), _src=src) |
| 126 | if self.fail_tb: |
| 127 | self.log(self.fail_tb, _level=log.L_TRACEBACK) |
| 128 | self.log('Test FAILED (%.1f sec)' % self.duration) |
| 129 | |
| 130 | def set_pass(self): |
| 131 | self.status = Test.PASS |
Pau Espin Pedrol | 444129e | 2020-06-12 16:38:37 +0200 | [diff] [blame] | 132 | self.duration = self.elapsed_time() |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 133 | self.log('Test passed (%.1f sec)' % self.duration) |
| 134 | |
| 135 | def set_skip(self): |
| 136 | self.status = Test.SKIP |
| 137 | self.duration = 0 |
| 138 | |
Pau Espin Pedrol | a75f85a | 2020-06-12 17:13:26 +0200 | [diff] [blame] | 139 | def config_test_specific(self): |
| 140 | return self._config_test_specific |
| 141 | |
Pau Espin Pedrol | 644cb41 | 2020-03-04 16:14:31 +0100 | [diff] [blame] | 142 | def set_report_stdout(self, text): |
| 143 | 'Overwrite stdout text stored in report from inside a test' |
| 144 | self._report_stdout = text |
| 145 | |
| 146 | def report_stdout(self): |
| 147 | # If test overwrote the text, provide it: |
| 148 | if self._report_stdout is not None: |
| 149 | return self._report_stdout |
Pau Espin Pedrol | ec28572 | 2020-06-11 17:57:43 +0200 | [diff] [blame] | 150 | # Otherwise vy default provide the entire test brief log: |
| 151 | if len(self.log_targets) == 2 and self.log_targets[1].log_file_path() is not None: |
| 152 | with open(self.log_targets[1].log_file_path(), 'r') as myfile: |
Pau Espin Pedrol | 644cb41 | 2020-03-04 16:14:31 +0100 | [diff] [blame] | 153 | return myfile.read() |
| 154 | else: |
| 155 | return 'test log file not available' |
Pau Espin Pedrol | 5bbdab8 | 2020-02-24 18:19:10 +0100 | [diff] [blame] | 156 | |
Pau Espin Pedrol | fd5de3d | 2017-11-09 14:26:35 +0100 | [diff] [blame] | 157 | # vim: expandtab tabstop=4 shiftwidth=4 |