Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | |
| 3 | # osmo_gsm_tester: invoke a single test run |
| 4 | # |
| 5 | # Copyright (C) 2016-2017 by sysmocom - s.f.m.c. GmbH |
| 6 | # |
| 7 | # Author: Neels Hofmeyr <neels@hofmeyr.de> |
| 8 | # |
| 9 | # This program is free software: you can redistribute it and/or modify |
| 10 | # it under the terms of the GNU Affero General Public License as |
| 11 | # published by the Free Software Foundation, either version 3 of the |
| 12 | # License, or (at your option) any later version. |
| 13 | # |
| 14 | # This program is distributed in the hope that it will be useful, |
| 15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 17 | # GNU Affero General Public License for more details. |
| 18 | # |
| 19 | # You should have received a copy of the GNU Affero General Public License |
| 20 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 21 | |
| 22 | '''osmo_gsm_tester: invoke a single test run. |
| 23 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 24 | Examples: |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 25 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 26 | ./run_once.py ~/my_trial_package/ -s osmo_trx |
| 27 | ./run_once.py ~/my_trial_package/ -c sms_tests:dyn_ts+eu_band+bts_sysmo |
| 28 | ./run_once.py ~/my_trial_package/ -c sms_tests/mo_mt_sms:bts_trx |
| 29 | |
| 30 | (The names for test suite, scenario and series names used in these examples |
| 31 | must be defined by the osmo-gsm-tester configuration.) |
| 32 | |
| 33 | A trial package contains binaries (usually built by a jenkins job) of GSM |
| 34 | software, including the core network programs as well as binaries for the |
| 35 | various BTS models. |
| 36 | |
| 37 | A test suite defines specific actions to be taken and verifies their outcome. |
| 38 | Such a test suite may leave certain aspects of a setup undefined, e.g. it may |
| 39 | be BTS model agnostic or does not care which voice codecs are chosen. |
| 40 | |
| 41 | A test scenario completes the picture in that it defines which specific choices |
| 42 | shall be made to run a test suite. Any one test suite may thus run on any |
| 43 | number of different scenarios, e.g. to test various voice codecs. |
| 44 | |
| 45 | Test scenarios may be combined. For example, one scenario may define a timeslot |
| 46 | configuration to use, while another scenario may define the voice codec |
| 47 | configuration. |
| 48 | |
| 49 | There may still be aspects that are neither required by a test suite nor |
| 50 | strictly defined by a scenario, which will be resolved automatically, e.g. by |
| 51 | choosing the first available item that matches the other constraints. |
| 52 | |
| 53 | A test run thus needs to define: a trial package containing built binaries, a |
| 54 | combination of scenarios to run a suite in, and a test suite to launch in the |
| 55 | given scenario with the given binaries. |
| 56 | |
| 57 | The osmo-gsm-tester configuration may define one or more series as a number of |
| 58 | suite:scenario combinations. So instead of a specific suite:scenario |
| 59 | combination, the name of such a series can be passed. |
| 60 | |
| 61 | If neither a combination or series is specified, the default series will be run |
| 62 | as defined in the osmo-gsm-tester configuration. |
| 63 | |
| 64 | The scenarios and suites run for a given trial will be recorded in a trial |
| 65 | package's directory: Upon launch, a 'test_package/run.<date>' directory will be |
| 66 | created, which will collect logs and reports. |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 67 | ''' |
| 68 | |
Neels Hofmeyr | e352844 | 2017-04-08 21:18:30 +0200 | [diff] [blame] | 69 | import sys |
Neels Hofmeyr | e60df69 | 2017-05-14 03:05:48 +0200 | [diff] [blame] | 70 | import argparse |
Neels Hofmeyr | d46ea13 | 2017-04-08 15:56:31 +0200 | [diff] [blame] | 71 | from osmo_gsm_tester import __version__ |
| 72 | from osmo_gsm_tester import trial, suite, log, config |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 73 | |
Neels Hofmeyr | 2123541 | 2017-05-14 03:00:21 +0200 | [diff] [blame] | 74 | def main(): |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 75 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 76 | parser = argparse.ArgumentParser(epilog=__doc__, formatter_class=argparse.RawTextHelpFormatter) |
Neels Hofmeyr | cf0304f | 2017-05-06 22:07:39 +0200 | [diff] [blame] | 77 | # Note: since we're using RawTextHelpFormatter to keep nicely separate |
| 78 | # paragraphs in the long help text, we unfortunately also need to take care |
| 79 | # of line wraps in the shorter cmdline options help. |
| 80 | # The line width here is what remains of screen width after the list of |
| 81 | # options placed by ArgumentParser. That's unfortunately subject to change |
| 82 | # and undefined, so when things change, just run a local |
| 83 | # ./osmo-gsm-tester.py --help and try to keep everything in 80 chars width. |
| 84 | # The help text is indented automatically, but line width is manual. |
| 85 | # Using multi-line strings here -- doesn't look nice in the python flow but |
| 86 | # is easiest to maintain. |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 87 | parser.add_argument('-V', '--version', action='store_true', |
| 88 | help='Show version') |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 89 | parser.add_argument('trial_package', nargs='+', |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 90 | help='Directory containing binaries to test') |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 91 | parser.add_argument('-s', '--suite-scenario', dest='suite_scenario', action='append', |
Neels Hofmeyr | cf0304f | 2017-05-06 22:07:39 +0200 | [diff] [blame] | 92 | help='''A suite-scenarios combination |
| 93 | like suite:scenario+scenario''') |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 94 | parser.add_argument('-S', '--series', dest='series', action='append', |
Neels Hofmeyr | cf0304f | 2017-05-06 22:07:39 +0200 | [diff] [blame] | 95 | help='''A series of suite-scenarios combinations |
| 96 | as defined in the osmo-gsm-tester configuration''') |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 97 | parser.add_argument('-t', '--test', dest='test', action='append', |
Neels Hofmeyr | cf0304f | 2017-05-06 22:07:39 +0200 | [diff] [blame] | 98 | help='''Run only tests matching this name. |
| 99 | Any test name that contains the given string is run. |
| 100 | To get an exact match, prepend a "=" like |
| 101 | "-t =my_exact_name". The ".py" suffix is always |
| 102 | optional.''') |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 103 | parser.add_argument('-l', '--log-level', dest='log_level', choices=log.LEVEL_STRS.keys(), |
| 104 | default=None, |
| 105 | help='Set logging level for all categories (on stdout)') |
| 106 | parser.add_argument('-T', '--traceback', dest='trace', action='store_true', |
| 107 | help='Enable logging of tracebacks') |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 108 | args = parser.parse_args() |
| 109 | |
| 110 | if args.version: |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 111 | print(__version__) |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 112 | exit(0) |
| 113 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 114 | print('combinations:', repr(args.suite_scenario)) |
| 115 | print('series:', repr(args.series)) |
| 116 | print('trials:', repr(args.trial_package)) |
| 117 | print('tests:', repr(args.test)) |
| 118 | |
Neels Hofmeyr | f816688 | 2017-05-05 19:48:35 +0200 | [diff] [blame] | 119 | # create a default log to stdout |
Neels Hofmeyr | 2296883 | 2017-05-14 16:15:58 +0200 | [diff] [blame] | 120 | log.LogTarget().style(all_origins=False, src=False) |
Neels Hofmeyr | f816688 | 2017-05-05 19:48:35 +0200 | [diff] [blame] | 121 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 122 | if args.log_level: |
| 123 | log.set_all_levels(log.LEVEL_STRS.get(args.log_level)) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 124 | if args.trace: |
| 125 | log.style_change(trace=True) |
| 126 | |
| 127 | combination_strs = list(args.suite_scenario or []) |
| 128 | # for series in args.series: |
| 129 | # combination_strs.extend(config.get_series(series)) |
| 130 | |
| 131 | if not combination_strs: |
Neels Hofmeyr | d46ea13 | 2017-04-08 15:56:31 +0200 | [diff] [blame] | 132 | combination_strs = config.read_config_file(config.DEFAULT_SUITES_CONF, if_missing_return=[]) |
| 133 | |
| 134 | if combination_strs: |
| 135 | print('Running default suites:\n ' + ('\n '.join(combination_strs))) |
Your Name | 3c6673a | 2017-04-08 18:52:39 +0200 | [diff] [blame] | 136 | else: |
| 137 | print('No default suites configured (%r)' % config.DEFAULT_SUITES_CONF) |
| 138 | |
Neels Hofmeyr | d46ea13 | 2017-04-08 15:56:31 +0200 | [diff] [blame] | 139 | |
| 140 | if not combination_strs: |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 141 | raise RuntimeError('Need at least one suite:scenario or series to run') |
| 142 | |
| 143 | suite_scenarios = [] |
| 144 | for combination_str in combination_strs: |
| 145 | suite_scenarios.append(suite.load_suite_scenario_str(combination_str)) |
| 146 | |
| 147 | test_names = [] |
| 148 | for test_name in (args.test or []): |
| 149 | found = False |
Neels Hofmeyr | f9de78f | 2017-05-06 16:04:37 +0200 | [diff] [blame] | 150 | if test_name.startswith('=') and not test_name.endswith('.py'): |
| 151 | test_name = test_name + '.py' |
Neels Hofmeyr | 930ac95 | 2017-05-06 15:49:53 +0200 | [diff] [blame] | 152 | for suite_scenario_str, suite_def, scenarios in suite_scenarios: |
| 153 | for test in suite_def.tests: |
Neels Hofmeyr | f9de78f | 2017-05-06 16:04:37 +0200 | [diff] [blame] | 154 | if test_name.startswith('='): |
| 155 | match = test_name[1:] == test.name() |
| 156 | else: |
| 157 | match = test_name in test.name() |
| 158 | if match: |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 159 | found = True |
| 160 | test_names.append(test.name()) |
| 161 | if not found: |
| 162 | raise RuntimeError('No test found for %r' % test_name) |
| 163 | if test_names: |
| 164 | print(repr(test_names)) |
| 165 | |
| 166 | trials = [] |
| 167 | for trial_package in args.trial_package: |
| 168 | t = trial.Trial(trial_package) |
Neels Hofmeyr | 2123541 | 2017-05-14 03:00:21 +0200 | [diff] [blame] | 169 | t.verify() |
| 170 | trials.append(t) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 171 | |
Neels Hofmeyr | ef42cb5 | 2017-04-08 19:38:58 +0200 | [diff] [blame] | 172 | trials_passed = [] |
| 173 | trials_failed = [] |
| 174 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 175 | for current_trial in trials: |
Your Name | 44af341 | 2017-04-13 03:11:59 +0200 | [diff] [blame] | 176 | try: |
| 177 | with current_trial: |
| 178 | suites_passed = [] |
| 179 | suites_failed = [] |
| 180 | for suite_scenario_str, suite_def, scenarios in suite_scenarios: |
| 181 | log.large_separator(current_trial.name(), suite_scenario_str) |
| 182 | suite_run = suite.SuiteRun(current_trial, suite_scenario_str, suite_def, scenarios) |
| 183 | result = suite_run.run_tests(test_names) |
| 184 | if result.all_passed: |
| 185 | suites_passed.append(suite_scenario_str) |
| 186 | suite_run.log('PASS') |
| 187 | else: |
| 188 | suites_failed.append(suite_scenario_str) |
| 189 | suite_run.err('FAIL') |
| 190 | if not suites_failed: |
| 191 | current_trial.log('PASS') |
| 192 | trials_passed.append(current_trial.name()) |
Neels Hofmeyr | ef42cb5 | 2017-04-08 19:38:58 +0200 | [diff] [blame] | 193 | else: |
Your Name | 44af341 | 2017-04-13 03:11:59 +0200 | [diff] [blame] | 194 | current_trial.err('FAIL') |
| 195 | trials_failed.append((current_trial.name(), suites_passed, suites_failed)) |
| 196 | except: |
| 197 | current_trial.log_exn() |
Neels Hofmeyr | ef42cb5 | 2017-04-08 19:38:58 +0200 | [diff] [blame] | 198 | |
Neels Hofmeyr | 2ef9b00 | 2017-04-08 21:16:27 +0200 | [diff] [blame] | 199 | sys.stderr.flush() |
| 200 | sys.stdout.flush() |
Your Name | 44af341 | 2017-04-13 03:11:59 +0200 | [diff] [blame] | 201 | log.large_separator() |
Neels Hofmeyr | ef42cb5 | 2017-04-08 19:38:58 +0200 | [diff] [blame] | 202 | if trials_passed: |
| 203 | print('Trials passed:\n ' + ('\n '.join(trials_passed))) |
| 204 | if trials_failed: |
Your Name | 44af341 | 2017-04-13 03:11:59 +0200 | [diff] [blame] | 205 | print('Trials failed:') |
| 206 | for trial_name, suites_passed, suites_failed in trials_failed: |
| 207 | print(' %s (%d of %d suite runs failed)' % (trial_name, len(suites_failed), len(suites_failed) + len(suites_passed))) |
Neels Hofmeyr | e60df69 | 2017-05-14 03:05:48 +0200 | [diff] [blame] | 208 | for suite_failed in suites_failed: |
| 209 | print(' FAIL:', suite_failed) |
Neels Hofmeyr | ef42cb5 | 2017-04-08 19:38:58 +0200 | [diff] [blame] | 210 | exit(1) |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 211 | |
Neels Hofmeyr | 2123541 | 2017-05-14 03:00:21 +0200 | [diff] [blame] | 212 | if __name__ == '__main__': |
| 213 | try: |
| 214 | main() |
| 215 | except: |
| 216 | # Tell the log to show the exception, then terminate the program with the exception anyway. |
| 217 | # Since exceptions within test runs should be caught and evaluated, this is basically about |
| 218 | # exceptions during command line parsing and such, so it's appropriate to abort immediately. |
| 219 | log.log_exn() |
| 220 | raise |
| 221 | |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 222 | # vim: expandtab tabstop=4 shiftwidth=4 |