Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | |
| 3 | # osmo_gsm_tester: invoke a single test run |
| 4 | # |
| 5 | # Copyright (C) 2016-2017 by sysmocom - s.f.m.c. GmbH |
| 6 | # |
| 7 | # Author: Neels Hofmeyr <neels@hofmeyr.de> |
| 8 | # |
| 9 | # This program is free software: you can redistribute it and/or modify |
| 10 | # it under the terms of the GNU Affero General Public License as |
| 11 | # published by the Free Software Foundation, either version 3 of the |
| 12 | # License, or (at your option) any later version. |
| 13 | # |
| 14 | # This program is distributed in the hope that it will be useful, |
| 15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 17 | # GNU Affero General Public License for more details. |
| 18 | # |
| 19 | # You should have received a copy of the GNU Affero General Public License |
| 20 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 21 | |
| 22 | '''osmo_gsm_tester: invoke a single test run. |
| 23 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 24 | Examples: |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 25 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 26 | ./run_once.py ~/my_trial_package/ -s osmo_trx |
| 27 | ./run_once.py ~/my_trial_package/ -c sms_tests:dyn_ts+eu_band+bts_sysmo |
| 28 | ./run_once.py ~/my_trial_package/ -c sms_tests/mo_mt_sms:bts_trx |
| 29 | |
| 30 | (The names for test suite, scenario and series names used in these examples |
| 31 | must be defined by the osmo-gsm-tester configuration.) |
| 32 | |
| 33 | A trial package contains binaries (usually built by a jenkins job) of GSM |
| 34 | software, including the core network programs as well as binaries for the |
| 35 | various BTS models. |
| 36 | |
| 37 | A test suite defines specific actions to be taken and verifies their outcome. |
| 38 | Such a test suite may leave certain aspects of a setup undefined, e.g. it may |
| 39 | be BTS model agnostic or does not care which voice codecs are chosen. |
| 40 | |
| 41 | A test scenario completes the picture in that it defines which specific choices |
| 42 | shall be made to run a test suite. Any one test suite may thus run on any |
| 43 | number of different scenarios, e.g. to test various voice codecs. |
| 44 | |
| 45 | Test scenarios may be combined. For example, one scenario may define a timeslot |
| 46 | configuration to use, while another scenario may define the voice codec |
| 47 | configuration. |
| 48 | |
| 49 | There may still be aspects that are neither required by a test suite nor |
| 50 | strictly defined by a scenario, which will be resolved automatically, e.g. by |
| 51 | choosing the first available item that matches the other constraints. |
| 52 | |
| 53 | A test run thus needs to define: a trial package containing built binaries, a |
| 54 | combination of scenarios to run a suite in, and a test suite to launch in the |
| 55 | given scenario with the given binaries. |
| 56 | |
| 57 | The osmo-gsm-tester configuration may define one or more series as a number of |
| 58 | suite:scenario combinations. So instead of a specific suite:scenario |
| 59 | combination, the name of such a series can be passed. |
| 60 | |
| 61 | If neither a combination or series is specified, the default series will be run |
| 62 | as defined in the osmo-gsm-tester configuration. |
| 63 | |
| 64 | The scenarios and suites run for a given trial will be recorded in a trial |
| 65 | package's directory: Upon launch, a 'test_package/run.<date>' directory will be |
| 66 | created, which will collect logs and reports. |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 67 | ''' |
| 68 | |
Neels Hofmeyr | e352844 | 2017-04-08 21:18:30 +0200 | [diff] [blame] | 69 | import sys |
Neels Hofmeyr | d46ea13 | 2017-04-08 15:56:31 +0200 | [diff] [blame] | 70 | from osmo_gsm_tester import __version__ |
| 71 | from osmo_gsm_tester import trial, suite, log, config |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 72 | |
| 73 | if __name__ == '__main__': |
| 74 | import argparse |
| 75 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 76 | parser = argparse.ArgumentParser(epilog=__doc__, formatter_class=argparse.RawTextHelpFormatter) |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 77 | parser.add_argument('-V', '--version', action='store_true', |
| 78 | help='Show version') |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 79 | parser.add_argument('trial_package', nargs='+', |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 80 | help='Directory containing binaries to test') |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 81 | parser.add_argument('-s', '--suite-scenario', dest='suite_scenario', action='append', |
| 82 | help='A suite-scenarios combination like suite:scenario+scenario') |
| 83 | parser.add_argument('-S', '--series', dest='series', action='append', |
| 84 | help='A series of suite-scenarios combinations as defined in the' |
| 85 | ' osmo-gsm-tester configuration') |
| 86 | parser.add_argument('-t', '--test', dest='test', action='append', |
Neels Hofmeyr | f9de78f | 2017-05-06 16:04:37 +0200 | [diff] [blame] | 87 | help='Run only tests matching this name. Any test name that' |
| 88 | ' contains the given string is run. To get an exact patch,' |
| 89 | ' prepend a "=" like "-t =my_exact_name". The ".py" suffix is' |
| 90 | ' always optional.') |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 91 | parser.add_argument('-l', '--log-level', dest='log_level', choices=log.LEVEL_STRS.keys(), |
| 92 | default=None, |
| 93 | help='Set logging level for all categories (on stdout)') |
| 94 | parser.add_argument('-T', '--traceback', dest='trace', action='store_true', |
| 95 | help='Enable logging of tracebacks') |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 96 | args = parser.parse_args() |
| 97 | |
| 98 | if args.version: |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 99 | print(__version__) |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 100 | exit(0) |
| 101 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 102 | print('combinations:', repr(args.suite_scenario)) |
| 103 | print('series:', repr(args.series)) |
| 104 | print('trials:', repr(args.trial_package)) |
| 105 | print('tests:', repr(args.test)) |
| 106 | |
Neels Hofmeyr | f816688 | 2017-05-05 19:48:35 +0200 | [diff] [blame] | 107 | # create a default log to stdout |
| 108 | log.LogTarget() |
| 109 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 110 | if args.log_level: |
| 111 | log.set_all_levels(log.LEVEL_STRS.get(args.log_level)) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 112 | if args.trace: |
| 113 | log.style_change(trace=True) |
| 114 | |
| 115 | combination_strs = list(args.suite_scenario or []) |
| 116 | # for series in args.series: |
| 117 | # combination_strs.extend(config.get_series(series)) |
| 118 | |
| 119 | if not combination_strs: |
Neels Hofmeyr | d46ea13 | 2017-04-08 15:56:31 +0200 | [diff] [blame] | 120 | combination_strs = config.read_config_file(config.DEFAULT_SUITES_CONF, if_missing_return=[]) |
| 121 | |
| 122 | if combination_strs: |
| 123 | print('Running default suites:\n ' + ('\n '.join(combination_strs))) |
Your Name | 3c6673a | 2017-04-08 18:52:39 +0200 | [diff] [blame] | 124 | else: |
| 125 | print('No default suites configured (%r)' % config.DEFAULT_SUITES_CONF) |
| 126 | |
Neels Hofmeyr | d46ea13 | 2017-04-08 15:56:31 +0200 | [diff] [blame] | 127 | |
| 128 | if not combination_strs: |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 129 | raise RuntimeError('Need at least one suite:scenario or series to run') |
| 130 | |
| 131 | suite_scenarios = [] |
| 132 | for combination_str in combination_strs: |
| 133 | suite_scenarios.append(suite.load_suite_scenario_str(combination_str)) |
| 134 | |
| 135 | test_names = [] |
| 136 | for test_name in (args.test or []): |
| 137 | found = False |
Neels Hofmeyr | f9de78f | 2017-05-06 16:04:37 +0200 | [diff] [blame] | 138 | if test_name.startswith('=') and not test_name.endswith('.py'): |
| 139 | test_name = test_name + '.py' |
Neels Hofmeyr | 930ac95 | 2017-05-06 15:49:53 +0200 | [diff] [blame] | 140 | for suite_scenario_str, suite_def, scenarios in suite_scenarios: |
| 141 | for test in suite_def.tests: |
Neels Hofmeyr | f9de78f | 2017-05-06 16:04:37 +0200 | [diff] [blame] | 142 | if test_name.startswith('='): |
| 143 | match = test_name[1:] == test.name() |
| 144 | else: |
| 145 | match = test_name in test.name() |
| 146 | if match: |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 147 | found = True |
| 148 | test_names.append(test.name()) |
| 149 | if not found: |
| 150 | raise RuntimeError('No test found for %r' % test_name) |
| 151 | if test_names: |
| 152 | print(repr(test_names)) |
| 153 | |
| 154 | trials = [] |
| 155 | for trial_package in args.trial_package: |
| 156 | t = trial.Trial(trial_package) |
Your Name | 44af341 | 2017-04-13 03:11:59 +0200 | [diff] [blame] | 157 | try: |
| 158 | t.verify() |
| 159 | trials.append(t) |
| 160 | except: |
| 161 | t.log_exn() |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 162 | |
Neels Hofmeyr | ef42cb5 | 2017-04-08 19:38:58 +0200 | [diff] [blame] | 163 | trials_passed = [] |
| 164 | trials_failed = [] |
| 165 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 166 | for current_trial in trials: |
Your Name | 44af341 | 2017-04-13 03:11:59 +0200 | [diff] [blame] | 167 | try: |
| 168 | with current_trial: |
| 169 | suites_passed = [] |
| 170 | suites_failed = [] |
| 171 | for suite_scenario_str, suite_def, scenarios in suite_scenarios: |
| 172 | log.large_separator(current_trial.name(), suite_scenario_str) |
| 173 | suite_run = suite.SuiteRun(current_trial, suite_scenario_str, suite_def, scenarios) |
| 174 | result = suite_run.run_tests(test_names) |
| 175 | if result.all_passed: |
| 176 | suites_passed.append(suite_scenario_str) |
| 177 | suite_run.log('PASS') |
| 178 | else: |
| 179 | suites_failed.append(suite_scenario_str) |
| 180 | suite_run.err('FAIL') |
| 181 | if not suites_failed: |
| 182 | current_trial.log('PASS') |
| 183 | trials_passed.append(current_trial.name()) |
Neels Hofmeyr | ef42cb5 | 2017-04-08 19:38:58 +0200 | [diff] [blame] | 184 | else: |
Your Name | 44af341 | 2017-04-13 03:11:59 +0200 | [diff] [blame] | 185 | current_trial.err('FAIL') |
| 186 | trials_failed.append((current_trial.name(), suites_passed, suites_failed)) |
| 187 | except: |
| 188 | current_trial.log_exn() |
Neels Hofmeyr | ef42cb5 | 2017-04-08 19:38:58 +0200 | [diff] [blame] | 189 | |
Neels Hofmeyr | 2ef9b00 | 2017-04-08 21:16:27 +0200 | [diff] [blame] | 190 | sys.stderr.flush() |
| 191 | sys.stdout.flush() |
Your Name | 44af341 | 2017-04-13 03:11:59 +0200 | [diff] [blame] | 192 | log.large_separator() |
Neels Hofmeyr | ef42cb5 | 2017-04-08 19:38:58 +0200 | [diff] [blame] | 193 | if trials_passed: |
| 194 | print('Trials passed:\n ' + ('\n '.join(trials_passed))) |
| 195 | if trials_failed: |
Your Name | 44af341 | 2017-04-13 03:11:59 +0200 | [diff] [blame] | 196 | print('Trials failed:') |
| 197 | for trial_name, suites_passed, suites_failed in trials_failed: |
| 198 | print(' %s (%d of %d suite runs failed)' % (trial_name, len(suites_failed), len(suites_failed) + len(suites_passed))) |
| 199 | for suite in suites_failed: |
| 200 | print(' FAIL:', suite) |
Neels Hofmeyr | ef42cb5 | 2017-04-08 19:38:58 +0200 | [diff] [blame] | 201 | exit(1) |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 202 | |
| 203 | # vim: expandtab tabstop=4 shiftwidth=4 |