Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | |
| 3 | # osmo_gsm_tester: invoke a single test run |
| 4 | # |
| 5 | # Copyright (C) 2016-2017 by sysmocom - s.f.m.c. GmbH |
| 6 | # |
| 7 | # Author: Neels Hofmeyr <neels@hofmeyr.de> |
| 8 | # |
| 9 | # This program is free software: you can redistribute it and/or modify |
| 10 | # it under the terms of the GNU Affero General Public License as |
| 11 | # published by the Free Software Foundation, either version 3 of the |
| 12 | # License, or (at your option) any later version. |
| 13 | # |
| 14 | # This program is distributed in the hope that it will be useful, |
| 15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 17 | # GNU Affero General Public License for more details. |
| 18 | # |
| 19 | # You should have received a copy of the GNU Affero General Public License |
| 20 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 21 | |
| 22 | '''osmo_gsm_tester: invoke a single test run. |
| 23 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 24 | Examples: |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 25 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 26 | ./run_once.py ~/my_trial_package/ -s osmo_trx |
| 27 | ./run_once.py ~/my_trial_package/ -c sms_tests:dyn_ts+eu_band+bts_sysmo |
| 28 | ./run_once.py ~/my_trial_package/ -c sms_tests/mo_mt_sms:bts_trx |
| 29 | |
| 30 | (The names for test suite, scenario and series names used in these examples |
| 31 | must be defined by the osmo-gsm-tester configuration.) |
| 32 | |
| 33 | A trial package contains binaries (usually built by a jenkins job) of GSM |
| 34 | software, including the core network programs as well as binaries for the |
| 35 | various BTS models. |
| 36 | |
| 37 | A test suite defines specific actions to be taken and verifies their outcome. |
| 38 | Such a test suite may leave certain aspects of a setup undefined, e.g. it may |
| 39 | be BTS model agnostic or does not care which voice codecs are chosen. |
| 40 | |
| 41 | A test scenario completes the picture in that it defines which specific choices |
| 42 | shall be made to run a test suite. Any one test suite may thus run on any |
| 43 | number of different scenarios, e.g. to test various voice codecs. |
| 44 | |
| 45 | Test scenarios may be combined. For example, one scenario may define a timeslot |
| 46 | configuration to use, while another scenario may define the voice codec |
| 47 | configuration. |
| 48 | |
| 49 | There may still be aspects that are neither required by a test suite nor |
| 50 | strictly defined by a scenario, which will be resolved automatically, e.g. by |
| 51 | choosing the first available item that matches the other constraints. |
| 52 | |
| 53 | A test run thus needs to define: a trial package containing built binaries, a |
| 54 | combination of scenarios to run a suite in, and a test suite to launch in the |
| 55 | given scenario with the given binaries. |
| 56 | |
| 57 | The osmo-gsm-tester configuration may define one or more series as a number of |
| 58 | suite:scenario combinations. So instead of a specific suite:scenario |
| 59 | combination, the name of such a series can be passed. |
| 60 | |
| 61 | If neither a combination or series is specified, the default series will be run |
| 62 | as defined in the osmo-gsm-tester configuration. |
| 63 | |
| 64 | The scenarios and suites run for a given trial will be recorded in a trial |
| 65 | package's directory: Upon launch, a 'test_package/run.<date>' directory will be |
| 66 | created, which will collect logs and reports. |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 67 | ''' |
| 68 | |
Neels Hofmeyr | e352844 | 2017-04-08 21:18:30 +0200 | [diff] [blame] | 69 | import sys |
Neels Hofmeyr | d46ea13 | 2017-04-08 15:56:31 +0200 | [diff] [blame] | 70 | from osmo_gsm_tester import __version__ |
| 71 | from osmo_gsm_tester import trial, suite, log, config |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 72 | |
| 73 | if __name__ == '__main__': |
| 74 | import argparse |
| 75 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 76 | parser = argparse.ArgumentParser(epilog=__doc__, formatter_class=argparse.RawTextHelpFormatter) |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 77 | parser.add_argument('-V', '--version', action='store_true', |
| 78 | help='Show version') |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 79 | parser.add_argument('trial_package', nargs='+', |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 80 | help='Directory containing binaries to test') |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 81 | parser.add_argument('-s', '--suite-scenario', dest='suite_scenario', action='append', |
| 82 | help='A suite-scenarios combination like suite:scenario+scenario') |
| 83 | parser.add_argument('-S', '--series', dest='series', action='append', |
| 84 | help='A series of suite-scenarios combinations as defined in the' |
| 85 | ' osmo-gsm-tester configuration') |
| 86 | parser.add_argument('-t', '--test', dest='test', action='append', |
| 87 | help='Run only tests matching this name') |
| 88 | parser.add_argument('-l', '--log-level', dest='log_level', choices=log.LEVEL_STRS.keys(), |
| 89 | default=None, |
| 90 | help='Set logging level for all categories (on stdout)') |
| 91 | parser.add_argument('-T', '--traceback', dest='trace', action='store_true', |
| 92 | help='Enable logging of tracebacks') |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 93 | args = parser.parse_args() |
| 94 | |
| 95 | if args.version: |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 96 | print(__version__) |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 97 | exit(0) |
| 98 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 99 | print('combinations:', repr(args.suite_scenario)) |
| 100 | print('series:', repr(args.series)) |
| 101 | print('trials:', repr(args.trial_package)) |
| 102 | print('tests:', repr(args.test)) |
| 103 | |
| 104 | if args.log_level: |
| 105 | log.set_all_levels(log.LEVEL_STRS.get(args.log_level)) |
| 106 | log.style_change(origin_width=32) |
| 107 | if args.trace: |
| 108 | log.style_change(trace=True) |
| 109 | |
| 110 | combination_strs = list(args.suite_scenario or []) |
| 111 | # for series in args.series: |
| 112 | # combination_strs.extend(config.get_series(series)) |
| 113 | |
| 114 | if not combination_strs: |
Neels Hofmeyr | d46ea13 | 2017-04-08 15:56:31 +0200 | [diff] [blame] | 115 | combination_strs = config.read_config_file(config.DEFAULT_SUITES_CONF, if_missing_return=[]) |
| 116 | |
| 117 | if combination_strs: |
| 118 | print('Running default suites:\n ' + ('\n '.join(combination_strs))) |
Your Name | 3c6673a | 2017-04-08 18:52:39 +0200 | [diff] [blame] | 119 | else: |
| 120 | print('No default suites configured (%r)' % config.DEFAULT_SUITES_CONF) |
| 121 | |
Neels Hofmeyr | d46ea13 | 2017-04-08 15:56:31 +0200 | [diff] [blame] | 122 | |
| 123 | if not combination_strs: |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 124 | raise RuntimeError('Need at least one suite:scenario or series to run') |
| 125 | |
| 126 | suite_scenarios = [] |
| 127 | for combination_str in combination_strs: |
| 128 | suite_scenarios.append(suite.load_suite_scenario_str(combination_str)) |
| 129 | |
| 130 | test_names = [] |
| 131 | for test_name in (args.test or []): |
| 132 | found = False |
Neels Hofmeyr | 930ac95 | 2017-05-06 15:49:53 +0200 | [diff] [blame] | 133 | for suite_scenario_str, suite_def, scenarios in suite_scenarios: |
| 134 | for test in suite_def.tests: |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 135 | if test_name in test.name(): |
| 136 | found = True |
| 137 | test_names.append(test.name()) |
| 138 | if not found: |
| 139 | raise RuntimeError('No test found for %r' % test_name) |
| 140 | if test_names: |
| 141 | print(repr(test_names)) |
| 142 | |
| 143 | trials = [] |
| 144 | for trial_package in args.trial_package: |
| 145 | t = trial.Trial(trial_package) |
Your Name | 44af341 | 2017-04-13 03:11:59 +0200 | [diff] [blame] | 146 | try: |
| 147 | t.verify() |
| 148 | trials.append(t) |
| 149 | except: |
| 150 | t.log_exn() |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 151 | |
Neels Hofmeyr | ef42cb5 | 2017-04-08 19:38:58 +0200 | [diff] [blame] | 152 | trials_passed = [] |
| 153 | trials_failed = [] |
| 154 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 155 | for current_trial in trials: |
Your Name | 44af341 | 2017-04-13 03:11:59 +0200 | [diff] [blame] | 156 | try: |
| 157 | with current_trial: |
| 158 | suites_passed = [] |
| 159 | suites_failed = [] |
| 160 | for suite_scenario_str, suite_def, scenarios in suite_scenarios: |
| 161 | log.large_separator(current_trial.name(), suite_scenario_str) |
| 162 | suite_run = suite.SuiteRun(current_trial, suite_scenario_str, suite_def, scenarios) |
| 163 | result = suite_run.run_tests(test_names) |
| 164 | if result.all_passed: |
| 165 | suites_passed.append(suite_scenario_str) |
| 166 | suite_run.log('PASS') |
| 167 | else: |
| 168 | suites_failed.append(suite_scenario_str) |
| 169 | suite_run.err('FAIL') |
| 170 | if not suites_failed: |
| 171 | current_trial.log('PASS') |
| 172 | trials_passed.append(current_trial.name()) |
Neels Hofmeyr | ef42cb5 | 2017-04-08 19:38:58 +0200 | [diff] [blame] | 173 | else: |
Your Name | 44af341 | 2017-04-13 03:11:59 +0200 | [diff] [blame] | 174 | current_trial.err('FAIL') |
| 175 | trials_failed.append((current_trial.name(), suites_passed, suites_failed)) |
| 176 | except: |
| 177 | current_trial.log_exn() |
Neels Hofmeyr | ef42cb5 | 2017-04-08 19:38:58 +0200 | [diff] [blame] | 178 | |
Neels Hofmeyr | 2ef9b00 | 2017-04-08 21:16:27 +0200 | [diff] [blame] | 179 | sys.stderr.flush() |
| 180 | sys.stdout.flush() |
Your Name | 44af341 | 2017-04-13 03:11:59 +0200 | [diff] [blame] | 181 | log.large_separator() |
Neels Hofmeyr | ef42cb5 | 2017-04-08 19:38:58 +0200 | [diff] [blame] | 182 | if trials_passed: |
| 183 | print('Trials passed:\n ' + ('\n '.join(trials_passed))) |
| 184 | if trials_failed: |
Your Name | 44af341 | 2017-04-13 03:11:59 +0200 | [diff] [blame] | 185 | print('Trials failed:') |
| 186 | for trial_name, suites_passed, suites_failed in trials_failed: |
| 187 | print(' %s (%d of %d suite runs failed)' % (trial_name, len(suites_failed), len(suites_failed) + len(suites_passed))) |
| 188 | for suite in suites_failed: |
| 189 | print(' FAIL:', suite) |
Neels Hofmeyr | ef42cb5 | 2017-04-08 19:38:58 +0200 | [diff] [blame] | 190 | exit(1) |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 191 | |
| 192 | # vim: expandtab tabstop=4 shiftwidth=4 |