Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | |
| 3 | # osmo_gsm_tester: invoke a single test run |
| 4 | # |
| 5 | # Copyright (C) 2016-2017 by sysmocom - s.f.m.c. GmbH |
| 6 | # |
| 7 | # Author: Neels Hofmeyr <neels@hofmeyr.de> |
| 8 | # |
| 9 | # This program is free software: you can redistribute it and/or modify |
Harald Welte | 2720534 | 2017-06-03 09:51:45 +0200 | [diff] [blame] | 10 | # it under the terms of the GNU General Public License as |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 11 | # published by the Free Software Foundation, either version 3 of the |
| 12 | # License, or (at your option) any later version. |
| 13 | # |
| 14 | # This program is distributed in the hope that it will be useful, |
| 15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
Harald Welte | 2720534 | 2017-06-03 09:51:45 +0200 | [diff] [blame] | 17 | # GNU General Public License for more details. |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 18 | # |
Harald Welte | 2720534 | 2017-06-03 09:51:45 +0200 | [diff] [blame] | 19 | # You should have received a copy of the GNU General Public License |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 20 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 21 | |
| 22 | '''osmo_gsm_tester: invoke a single test run. |
| 23 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 24 | Examples: |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 25 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 26 | ./run_once.py ~/my_trial_package/ -s osmo_trx |
| 27 | ./run_once.py ~/my_trial_package/ -c sms_tests:dyn_ts+eu_band+bts_sysmo |
| 28 | ./run_once.py ~/my_trial_package/ -c sms_tests/mo_mt_sms:bts_trx |
| 29 | |
| 30 | (The names for test suite, scenario and series names used in these examples |
| 31 | must be defined by the osmo-gsm-tester configuration.) |
| 32 | |
| 33 | A trial package contains binaries (usually built by a jenkins job) of GSM |
| 34 | software, including the core network programs as well as binaries for the |
| 35 | various BTS models. |
| 36 | |
| 37 | A test suite defines specific actions to be taken and verifies their outcome. |
| 38 | Such a test suite may leave certain aspects of a setup undefined, e.g. it may |
| 39 | be BTS model agnostic or does not care which voice codecs are chosen. |
| 40 | |
| 41 | A test scenario completes the picture in that it defines which specific choices |
| 42 | shall be made to run a test suite. Any one test suite may thus run on any |
| 43 | number of different scenarios, e.g. to test various voice codecs. |
| 44 | |
| 45 | Test scenarios may be combined. For example, one scenario may define a timeslot |
| 46 | configuration to use, while another scenario may define the voice codec |
| 47 | configuration. |
| 48 | |
| 49 | There may still be aspects that are neither required by a test suite nor |
| 50 | strictly defined by a scenario, which will be resolved automatically, e.g. by |
| 51 | choosing the first available item that matches the other constraints. |
| 52 | |
| 53 | A test run thus needs to define: a trial package containing built binaries, a |
| 54 | combination of scenarios to run a suite in, and a test suite to launch in the |
| 55 | given scenario with the given binaries. |
| 56 | |
| 57 | The osmo-gsm-tester configuration may define one or more series as a number of |
| 58 | suite:scenario combinations. So instead of a specific suite:scenario |
| 59 | combination, the name of such a series can be passed. |
| 60 | |
| 61 | If neither a combination or series is specified, the default series will be run |
| 62 | as defined in the osmo-gsm-tester configuration. |
| 63 | |
| 64 | The scenarios and suites run for a given trial will be recorded in a trial |
| 65 | package's directory: Upon launch, a 'test_package/run.<date>' directory will be |
| 66 | created, which will collect logs and reports. |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 67 | ''' |
| 68 | |
Neels Hofmeyr | e352844 | 2017-04-08 21:18:30 +0200 | [diff] [blame] | 69 | import sys |
Neels Hofmeyr | e60df69 | 2017-05-14 03:05:48 +0200 | [diff] [blame] | 70 | import argparse |
Pau Espin Pedrol | 469316f | 2017-05-17 14:51:31 +0200 | [diff] [blame] | 71 | from signal import * |
Neels Hofmeyr | d46ea13 | 2017-04-08 15:56:31 +0200 | [diff] [blame] | 72 | from osmo_gsm_tester import __version__ |
| 73 | from osmo_gsm_tester import trial, suite, log, config |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 74 | |
Pau Espin Pedrol | 469316f | 2017-05-17 14:51:31 +0200 | [diff] [blame] | 75 | def sig_handler_cleanup(signum, frame): |
| 76 | print("killed by signal %d" % signum) |
| 77 | # This sys.exit() will raise a SystemExit base exception at the current |
| 78 | # point of execution. Code must be prepared to clean system-wide resources |
| 79 | # by using the "finally" section. This allows at the end 'atexit' hooks to |
| 80 | # be called before exiting. |
| 81 | sys.exit(1) |
| 82 | |
Neels Hofmeyr | 2123541 | 2017-05-14 03:00:21 +0200 | [diff] [blame] | 83 | def main(): |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 84 | |
Pau Espin Pedrol | 469316f | 2017-05-17 14:51:31 +0200 | [diff] [blame] | 85 | for sig in (SIGINT, SIGTERM, SIGQUIT, SIGPIPE, SIGHUP): |
| 86 | signal(sig, sig_handler_cleanup) |
| 87 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 88 | parser = argparse.ArgumentParser(epilog=__doc__, formatter_class=argparse.RawTextHelpFormatter) |
Neels Hofmeyr | cf0304f | 2017-05-06 22:07:39 +0200 | [diff] [blame] | 89 | # Note: since we're using RawTextHelpFormatter to keep nicely separate |
| 90 | # paragraphs in the long help text, we unfortunately also need to take care |
| 91 | # of line wraps in the shorter cmdline options help. |
| 92 | # The line width here is what remains of screen width after the list of |
| 93 | # options placed by ArgumentParser. That's unfortunately subject to change |
| 94 | # and undefined, so when things change, just run a local |
| 95 | # ./osmo-gsm-tester.py --help and try to keep everything in 80 chars width. |
| 96 | # The help text is indented automatically, but line width is manual. |
| 97 | # Using multi-line strings here -- doesn't look nice in the python flow but |
| 98 | # is easiest to maintain. |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 99 | parser.add_argument('-V', '--version', action='store_true', |
| 100 | help='Show version') |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 101 | parser.add_argument('trial_package', |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 102 | help='Directory containing binaries to test') |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 103 | parser.add_argument('-s', '--suite-scenario', dest='suite_scenario', action='append', |
Neels Hofmeyr | cf0304f | 2017-05-06 22:07:39 +0200 | [diff] [blame] | 104 | help='''A suite-scenarios combination |
| 105 | like suite:scenario+scenario''') |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 106 | parser.add_argument('-S', '--series', dest='series', action='append', |
Neels Hofmeyr | cf0304f | 2017-05-06 22:07:39 +0200 | [diff] [blame] | 107 | help='''A series of suite-scenarios combinations |
| 108 | as defined in the osmo-gsm-tester configuration''') |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 109 | parser.add_argument('-t', '--test', dest='test', action='append', |
Neels Hofmeyr | cf0304f | 2017-05-06 22:07:39 +0200 | [diff] [blame] | 110 | help='''Run only tests matching this name. |
| 111 | Any test name that contains the given string is run. |
| 112 | To get an exact match, prepend a "=" like |
| 113 | "-t =my_exact_name". The ".py" suffix is always |
| 114 | optional.''') |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 115 | parser.add_argument('-l', '--log-level', dest='log_level', choices=log.LEVEL_STRS.keys(), |
| 116 | default=None, |
| 117 | help='Set logging level for all categories (on stdout)') |
| 118 | parser.add_argument('-T', '--traceback', dest='trace', action='store_true', |
Neels Hofmeyr | 42bebfa | 2017-06-06 19:41:42 +0200 | [diff] [blame] | 119 | help='Enable stdout logging of tracebacks') |
| 120 | parser.add_argument('-R', '--source', dest='source', action='store_true', |
| 121 | help='Enable stdout logging of source file') |
Neels Hofmeyr | f15eaf9 | 2017-06-05 18:03:53 +0200 | [diff] [blame] | 122 | parser.add_argument('-c', '--conf-dir', dest='conf_dir', |
| 123 | help='''Specify configuration dir (overrides |
| 124 | OSMO_GSM_TESTER_CONF env and default locations)''') |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 125 | args = parser.parse_args() |
| 126 | |
| 127 | if args.version: |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 128 | print(__version__) |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 129 | exit(0) |
| 130 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 131 | print('combinations:', repr(args.suite_scenario)) |
| 132 | print('series:', repr(args.series)) |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 133 | print('trial:', repr(args.trial_package)) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 134 | print('tests:', repr(args.test)) |
| 135 | |
Neels Hofmeyr | f816688 | 2017-05-05 19:48:35 +0200 | [diff] [blame] | 136 | # create a default log to stdout |
Neels Hofmeyr | 9576f5f | 2017-05-24 18:31:01 +0200 | [diff] [blame] | 137 | log.LogTarget().style(all_origins_on_levels=(log.L_ERR, log.L_TRACEBACK), src=False) |
Neels Hofmeyr | f816688 | 2017-05-05 19:48:35 +0200 | [diff] [blame] | 138 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 139 | if args.log_level: |
| 140 | log.set_all_levels(log.LEVEL_STRS.get(args.log_level)) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 141 | if args.trace: |
| 142 | log.style_change(trace=True) |
Neels Hofmeyr | 42bebfa | 2017-06-06 19:41:42 +0200 | [diff] [blame] | 143 | if args.source: |
| 144 | log.style_change(src=True) |
Neels Hofmeyr | f15eaf9 | 2017-06-05 18:03:53 +0200 | [diff] [blame] | 145 | if args.conf_dir: |
| 146 | config.override_conf = args.conf_dir |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 147 | |
| 148 | combination_strs = list(args.suite_scenario or []) |
| 149 | # for series in args.series: |
| 150 | # combination_strs.extend(config.get_series(series)) |
| 151 | |
| 152 | if not combination_strs: |
Neels Hofmeyr | d46ea13 | 2017-04-08 15:56:31 +0200 | [diff] [blame] | 153 | combination_strs = config.read_config_file(config.DEFAULT_SUITES_CONF, if_missing_return=[]) |
| 154 | |
| 155 | if combination_strs: |
| 156 | print('Running default suites:\n ' + ('\n '.join(combination_strs))) |
Your Name | 3c6673a | 2017-04-08 18:52:39 +0200 | [diff] [blame] | 157 | else: |
| 158 | print('No default suites configured (%r)' % config.DEFAULT_SUITES_CONF) |
| 159 | |
Neels Hofmeyr | d46ea13 | 2017-04-08 15:56:31 +0200 | [diff] [blame] | 160 | |
| 161 | if not combination_strs: |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 162 | raise RuntimeError('Need at least one suite:scenario or series to run') |
| 163 | |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 164 | # make sure all suite:scenarios exist |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 165 | suite_scenarios = [] |
| 166 | for combination_str in combination_strs: |
| 167 | suite_scenarios.append(suite.load_suite_scenario_str(combination_str)) |
| 168 | |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 169 | # pick tests and make sure they exist |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 170 | test_names = [] |
| 171 | for test_name in (args.test or []): |
| 172 | found = False |
Neels Hofmeyr | f9de78f | 2017-05-06 16:04:37 +0200 | [diff] [blame] | 173 | if test_name.startswith('=') and not test_name.endswith('.py'): |
| 174 | test_name = test_name + '.py' |
Neels Hofmeyr | 930ac95 | 2017-05-06 15:49:53 +0200 | [diff] [blame] | 175 | for suite_scenario_str, suite_def, scenarios in suite_scenarios: |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 176 | for def_test_name in suite_def.test_basenames: |
Neels Hofmeyr | f9de78f | 2017-05-06 16:04:37 +0200 | [diff] [blame] | 177 | if test_name.startswith('='): |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 178 | match = test_name[1:] == def_test_name |
Neels Hofmeyr | f9de78f | 2017-05-06 16:04:37 +0200 | [diff] [blame] | 179 | else: |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 180 | match = test_name in def_test_name |
Neels Hofmeyr | f9de78f | 2017-05-06 16:04:37 +0200 | [diff] [blame] | 181 | if match: |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 182 | found = True |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 183 | test_names.append(def_test_name) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 184 | if not found: |
| 185 | raise RuntimeError('No test found for %r' % test_name) |
| 186 | if test_names: |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 187 | test_names = sorted(set(test_names)) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 188 | print(repr(test_names)) |
| 189 | |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 190 | with trial.Trial(args.trial_package) as current_trial: |
| 191 | current_trial.verify() |
| 192 | for suite_scenario_str, suite_def, scenarios in suite_scenarios: |
| 193 | current_trial.add_suite_run(suite_scenario_str, suite_def, scenarios) |
| 194 | current_trial.run_suites(test_names) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 195 | |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 196 | if current_trial.status != trial.Trial.PASS: |
| 197 | return 1 |
| 198 | return 0 |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 199 | |
Neels Hofmeyr | 2123541 | 2017-05-14 03:00:21 +0200 | [diff] [blame] | 200 | if __name__ == '__main__': |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 201 | rc = 2 |
Neels Hofmeyr | 2123541 | 2017-05-14 03:00:21 +0200 | [diff] [blame] | 202 | try: |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 203 | rc = main() |
Neels Hofmeyr | 2123541 | 2017-05-14 03:00:21 +0200 | [diff] [blame] | 204 | except: |
| 205 | # Tell the log to show the exception, then terminate the program with the exception anyway. |
| 206 | # Since exceptions within test runs should be caught and evaluated, this is basically about |
| 207 | # exceptions during command line parsing and such, so it's appropriate to abort immediately. |
| 208 | log.log_exn() |
| 209 | raise |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 210 | exit(rc) |
Neels Hofmeyr | 2123541 | 2017-05-14 03:00:21 +0200 | [diff] [blame] | 211 | |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 212 | # vim: expandtab tabstop=4 shiftwidth=4 |