Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | |
| 3 | # (C) 2013 by Katerina Barone-Adesi <kat.obsc@gmail.com> |
| 4 | # This program is free software: you can redistribute it and/or modify |
| 5 | # it under the terms of the GNU General Public License as published by |
| 6 | # the Free Software Foundation, either version 3 of the License, or |
| 7 | # (at your option) any later version. |
| 8 | |
| 9 | # This program is distributed in the hope that it will be useful, |
| 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 12 | # GNU General Public License for more details. |
| 13 | |
| 14 | # You should have received a copy of the GNU General Public License |
| 15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
Max | 2f266e0 | 2018-01-03 17:35:59 +0100 | [diff] [blame] | 16 | from __future__ import print_function |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 17 | import os |
| 18 | import os.path |
| 19 | import time |
Max | 6c33a15 | 2016-04-13 17:29:51 +0200 | [diff] [blame] | 20 | import sys, shutil, stat |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 21 | import tempfile |
| 22 | |
| 23 | import osmopy.obscvty as obscvty |
| 24 | import osmopy.osmoutil as osmoutil |
| 25 | |
| 26 | |
| 27 | # Return true iff all the tests for the given config pass |
| 28 | def test_config(app_desc, config, tmpdir, verbose=True): |
| 29 | try: |
Holger Hans Peter Freyther | 4e98c26 | 2014-07-04 20:43:38 +0200 | [diff] [blame] | 30 | err = 0 |
Holger Hans Peter Freyther | b819b57 | 2015-01-31 21:15:06 +0100 | [diff] [blame] | 31 | if test_config_atest(app_desc, config, verify_doc, verbose)[0] > 0: |
Holger Hans Peter Freyther | 4e98c26 | 2014-07-04 20:43:38 +0200 | [diff] [blame] | 32 | err += 1 |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 33 | |
| 34 | newconfig = copy_config(tmpdir, config) |
Holger Hans Peter Freyther | 4e98c26 | 2014-07-04 20:43:38 +0200 | [diff] [blame] | 35 | if test_config_atest(app_desc, newconfig, write_config, verbose) > 0: |
| 36 | err += 1 |
| 37 | |
| 38 | if test_config_atest(app_desc, newconfig, token_vty_command, verbose) > 0: |
| 39 | err += 1 |
| 40 | |
| 41 | return err |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 42 | |
| 43 | # If there's a socket error, skip the rest of the tests for this config |
| 44 | except IOError: |
| 45 | return 1 |
| 46 | |
| 47 | |
| 48 | def test_config_atest(app_desc, config, run_test, verbose=True): |
| 49 | proc = None |
| 50 | ret = None |
Neels Hofmeyr | 8972d06 | 2017-02-24 20:49:34 +0100 | [diff] [blame] | 51 | vty = None |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 52 | try: |
Max | 5f4567b | 2016-03-30 14:58:17 +0200 | [diff] [blame] | 53 | cmd = app_desc[1].split(' ') + [ "-c", config] |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 54 | if verbose: |
Max | 2f266e0 | 2018-01-03 17:35:59 +0100 | [diff] [blame] | 55 | print("Verifying %s, test %s" % (' '.join(cmd), run_test.__name__)) |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 56 | |
| 57 | proc = osmoutil.popen_devnull(cmd) |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 58 | end = app_desc[2] |
| 59 | port = app_desc[0] |
| 60 | vty = obscvty.VTYInteract(end, "127.0.0.1", port) |
| 61 | ret = run_test(vty) |
| 62 | |
| 63 | except IOError as se: |
Max | 2f266e0 | 2018-01-03 17:35:59 +0100 | [diff] [blame] | 64 | print("Failed to verify %s" % ' '.join(cmd), file=sys.stderr) |
| 65 | print("Current directory: %s" % os.getcwd(), file=sys.stderr) |
| 66 | print("Error was %s" % se, file=sys.stderr) |
| 67 | print("Config was\n%s" % open(config).read(), file=sys.stderr) |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 68 | raise se |
| 69 | |
| 70 | finally: |
| 71 | if proc: |
| 72 | osmoutil.end_proc(proc) |
Neels Hofmeyr | 8972d06 | 2017-02-24 20:49:34 +0100 | [diff] [blame] | 73 | if vty: |
| 74 | vty._close_socket() |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 75 | |
| 76 | return ret |
| 77 | |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 78 | def copy_config(dirname, config): |
Max | 334d680 | 2016-04-07 14:11:25 +0200 | [diff] [blame] | 79 | shutil.rmtree(dirname, True) |
| 80 | ign = shutil.ignore_patterns('*.cfg') |
| 81 | shutil.copytree(os.path.dirname(config), dirname, ignore=ign) |
Max | 6c33a15 | 2016-04-13 17:29:51 +0200 | [diff] [blame] | 82 | os.chmod(dirname, stat.S_IRWXU) |
Max | 334d680 | 2016-04-07 14:11:25 +0200 | [diff] [blame] | 83 | |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 84 | try: |
| 85 | os.stat(dirname) |
| 86 | except OSError: |
| 87 | os.mkdir(dirname) |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 88 | |
| 89 | prefix = os.path.basename(config) |
| 90 | tmpfile = tempfile.NamedTemporaryFile( |
| 91 | dir=dirname, prefix=prefix, delete=False) |
| 92 | tmpfile.write(open(config).read()) |
| 93 | tmpfile.close() |
| 94 | # This works around the precautions NamedTemporaryFile is made for... |
| 95 | return tmpfile.name |
| 96 | |
| 97 | |
| 98 | def write_config(vty): |
| 99 | new_config = vty.enabled_command("write") |
Holger Hans Peter Freyther | 0edf0c9 | 2016-08-15 12:31:46 +0200 | [diff] [blame] | 100 | if not new_config.startswith("Configuration saved to "): |
| 101 | print(new_config) |
| 102 | return 1, [new_config] |
Holger Hans Peter Freyther | b819b57 | 2015-01-31 21:15:06 +0100 | [diff] [blame] | 103 | return 0 |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 104 | |
| 105 | |
| 106 | # The only purpose of this function is to verify a working vty |
| 107 | def token_vty_command(vty): |
| 108 | vty.command("help") |
Holger Hans Peter Freyther | 4e98c26 | 2014-07-04 20:43:38 +0200 | [diff] [blame] | 109 | return 0 |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 110 | |
| 111 | |
| 112 | # This may warn about the same doc missing multiple times, by design |
| 113 | def verify_doc(vty): |
| 114 | xml = vty.command("show online-help") |
| 115 | split_at = "<command" |
| 116 | all_errs = [] |
| 117 | for command in xml.split(split_at): |
| 118 | if "(null)" in command: |
| 119 | lines = command.split("\n") |
| 120 | cmd_line = split_at + lines[0] |
| 121 | err_lines = [] |
| 122 | for line in lines: |
| 123 | if '(null)' in line: |
| 124 | err_lines.append(line) |
| 125 | |
| 126 | all_errs.append(err_lines) |
| 127 | |
Max | 2f266e0 | 2018-01-03 17:35:59 +0100 | [diff] [blame] | 128 | print("Documentation error (missing docs): \n%s\n%s\n" % ( |
| 129 | cmd_line, '\n'.join(err_lines)), file=sys.stderr) |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 130 | |
| 131 | return (len(all_errs), all_errs) |
| 132 | |
| 133 | |
| 134 | # Skip testing the configurations of anything that hasn't been compiled |
| 135 | def app_exists(app_desc): |
Max | 5f4567b | 2016-03-30 14:58:17 +0200 | [diff] [blame] | 136 | cmd = app_desc[1].split(' ')[0] |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 137 | return os.path.exists(cmd) |
| 138 | |
| 139 | |
| 140 | def remove_tmpdir(tmpdir): |
| 141 | files = os.listdir(tmpdir) |
| 142 | for f in files: |
| 143 | os.unlink(os.path.join(tmpdir, f)) |
| 144 | os.rmdir(tmpdir) |
| 145 | |
| 146 | |
Max | d401cc1 | 2016-03-30 14:58:16 +0200 | [diff] [blame] | 147 | def check_configs_tested(basedir, app_configs, ignore_configs): |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 148 | configs = [] |
| 149 | for root, dirs, files in os.walk(basedir): |
| 150 | for f in files: |
Max | d401cc1 | 2016-03-30 14:58:16 +0200 | [diff] [blame] | 151 | if f.endswith(".cfg") and f not in ignore_configs: |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 152 | configs.append(os.path.join(root, f)) |
| 153 | for config in configs: |
| 154 | found = False |
| 155 | for app in app_configs: |
| 156 | if config in app_configs[app]: |
| 157 | found = True |
| 158 | if not found: |
Max | 2f266e0 | 2018-01-03 17:35:59 +0100 | [diff] [blame] | 159 | print("Warning: %s is not being tested" % config, file=sys.stderr) |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 160 | |
| 161 | |
| 162 | def test_all_apps(apps, app_configs, tmpdir="writtenconfig", verbose=True, |
Max | d401cc1 | 2016-03-30 14:58:16 +0200 | [diff] [blame] | 163 | confpath=".", rmtmp=False, ignore_configs=[]): |
| 164 | check_configs_tested("doc/examples/", app_configs, ignore_configs) |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 165 | errors = 0 |
| 166 | for app in apps: |
| 167 | if not app_exists(app): |
Max | 2f266e0 | 2018-01-03 17:35:59 +0100 | [diff] [blame] | 168 | print("Skipping app %s (not found)" % app[1], file=sys.stderr) |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 169 | continue |
| 170 | |
| 171 | configs = app_configs[app[3]] |
| 172 | for config in configs: |
Kat | 0248d3b | 2013-04-05 20:19:17 +0200 | [diff] [blame] | 173 | config = os.path.join(confpath, config) |
Neels Hofmeyr | baa6f12 | 2017-02-27 01:06:44 +0100 | [diff] [blame] | 174 | errors += test_config(app, config, tmpdir, verbose) |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 175 | |
Kat | 0248d3b | 2013-04-05 20:19:17 +0200 | [diff] [blame] | 176 | if rmtmp or not errors: |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 177 | remove_tmpdir(tmpdir) |
| 178 | |
Neels Hofmeyr | baa6f12 | 2017-02-27 01:06:44 +0100 | [diff] [blame] | 179 | if errors: |
Max | 2f266e0 | 2018-01-03 17:35:59 +0100 | [diff] [blame] | 180 | print("ERRORS: %d" % errors, file=sys.stderr) |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 181 | return errors |
| 182 | |
| 183 | |
| 184 | if __name__ == '__main__': |
| 185 | import argparse |
| 186 | |
| 187 | confpath = "." |
Kat | 0248d3b | 2013-04-05 20:19:17 +0200 | [diff] [blame] | 188 | wordir = "." |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 189 | |
| 190 | parser = argparse.ArgumentParser() |
| 191 | parser.add_argument("--e1nitb", action="store_true", dest="e1nitb") |
| 192 | parser.add_argument("-v", "--verbose", dest="verbose", |
| 193 | action="store_true", help="verbose mode") |
| 194 | parser.add_argument("-p", "--pythonconfpath", dest="p", |
| 195 | help="searchpath for config") |
Kat | 0248d3b | 2013-04-05 20:19:17 +0200 | [diff] [blame] | 196 | parser.add_argument("-w", "--workdir", dest="w", |
| 197 | help="Working directory to run in") |
| 198 | |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 199 | args = parser.parse_args() |
| 200 | |
| 201 | if args.p: |
| 202 | confpath = args.p |
| 203 | |
Kat | 0248d3b | 2013-04-05 20:19:17 +0200 | [diff] [blame] | 204 | if args.w: |
| 205 | workdir = args.w |
| 206 | |
Kat | 0270be4 | 2013-04-05 21:34:52 +0200 | [diff] [blame] | 207 | osmoappdesc = osmoutil.importappconf_or_quit(confpath, "osmoappdesc", |
| 208 | args.p) |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 209 | |
| 210 | apps = osmoappdesc.apps |
| 211 | configs = osmoappdesc.app_configs |
Max | d401cc1 | 2016-03-30 14:58:16 +0200 | [diff] [blame] | 212 | ignores = getattr(osmoappdesc, 'ignore_configs', []) |
Kat | a7185c6 | 2013-04-04 17:31:13 +0200 | [diff] [blame] | 213 | |
| 214 | if args.e1nitb: |
| 215 | configs['nitb'].extend(osmoappdesc.nitb_e1_configs) |
| 216 | |
Kat | 0248d3b | 2013-04-05 20:19:17 +0200 | [diff] [blame] | 217 | os.chdir(workdir) |
Max | d401cc1 | 2016-03-30 14:58:16 +0200 | [diff] [blame] | 218 | sys.exit(test_all_apps(apps, configs, ignore_configs=ignores, |
| 219 | confpath=confpath, verbose=args.verbose)) |