Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | # |
| 3 | # (C) 2017 by sysmocom s.f.m.c. GmbH <info@sysmocom.de> |
| 4 | # All rights reserved. |
| 5 | # |
| 6 | # Author: Neels Hofmeyr <nhofmeyr@sysmocom.de> |
| 7 | # |
| 8 | # This program is free software: you can redistribute it and/or modify |
| 9 | # it under the terms of the GNU General Public License as published by |
| 10 | # the Free Software Foundation, either version 3 of the License, or |
| 11 | # (at your option) any later version. |
| 12 | # |
| 13 | # This program is distributed in the hope that it will be useful, |
| 14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 16 | # GNU General Public License for more details. |
| 17 | # |
| 18 | # You should have received a copy of the GNU General Public License |
| 19 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 20 | |
| 21 | ''' |
Neels Hofmeyr | 6562c08 | 2017-10-18 03:20:04 +0200 | [diff] [blame^] | 22 | Common code for osmo_interact_vty.py and osmo_interact_ctrl.py. |
| 23 | This implements all of application interaction, piping and verification. |
| 24 | osmo_interact_{vty,ctrl}.py plug VTY and CTRL interface specific bits. |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 25 | ''' |
| 26 | |
| 27 | import argparse |
| 28 | import sys |
| 29 | import os |
| 30 | import subprocess |
| 31 | import time |
| 32 | import traceback |
| 33 | import socket |
| 34 | import shlex |
| 35 | |
| 36 | |
| 37 | class Interact: |
| 38 | |
| 39 | class StepBase: |
| 40 | command = None |
| 41 | result = None |
| 42 | leading_blanks = None |
| 43 | |
| 44 | def __init__(self): |
| 45 | self.result = [] |
| 46 | |
| 47 | def verify_interact_state(self, interact_instance): |
| 48 | # for example to verify that the last VTY prompt received shows the |
| 49 | # right node. |
| 50 | pass |
| 51 | |
| 52 | def command_str(self, interact_instance=None): |
| 53 | return self.command |
| 54 | |
| 55 | def __str__(self): |
| 56 | return '%s\n%s' % (self.command_str(), '\n'.join(self.result)) |
| 57 | |
| 58 | @staticmethod |
| 59 | def is_next_step(line, interact_instance): |
| 60 | assert not "implemented by InteractVty.VtyStep and InteractCtrl.CtrlStep" |
| 61 | |
| 62 | socket = None |
| 63 | |
| 64 | def __init__(self, step_class, port, host, verbose=False, update=False): |
| 65 | ''' |
| 66 | host is the hostname to connect to. |
| 67 | port is the CTRL port to connect on. |
| 68 | ''' |
| 69 | self.Step = step_class |
| 70 | self.port = port |
| 71 | self.host = host |
| 72 | self.verbose = verbose |
| 73 | self.update = update |
| 74 | |
Neels Hofmeyr | 7b5203f | 2017-10-18 02:09:08 +0200 | [diff] [blame] | 75 | if not port: |
| 76 | raise Exception("You need to provide port number to connect to") |
| 77 | |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 78 | def connect(self): |
| 79 | assert self.socket is None |
| 80 | retries = 30 |
| 81 | took = 0 |
| 82 | while True: |
| 83 | took += 1 |
| 84 | try: |
| 85 | self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) |
| 86 | self.socket.setblocking(1) |
| 87 | self.socket.connect((self.host, int(self.port))) |
| 88 | except IOError: |
| 89 | retries -= 1 |
| 90 | if retries <= 0: |
| 91 | raise |
| 92 | time.sleep(.1) |
| 93 | continue |
| 94 | break |
| 95 | |
| 96 | def close(self): |
| 97 | if self.socket is None: |
| 98 | return |
| 99 | self.socket.close() |
| 100 | self.socket = None |
| 101 | |
| 102 | def command(self, command): |
| 103 | assert not "implemented separately by InteractVty and InteractCtrl" |
| 104 | |
| 105 | def verify_transcript_file(self, transcript_file): |
| 106 | with open(transcript_file, 'r') as f: |
| 107 | content = f.read() |
| 108 | |
| 109 | try: |
| 110 | result = self.verify_transcript(content) |
| 111 | except: |
| 112 | print('Error while verifying transcript file %r' % transcript_file, file=sys.stderr) |
| 113 | sys.stderr.flush() |
| 114 | raise |
| 115 | |
| 116 | if not self.update: |
| 117 | return |
| 118 | content = '\n'.join(result) |
| 119 | with open(transcript_file, 'w') as f: |
| 120 | f.write(content) |
| 121 | |
| 122 | def verify_transcript(self, transcript): |
| 123 | '''' |
| 124 | transcript is a "screenshot" of a session, a multi-line string |
| 125 | including commands and expected results. |
| 126 | Feed commands to self.command() and verify the expected results. |
| 127 | ''' |
| 128 | |
| 129 | # parse steps |
| 130 | steps = [] |
| 131 | step = None |
| 132 | blank_lines = 0 |
| 133 | for line in transcript.splitlines(): |
| 134 | if not line: |
| 135 | blank_lines += 1 |
| 136 | continue |
| 137 | next_step_started = self.Step.is_next_step(line, self) |
| 138 | if next_step_started: |
| 139 | if step: |
| 140 | steps.append(step) |
| 141 | step = next_step_started |
| 142 | step.leading_blanks = blank_lines |
| 143 | blank_lines = 0 |
| 144 | elif step: |
| 145 | # we only count blank lines directly preceding the start of a |
| 146 | # next step. Insert blank lines in the middle of a response |
| 147 | # back into the response: |
| 148 | if blank_lines: |
| 149 | step.result.extend([''] * blank_lines) |
| 150 | blank_lines = 0 |
| 151 | step.result.append(line) |
| 152 | if step: |
| 153 | steps.append(step) |
| 154 | step = None |
| 155 | |
| 156 | actual_result = [] |
| 157 | |
| 158 | # run steps |
| 159 | step_nr = 0 |
| 160 | for step in steps: |
| 161 | step_nr += 1 |
| 162 | try: |
| 163 | if self.verbose: |
| 164 | if step.leading_blanks: |
| 165 | print('\n' * step.leading_blanks, end='') |
| 166 | print(step.command_str()) |
| 167 | sys.stdout.flush() |
| 168 | |
| 169 | step.verify_interact_state(self) |
| 170 | |
| 171 | res = self.command(step.command) |
| 172 | |
| 173 | if self.verbose: |
| 174 | sys.stderr.flush() |
| 175 | sys.stdout.flush() |
| 176 | print('\n'.join(res)) |
| 177 | sys.stdout.flush() |
| 178 | |
| 179 | if step.leading_blanks: |
| 180 | actual_result.extend([''] * step.leading_blanks) |
| 181 | actual_result.append(step.command_str(self)) |
| 182 | |
| 183 | match_result = self.match_lines(step.result, res) |
| 184 | |
| 185 | if self.update: |
| 186 | if match_result is True: |
| 187 | # preserve any wildcards |
| 188 | actual_result.extend(step.result) |
| 189 | else: |
| 190 | # mismatch, take exactly what came in |
| 191 | actual_result.extend(res) |
| 192 | continue |
| 193 | if match_result is not True: |
| 194 | raise Exception('Result mismatch:\n%s\n\nExpected:\n[\n%s\n]\n\nGot:\n[\n%s\n%s\n]' |
| 195 | % (match_result, step, step.command_str(), '\n'.join(res))) |
| 196 | except: |
| 197 | print('Error during transcript step %d:\n[\n%s\n]' % (step_nr, step), |
| 198 | file=sys.stderr) |
| 199 | sys.stderr.flush() |
| 200 | raise |
| 201 | |
| 202 | # final line ending |
| 203 | actual_result.append('') |
| 204 | return actual_result |
| 205 | |
| 206 | @staticmethod |
| 207 | def match_lines(expect, got): |
| 208 | ''' |
| 209 | Match two lists of strings, allowing certain wildcards: |
| 210 | - In 'expect', if a line is exactly '...', it matches any number of |
| 211 | arbitrary lines in 'got'; the implementation is trivial and skips |
| 212 | lines to the first occurence in 'got' that continues after '...'. |
| 213 | |
| 214 | Return 'True' on match, or a string describing the mismatch. |
| 215 | ''' |
| 216 | def match_line(expect_line, got_line): |
| 217 | return expect_line == got_line |
| 218 | |
| 219 | e = 0 |
| 220 | g = 0 |
| 221 | while e < len(expect): |
| 222 | if expect[e] == '...': |
| 223 | e += 1 |
| 224 | |
| 225 | if e >= len(expect): |
| 226 | # anything left in 'got' is accepted. |
| 227 | return True |
| 228 | |
| 229 | # look for the next occurence of the expected line in 'got' |
| 230 | while g < len(got) and not match_line(expect[e], got[g]): |
| 231 | g += 1 |
| 232 | continue |
| 233 | |
| 234 | if g >= len(got): |
| 235 | return 'Cannot find line %r' % expect[e] |
| 236 | |
| 237 | if not match_line(expect[e], got[g]): |
| 238 | return 'Mismatch:\nExpect:\n%r\nGot:\n%r' % (expect[e], got[g]) |
| 239 | |
| 240 | e += 1 |
| 241 | g += 1 |
| 242 | |
| 243 | if g < len(got): |
| 244 | return 'Did not expect line %r' % got[g] |
| 245 | return True |
| 246 | |
Neels Hofmeyr | 08d645b | 2017-10-18 02:45:10 +0200 | [diff] [blame] | 247 | def feed_commands(self, output, command_strs): |
| 248 | for command_str in command_strs: |
| 249 | for command in command_str.splitlines(): |
| 250 | res = self.command(command) |
| 251 | output.write('\n'.join(res)) |
| 252 | output.write('\n') |
| 253 | |
Neels Hofmeyr | 6562c08 | 2017-10-18 03:20:04 +0200 | [diff] [blame^] | 254 | def end_process(proc, quiet=False): |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 255 | if not proc: |
| 256 | return |
| 257 | |
| 258 | rc = proc.poll() |
| 259 | if rc is not None: |
Neels Hofmeyr | 6562c08 | 2017-10-18 03:20:04 +0200 | [diff] [blame^] | 260 | if not quiet: |
| 261 | print('Process has already terminated with', rc) |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 262 | proc.wait() |
| 263 | return |
| 264 | |
| 265 | proc.terminate() |
| 266 | time_to_wait_for_term = 5 |
| 267 | wait_step = 0.001 |
| 268 | waited_time = 0 |
| 269 | while True: |
| 270 | # poll returns None if proc is still running |
| 271 | if proc.poll() is not None: |
| 272 | break |
| 273 | waited_time += wait_step |
| 274 | # make wait_step approach 1.0 |
| 275 | wait_step = (1. + 5. * wait_step) / 6. |
| 276 | if waited_time >= time_to_wait_for_term: |
| 277 | break |
| 278 | time.sleep(wait_step) |
| 279 | |
| 280 | if proc.poll() is None: |
| 281 | # termination seems to be slower than that, let's just kill |
| 282 | proc.kill() |
Neels Hofmeyr | 6562c08 | 2017-10-18 03:20:04 +0200 | [diff] [blame^] | 283 | if not quiet: |
| 284 | print("Killed child process") |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 285 | elif waited_time > .002: |
Neels Hofmeyr | 6562c08 | 2017-10-18 03:20:04 +0200 | [diff] [blame^] | 286 | if not quiet: |
| 287 | print("Terminating took %.3fs" % waited_time) |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 288 | proc.wait() |
| 289 | |
| 290 | class Application: |
| 291 | proc = None |
| 292 | _devnull = None |
| 293 | |
| 294 | @staticmethod |
| 295 | def devnull(): |
| 296 | if Application._devnull is None: |
| 297 | Application._devnull = open(os.devnull, 'w') |
| 298 | return Application._devnull |
| 299 | |
Neels Hofmeyr | 08d645b | 2017-10-18 02:45:10 +0200 | [diff] [blame] | 300 | def __init__(self, run_app_str, purge_output=True, quiet=False): |
| 301 | self.command_tuple = shlex.split(run_app_str) |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 302 | self.purge_output = purge_output |
Neels Hofmeyr | 08d645b | 2017-10-18 02:45:10 +0200 | [diff] [blame] | 303 | self.quiet = quiet |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 304 | |
| 305 | def run(self): |
| 306 | out_err = None |
| 307 | if self.purge_output: |
| 308 | out_err = Application.devnull() |
| 309 | |
Neels Hofmeyr | 08d645b | 2017-10-18 02:45:10 +0200 | [diff] [blame] | 310 | if not self.quiet: |
| 311 | print('Launching: cd %r; %s' % (os.getcwd(), ' '.join(self.command_tuple))) |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 312 | self.proc = subprocess.Popen(self.command_tuple, stdout=out_err, stderr=out_err) |
| 313 | |
| 314 | def stop(self): |
Neels Hofmeyr | 6562c08 | 2017-10-18 03:20:04 +0200 | [diff] [blame^] | 315 | end_process(self.proc, self.quiet) |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 316 | |
Neels Hofmeyr | 08d645b | 2017-10-18 02:45:10 +0200 | [diff] [blame] | 317 | def verify_application(run_app_str, interact, transcript_file, verbose): |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 318 | passed = None |
| 319 | application = None |
| 320 | |
| 321 | sys.stdout.flush() |
| 322 | sys.stderr.flush() |
| 323 | |
Neels Hofmeyr | 08d645b | 2017-10-18 02:45:10 +0200 | [diff] [blame] | 324 | if run_app_str: |
| 325 | application = Application(run_app_str, purge_output=not verbose) |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 326 | application.run() |
| 327 | |
| 328 | try: |
| 329 | interact.connect() |
| 330 | interact.verify_transcript_file(transcript_file) |
| 331 | passed = True |
| 332 | except: |
| 333 | traceback.print_exc() |
| 334 | passed = False |
| 335 | interact.close() |
| 336 | |
| 337 | if application: |
| 338 | application.stop() |
| 339 | |
| 340 | sys.stdout.flush() |
| 341 | sys.stderr.flush() |
| 342 | |
| 343 | return passed |
| 344 | |
| 345 | def common_parser(): |
| 346 | parser = argparse.ArgumentParser() |
Neels Hofmeyr | 08d645b | 2017-10-18 02:45:10 +0200 | [diff] [blame] | 347 | parser.add_argument('-r', '--run', dest='run_app_str', |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 348 | help='command to run to launch application to test,' |
| 349 | ' including command line arguments. If omitted, no' |
| 350 | ' application is launched.') |
| 351 | parser.add_argument('-p', '--port', dest='port', |
| 352 | help="Port that the application opens.") |
| 353 | parser.add_argument('-H', '--host', dest='host', default='localhost', |
| 354 | help="Host that the application opens the port on.") |
Neels Hofmeyr | 6562c08 | 2017-10-18 03:20:04 +0200 | [diff] [blame^] | 355 | return parser |
| 356 | |
| 357 | def parser_add_verify_args(parser): |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 358 | parser.add_argument('-u', '--update', dest='update', action='store_true', |
| 359 | help='Do not verify, but OVERWRITE transcripts based on' |
| 360 | ' the applications current behavior. OVERWRITES TRANSCRIPT' |
| 361 | ' FILES.') |
| 362 | parser.add_argument('-v', '--verbose', action='store_true', |
| 363 | help='Print commands and application output') |
Neels Hofmeyr | 6562c08 | 2017-10-18 03:20:04 +0200 | [diff] [blame^] | 364 | parser.add_argument('transcript_files', nargs='*', help='transcript file(s) to verify') |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 365 | return parser |
| 366 | |
Neels Hofmeyr | 6562c08 | 2017-10-18 03:20:04 +0200 | [diff] [blame^] | 367 | def parser_add_run_args(parser): |
| 368 | parser.add_argument('-O', '--output', dest='output_path', |
| 369 | help="Write command results to a file instead of stdout." |
| 370 | "('-O -' writes to stdout and is the default)") |
| 371 | parser.add_argument('-c', '--command', dest='cmd_str', |
| 372 | help="Run this command (before reading input files, if any)." |
| 373 | " multiple commands may be separated by ';'") |
| 374 | parser.add_argument('cmd_files', nargs='*', help='file(s) with plain commands to run') |
| 375 | return parser |
| 376 | |
| 377 | def main_run_commands(run_app_str, output_path, cmd_str, cmd_files, interact): |
Neels Hofmeyr | 08d645b | 2017-10-18 02:45:10 +0200 | [diff] [blame] | 378 | to_stdout = False |
Neels Hofmeyr | 6562c08 | 2017-10-18 03:20:04 +0200 | [diff] [blame^] | 379 | if not output_path or output_path == '-': |
Neels Hofmeyr | 08d645b | 2017-10-18 02:45:10 +0200 | [diff] [blame] | 380 | to_stdout = True |
| 381 | output = sys.stdout |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 382 | else: |
Neels Hofmeyr | 08d645b | 2017-10-18 02:45:10 +0200 | [diff] [blame] | 383 | output = open(output_path, 'w') |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 384 | |
Neels Hofmeyr | 08d645b | 2017-10-18 02:45:10 +0200 | [diff] [blame] | 385 | application = None |
| 386 | |
| 387 | if run_app_str: |
| 388 | application = Application(run_app_str, quiet=to_stdout) |
| 389 | application.run() |
| 390 | |
| 391 | try: |
| 392 | interact.connect() |
| 393 | |
| 394 | if cmd_str: |
| 395 | interact.feed_commands(output, cmd_str.split(';')) |
| 396 | |
| 397 | for f_path in (cmd_files or []): |
| 398 | with open(f_path, 'r') as f: |
| 399 | interact.feed_commands(output, f.read().decode('utf-8').splitlines()) |
| 400 | |
| 401 | if not (cmd_str or cmd_files): |
| 402 | while True: |
| 403 | line = sys.stdin.readline() |
| 404 | if not line: |
| 405 | break; |
| 406 | interact.feed_commands(output, line.split(';')) |
| 407 | except: |
| 408 | traceback.print_exc() |
| 409 | finally: |
| 410 | if not to_stdout: |
| 411 | try: |
| 412 | output.close() |
| 413 | except: |
| 414 | traceback.print_exc() |
| 415 | |
| 416 | try: |
| 417 | interact.close() |
| 418 | except: |
| 419 | traceback.print_exc() |
| 420 | |
| 421 | if application: |
| 422 | try: |
| 423 | application.stop() |
| 424 | except: |
| 425 | traceback.print_exc() |
| 426 | |
Neels Hofmeyr | 6562c08 | 2017-10-18 03:20:04 +0200 | [diff] [blame^] | 427 | def main_verify_transcripts(run_app_str, transcript_files, interact, verbose): |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 428 | results = [] |
| 429 | for t in transcript_files: |
Neels Hofmeyr | 08d645b | 2017-10-18 02:45:10 +0200 | [diff] [blame] | 430 | passed = verify_application(run_app_str=run_app_str, |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 431 | interact=interact, |
| 432 | transcript_file=t, |
| 433 | verbose=verbose) |
| 434 | results.append((passed, t)) |
| 435 | |
| 436 | print('\nRESULTS:') |
| 437 | all_passed = True |
| 438 | for passed, t in results: |
| 439 | print('%s: %s' % ('pass' if passed else 'FAIL', t)) |
| 440 | all_passed = all_passed and passed |
| 441 | print() |
| 442 | |
| 443 | if not all_passed: |
| 444 | sys.exit(1) |
| 445 | |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 446 | # vim: tabstop=4 shiftwidth=4 expandtab nocin ai |