Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | # |
| 3 | # (C) 2017 by sysmocom s.f.m.c. GmbH <info@sysmocom.de> |
| 4 | # All rights reserved. |
| 5 | # |
| 6 | # Author: Neels Hofmeyr <nhofmeyr@sysmocom.de> |
| 7 | # |
| 8 | # This program is free software: you can redistribute it and/or modify |
| 9 | # it under the terms of the GNU General Public License as published by |
| 10 | # the Free Software Foundation, either version 3 of the License, or |
| 11 | # (at your option) any later version. |
| 12 | # |
| 13 | # This program is distributed in the hope that it will be useful, |
| 14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 16 | # GNU General Public License for more details. |
| 17 | # |
| 18 | # You should have received a copy of the GNU General Public License |
| 19 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 20 | |
| 21 | ''' |
Neels Hofmeyr | be76f4d | 2017-12-19 13:46:57 +0100 | [diff] [blame] | 22 | Common code for VTY and CTRL interface interaction and transcript verification. |
Neels Hofmeyr | 6562c08 | 2017-10-18 03:20:04 +0200 | [diff] [blame] | 23 | This implements all of application interaction, piping and verification. |
Neels Hofmeyr | be76f4d | 2017-12-19 13:46:57 +0100 | [diff] [blame] | 24 | vty.py and ctrl.py plug VTY and CTRL interface specific bits. |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 25 | ''' |
| 26 | |
Neels Hofmeyr | 56aa478 | 2017-12-19 14:12:16 +0100 | [diff] [blame] | 27 | # Our setup.py currently wants everything to be parsable by both py2 and py3. |
| 28 | # IMHO that is not a good idea, but until that changes, let's just keep this |
| 29 | # py2 legacy shim in here so we can syntax-check this py3 module with py2. |
| 30 | from __future__ import print_function |
| 31 | |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 32 | import argparse |
| 33 | import sys |
| 34 | import os |
| 35 | import subprocess |
| 36 | import time |
| 37 | import traceback |
| 38 | import socket |
| 39 | import shlex |
Neels Hofmeyr | 48b951a | 2017-11-29 18:21:18 +0100 | [diff] [blame] | 40 | import re |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 41 | |
| 42 | |
| 43 | class Interact: |
| 44 | |
| 45 | class StepBase: |
| 46 | command = None |
| 47 | result = None |
| 48 | leading_blanks = None |
| 49 | |
| 50 | def __init__(self): |
| 51 | self.result = [] |
| 52 | |
| 53 | def verify_interact_state(self, interact_instance): |
| 54 | # for example to verify that the last VTY prompt received shows the |
| 55 | # right node. |
| 56 | pass |
| 57 | |
| 58 | def command_str(self, interact_instance=None): |
| 59 | return self.command |
| 60 | |
| 61 | def __str__(self): |
| 62 | return '%s\n%s' % (self.command_str(), '\n'.join(self.result)) |
| 63 | |
| 64 | @staticmethod |
| 65 | def is_next_step(line, interact_instance): |
| 66 | assert not "implemented by InteractVty.VtyStep and InteractCtrl.CtrlStep" |
| 67 | |
| 68 | socket = None |
| 69 | |
| 70 | def __init__(self, step_class, port, host, verbose=False, update=False): |
| 71 | ''' |
| 72 | host is the hostname to connect to. |
| 73 | port is the CTRL port to connect on. |
| 74 | ''' |
| 75 | self.Step = step_class |
| 76 | self.port = port |
| 77 | self.host = host |
| 78 | self.verbose = verbose |
| 79 | self.update = update |
| 80 | |
Neels Hofmeyr | 7b5203f | 2017-10-18 02:09:08 +0200 | [diff] [blame] | 81 | if not port: |
| 82 | raise Exception("You need to provide port number to connect to") |
| 83 | |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 84 | def connect(self): |
| 85 | assert self.socket is None |
| 86 | retries = 30 |
| 87 | took = 0 |
| 88 | while True: |
| 89 | took += 1 |
| 90 | try: |
| 91 | self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) |
| 92 | self.socket.setblocking(1) |
| 93 | self.socket.connect((self.host, int(self.port))) |
| 94 | except IOError: |
| 95 | retries -= 1 |
| 96 | if retries <= 0: |
| 97 | raise |
| 98 | time.sleep(.1) |
| 99 | continue |
| 100 | break |
| 101 | |
| 102 | def close(self): |
| 103 | if self.socket is None: |
| 104 | return |
| 105 | self.socket.close() |
| 106 | self.socket = None |
| 107 | |
| 108 | def command(self, command): |
| 109 | assert not "implemented separately by InteractVty and InteractCtrl" |
| 110 | |
| 111 | def verify_transcript_file(self, transcript_file): |
| 112 | with open(transcript_file, 'r') as f: |
| 113 | content = f.read() |
| 114 | |
| 115 | try: |
| 116 | result = self.verify_transcript(content) |
| 117 | except: |
| 118 | print('Error while verifying transcript file %r' % transcript_file, file=sys.stderr) |
| 119 | sys.stderr.flush() |
| 120 | raise |
| 121 | |
| 122 | if not self.update: |
| 123 | return |
| 124 | content = '\n'.join(result) |
| 125 | with open(transcript_file, 'w') as f: |
| 126 | f.write(content) |
| 127 | |
| 128 | def verify_transcript(self, transcript): |
| 129 | '''' |
| 130 | transcript is a "screenshot" of a session, a multi-line string |
| 131 | including commands and expected results. |
| 132 | Feed commands to self.command() and verify the expected results. |
| 133 | ''' |
| 134 | |
| 135 | # parse steps |
| 136 | steps = [] |
| 137 | step = None |
| 138 | blank_lines = 0 |
| 139 | for line in transcript.splitlines(): |
| 140 | if not line: |
| 141 | blank_lines += 1 |
| 142 | continue |
| 143 | next_step_started = self.Step.is_next_step(line, self) |
| 144 | if next_step_started: |
| 145 | if step: |
| 146 | steps.append(step) |
| 147 | step = next_step_started |
| 148 | step.leading_blanks = blank_lines |
| 149 | blank_lines = 0 |
| 150 | elif step: |
| 151 | # we only count blank lines directly preceding the start of a |
| 152 | # next step. Insert blank lines in the middle of a response |
| 153 | # back into the response: |
| 154 | if blank_lines: |
| 155 | step.result.extend([''] * blank_lines) |
| 156 | blank_lines = 0 |
| 157 | step.result.append(line) |
| 158 | if step: |
| 159 | steps.append(step) |
| 160 | step = None |
| 161 | |
| 162 | actual_result = [] |
| 163 | |
| 164 | # run steps |
| 165 | step_nr = 0 |
| 166 | for step in steps: |
| 167 | step_nr += 1 |
| 168 | try: |
| 169 | if self.verbose: |
| 170 | if step.leading_blanks: |
| 171 | print('\n' * step.leading_blanks, end='') |
| 172 | print(step.command_str()) |
| 173 | sys.stdout.flush() |
| 174 | |
| 175 | step.verify_interact_state(self) |
| 176 | |
| 177 | res = self.command(step.command) |
| 178 | |
| 179 | if self.verbose: |
| 180 | sys.stderr.flush() |
| 181 | sys.stdout.flush() |
| 182 | print('\n'.join(res)) |
| 183 | sys.stdout.flush() |
| 184 | |
| 185 | if step.leading_blanks: |
| 186 | actual_result.extend([''] * step.leading_blanks) |
| 187 | actual_result.append(step.command_str(self)) |
| 188 | |
| 189 | match_result = self.match_lines(step.result, res) |
| 190 | |
| 191 | if self.update: |
| 192 | if match_result is True: |
| 193 | # preserve any wildcards |
| 194 | actual_result.extend(step.result) |
| 195 | else: |
| 196 | # mismatch, take exactly what came in |
| 197 | actual_result.extend(res) |
| 198 | continue |
| 199 | if match_result is not True: |
| 200 | raise Exception('Result mismatch:\n%s\n\nExpected:\n[\n%s\n]\n\nGot:\n[\n%s\n%s\n]' |
| 201 | % (match_result, step, step.command_str(), '\n'.join(res))) |
| 202 | except: |
| 203 | print('Error during transcript step %d:\n[\n%s\n]' % (step_nr, step), |
| 204 | file=sys.stderr) |
| 205 | sys.stderr.flush() |
| 206 | raise |
| 207 | |
| 208 | # final line ending |
| 209 | actual_result.append('') |
| 210 | return actual_result |
| 211 | |
| 212 | @staticmethod |
| 213 | def match_lines(expect, got): |
| 214 | ''' |
| 215 | Match two lists of strings, allowing certain wildcards: |
| 216 | - In 'expect', if a line is exactly '...', it matches any number of |
| 217 | arbitrary lines in 'got'; the implementation is trivial and skips |
| 218 | lines to the first occurence in 'got' that continues after '...'. |
Neels Hofmeyr | 48b951a | 2017-11-29 18:21:18 +0100 | [diff] [blame] | 219 | - If an 'expect' line is '... !regex', it matches any number of |
| 220 | lines like '...', but the given regex must not match any of those |
| 221 | lines. |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 222 | |
| 223 | Return 'True' on match, or a string describing the mismatch. |
| 224 | ''' |
| 225 | def match_line(expect_line, got_line): |
| 226 | return expect_line == got_line |
| 227 | |
Neels Hofmeyr | 48b951a | 2017-11-29 18:21:18 +0100 | [diff] [blame] | 228 | ANY = '...' |
| 229 | ANY_EXCEPT = '... !' |
| 230 | |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 231 | e = 0 |
| 232 | g = 0 |
| 233 | while e < len(expect): |
Neels Hofmeyr | 48b951a | 2017-11-29 18:21:18 +0100 | [diff] [blame] | 234 | if expect[e] == ANY or expect[e].startswith(ANY_EXCEPT): |
| 235 | wildcard = expect[e] |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 236 | e += 1 |
Neels Hofmeyr | 48b951a | 2017-11-29 18:21:18 +0100 | [diff] [blame] | 237 | g_end = g |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 238 | |
| 239 | if e >= len(expect): |
| 240 | # anything left in 'got' is accepted. |
Neels Hofmeyr | 48b951a | 2017-11-29 18:21:18 +0100 | [diff] [blame] | 241 | g_end = len(got) |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 242 | |
| 243 | # look for the next occurence of the expected line in 'got' |
Neels Hofmeyr | 48b951a | 2017-11-29 18:21:18 +0100 | [diff] [blame] | 244 | while g_end < len(got) and not match_line(expect[e], got[g_end]): |
| 245 | g_end += 1 |
| 246 | |
| 247 | if wildcard == ANY: |
| 248 | # no restrictions on lines |
| 249 | g = g_end |
| 250 | |
| 251 | elif wildcard.startswith(ANY_EXCEPT): |
| 252 | except_re = re.compile(wildcard[len(ANY_EXCEPT):]) |
| 253 | while g < g_end: |
| 254 | if except_re.search(got[g]): |
| 255 | return ('Got forbidden line for wildcard %r:' |
| 256 | ' did not expect %r in line %d of response' |
| 257 | % (wildcard, got[g], g)) |
| 258 | g += 1 |
| 259 | |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 260 | continue |
| 261 | |
| 262 | if g >= len(got): |
| 263 | return 'Cannot find line %r' % expect[e] |
| 264 | |
| 265 | if not match_line(expect[e], got[g]): |
| 266 | return 'Mismatch:\nExpect:\n%r\nGot:\n%r' % (expect[e], got[g]) |
| 267 | |
| 268 | e += 1 |
| 269 | g += 1 |
| 270 | |
| 271 | if g < len(got): |
| 272 | return 'Did not expect line %r' % got[g] |
| 273 | return True |
| 274 | |
Neels Hofmeyr | 08d645b | 2017-10-18 02:45:10 +0200 | [diff] [blame] | 275 | def feed_commands(self, output, command_strs): |
| 276 | for command_str in command_strs: |
| 277 | for command in command_str.splitlines(): |
| 278 | res = self.command(command) |
| 279 | output.write('\n'.join(res)) |
| 280 | output.write('\n') |
| 281 | |
Neels Hofmeyr | 6562c08 | 2017-10-18 03:20:04 +0200 | [diff] [blame] | 282 | def end_process(proc, quiet=False): |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 283 | if not proc: |
| 284 | return |
| 285 | |
| 286 | rc = proc.poll() |
| 287 | if rc is not None: |
Neels Hofmeyr | 6562c08 | 2017-10-18 03:20:04 +0200 | [diff] [blame] | 288 | if not quiet: |
| 289 | print('Process has already terminated with', rc) |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 290 | proc.wait() |
| 291 | return |
| 292 | |
| 293 | proc.terminate() |
| 294 | time_to_wait_for_term = 5 |
| 295 | wait_step = 0.001 |
| 296 | waited_time = 0 |
| 297 | while True: |
| 298 | # poll returns None if proc is still running |
| 299 | if proc.poll() is not None: |
| 300 | break |
| 301 | waited_time += wait_step |
| 302 | # make wait_step approach 1.0 |
| 303 | wait_step = (1. + 5. * wait_step) / 6. |
| 304 | if waited_time >= time_to_wait_for_term: |
| 305 | break |
| 306 | time.sleep(wait_step) |
| 307 | |
| 308 | if proc.poll() is None: |
| 309 | # termination seems to be slower than that, let's just kill |
| 310 | proc.kill() |
Neels Hofmeyr | 6562c08 | 2017-10-18 03:20:04 +0200 | [diff] [blame] | 311 | if not quiet: |
| 312 | print("Killed child process") |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 313 | elif waited_time > .002: |
Neels Hofmeyr | 6562c08 | 2017-10-18 03:20:04 +0200 | [diff] [blame] | 314 | if not quiet: |
| 315 | print("Terminating took %.3fs" % waited_time) |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 316 | proc.wait() |
| 317 | |
| 318 | class Application: |
| 319 | proc = None |
| 320 | _devnull = None |
| 321 | |
| 322 | @staticmethod |
| 323 | def devnull(): |
| 324 | if Application._devnull is None: |
| 325 | Application._devnull = open(os.devnull, 'w') |
| 326 | return Application._devnull |
| 327 | |
Neels Hofmeyr | 08d645b | 2017-10-18 02:45:10 +0200 | [diff] [blame] | 328 | def __init__(self, run_app_str, purge_output=True, quiet=False): |
| 329 | self.command_tuple = shlex.split(run_app_str) |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 330 | self.purge_output = purge_output |
Neels Hofmeyr | 08d645b | 2017-10-18 02:45:10 +0200 | [diff] [blame] | 331 | self.quiet = quiet |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 332 | |
| 333 | def run(self): |
| 334 | out_err = None |
| 335 | if self.purge_output: |
| 336 | out_err = Application.devnull() |
| 337 | |
Neels Hofmeyr | 08d645b | 2017-10-18 02:45:10 +0200 | [diff] [blame] | 338 | if not self.quiet: |
| 339 | print('Launching: cd %r; %s' % (os.getcwd(), ' '.join(self.command_tuple))) |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 340 | self.proc = subprocess.Popen(self.command_tuple, stdout=out_err, stderr=out_err) |
| 341 | |
| 342 | def stop(self): |
Neels Hofmeyr | 6562c08 | 2017-10-18 03:20:04 +0200 | [diff] [blame] | 343 | end_process(self.proc, self.quiet) |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 344 | |
Neels Hofmeyr | 08d645b | 2017-10-18 02:45:10 +0200 | [diff] [blame] | 345 | def verify_application(run_app_str, interact, transcript_file, verbose): |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 346 | passed = None |
| 347 | application = None |
| 348 | |
| 349 | sys.stdout.flush() |
| 350 | sys.stderr.flush() |
| 351 | |
Neels Hofmeyr | 08d645b | 2017-10-18 02:45:10 +0200 | [diff] [blame] | 352 | if run_app_str: |
| 353 | application = Application(run_app_str, purge_output=not verbose) |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 354 | application.run() |
| 355 | |
| 356 | try: |
| 357 | interact.connect() |
| 358 | interact.verify_transcript_file(transcript_file) |
| 359 | passed = True |
| 360 | except: |
| 361 | traceback.print_exc() |
| 362 | passed = False |
| 363 | interact.close() |
| 364 | |
| 365 | if application: |
| 366 | application.stop() |
| 367 | |
| 368 | sys.stdout.flush() |
| 369 | sys.stderr.flush() |
| 370 | |
| 371 | return passed |
| 372 | |
Neels Hofmeyr | be76f4d | 2017-12-19 13:46:57 +0100 | [diff] [blame] | 373 | def common_parser(doc=None): |
| 374 | parser = argparse.ArgumentParser(description=doc, |
| 375 | formatter_class=argparse.RawDescriptionHelpFormatter) |
Neels Hofmeyr | 08d645b | 2017-10-18 02:45:10 +0200 | [diff] [blame] | 376 | parser.add_argument('-r', '--run', dest='run_app_str', |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 377 | help='command to run to launch application to test,' |
| 378 | ' including command line arguments. If omitted, no' |
| 379 | ' application is launched.') |
| 380 | parser.add_argument('-p', '--port', dest='port', |
Neels Hofmeyr | 066a95d | 2017-10-18 03:53:06 +0200 | [diff] [blame] | 381 | help="Port to reach the application at.") |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 382 | parser.add_argument('-H', '--host', dest='host', default='localhost', |
Neels Hofmeyr | 066a95d | 2017-10-18 03:53:06 +0200 | [diff] [blame] | 383 | help="Host to reach the application at.") |
Neels Hofmeyr | 6562c08 | 2017-10-18 03:20:04 +0200 | [diff] [blame] | 384 | return parser |
| 385 | |
| 386 | def parser_add_verify_args(parser): |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 387 | parser.add_argument('-u', '--update', dest='update', action='store_true', |
| 388 | help='Do not verify, but OVERWRITE transcripts based on' |
Neels Hofmeyr | 066a95d | 2017-10-18 03:53:06 +0200 | [diff] [blame] | 389 | ' the application\'s current behavior. OVERWRITES TRANSCRIPT' |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 390 | ' FILES.') |
| 391 | parser.add_argument('-v', '--verbose', action='store_true', |
| 392 | help='Print commands and application output') |
Neels Hofmeyr | 6562c08 | 2017-10-18 03:20:04 +0200 | [diff] [blame] | 393 | parser.add_argument('transcript_files', nargs='*', help='transcript file(s) to verify') |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 394 | return parser |
| 395 | |
Neels Hofmeyr | 6562c08 | 2017-10-18 03:20:04 +0200 | [diff] [blame] | 396 | def parser_add_run_args(parser): |
| 397 | parser.add_argument('-O', '--output', dest='output_path', |
| 398 | help="Write command results to a file instead of stdout." |
| 399 | "('-O -' writes to stdout and is the default)") |
| 400 | parser.add_argument('-c', '--command', dest='cmd_str', |
| 401 | help="Run this command (before reading input files, if any)." |
| 402 | " multiple commands may be separated by ';'") |
| 403 | parser.add_argument('cmd_files', nargs='*', help='file(s) with plain commands to run') |
| 404 | return parser |
| 405 | |
| 406 | def main_run_commands(run_app_str, output_path, cmd_str, cmd_files, interact): |
Neels Hofmeyr | 08d645b | 2017-10-18 02:45:10 +0200 | [diff] [blame] | 407 | to_stdout = False |
Neels Hofmeyr | 6562c08 | 2017-10-18 03:20:04 +0200 | [diff] [blame] | 408 | if not output_path or output_path == '-': |
Neels Hofmeyr | 08d645b | 2017-10-18 02:45:10 +0200 | [diff] [blame] | 409 | to_stdout = True |
| 410 | output = sys.stdout |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 411 | else: |
Neels Hofmeyr | 08d645b | 2017-10-18 02:45:10 +0200 | [diff] [blame] | 412 | output = open(output_path, 'w') |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 413 | |
Neels Hofmeyr | 08d645b | 2017-10-18 02:45:10 +0200 | [diff] [blame] | 414 | application = None |
| 415 | |
| 416 | if run_app_str: |
| 417 | application = Application(run_app_str, quiet=to_stdout) |
| 418 | application.run() |
| 419 | |
| 420 | try: |
| 421 | interact.connect() |
| 422 | |
| 423 | if cmd_str: |
| 424 | interact.feed_commands(output, cmd_str.split(';')) |
| 425 | |
| 426 | for f_path in (cmd_files or []): |
| 427 | with open(f_path, 'r') as f: |
| 428 | interact.feed_commands(output, f.read().decode('utf-8').splitlines()) |
| 429 | |
| 430 | if not (cmd_str or cmd_files): |
| 431 | while True: |
| 432 | line = sys.stdin.readline() |
| 433 | if not line: |
| 434 | break; |
| 435 | interact.feed_commands(output, line.split(';')) |
| 436 | except: |
| 437 | traceback.print_exc() |
| 438 | finally: |
| 439 | if not to_stdout: |
| 440 | try: |
| 441 | output.close() |
| 442 | except: |
| 443 | traceback.print_exc() |
| 444 | |
| 445 | try: |
| 446 | interact.close() |
| 447 | except: |
| 448 | traceback.print_exc() |
| 449 | |
| 450 | if application: |
| 451 | try: |
| 452 | application.stop() |
| 453 | except: |
| 454 | traceback.print_exc() |
| 455 | |
Neels Hofmeyr | 6562c08 | 2017-10-18 03:20:04 +0200 | [diff] [blame] | 456 | def main_verify_transcripts(run_app_str, transcript_files, interact, verbose): |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 457 | results = [] |
| 458 | for t in transcript_files: |
Neels Hofmeyr | 08d645b | 2017-10-18 02:45:10 +0200 | [diff] [blame] | 459 | passed = verify_application(run_app_str=run_app_str, |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 460 | interact=interact, |
| 461 | transcript_file=t, |
| 462 | verbose=verbose) |
| 463 | results.append((passed, t)) |
| 464 | |
| 465 | print('\nRESULTS:') |
| 466 | all_passed = True |
| 467 | for passed, t in results: |
| 468 | print('%s: %s' % ('pass' if passed else 'FAIL', t)) |
| 469 | all_passed = all_passed and passed |
| 470 | print() |
| 471 | |
| 472 | if not all_passed: |
| 473 | sys.exit(1) |
| 474 | |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 475 | # vim: tabstop=4 shiftwidth=4 expandtab nocin ai |