Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | # |
| 3 | # (C) 2017 by sysmocom s.f.m.c. GmbH <info@sysmocom.de> |
| 4 | # All rights reserved. |
| 5 | # |
| 6 | # Author: Neels Hofmeyr <nhofmeyr@sysmocom.de> |
| 7 | # |
| 8 | # This program is free software: you can redistribute it and/or modify |
| 9 | # it under the terms of the GNU General Public License as published by |
| 10 | # the Free Software Foundation, either version 3 of the License, or |
| 11 | # (at your option) any later version. |
| 12 | # |
| 13 | # This program is distributed in the hope that it will be useful, |
| 14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 16 | # GNU General Public License for more details. |
| 17 | # |
| 18 | # You should have received a copy of the GNU General Public License |
| 19 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 20 | |
| 21 | ''' |
| 22 | Common code for verify_transcript_vty.py and verify_transcript_ctrl.py. |
| 23 | ''' |
| 24 | |
| 25 | import argparse |
| 26 | import sys |
| 27 | import os |
| 28 | import subprocess |
| 29 | import time |
| 30 | import traceback |
| 31 | import socket |
| 32 | import shlex |
| 33 | |
| 34 | |
| 35 | class Interact: |
| 36 | |
| 37 | class StepBase: |
| 38 | command = None |
| 39 | result = None |
| 40 | leading_blanks = None |
| 41 | |
| 42 | def __init__(self): |
| 43 | self.result = [] |
| 44 | |
| 45 | def verify_interact_state(self, interact_instance): |
| 46 | # for example to verify that the last VTY prompt received shows the |
| 47 | # right node. |
| 48 | pass |
| 49 | |
| 50 | def command_str(self, interact_instance=None): |
| 51 | return self.command |
| 52 | |
| 53 | def __str__(self): |
| 54 | return '%s\n%s' % (self.command_str(), '\n'.join(self.result)) |
| 55 | |
| 56 | @staticmethod |
| 57 | def is_next_step(line, interact_instance): |
| 58 | assert not "implemented by InteractVty.VtyStep and InteractCtrl.CtrlStep" |
| 59 | |
| 60 | socket = None |
| 61 | |
| 62 | def __init__(self, step_class, port, host, verbose=False, update=False): |
| 63 | ''' |
| 64 | host is the hostname to connect to. |
| 65 | port is the CTRL port to connect on. |
| 66 | ''' |
| 67 | self.Step = step_class |
| 68 | self.port = port |
| 69 | self.host = host |
| 70 | self.verbose = verbose |
| 71 | self.update = update |
| 72 | |
Neels Hofmeyr | 7b5203f | 2017-10-18 02:09:08 +0200 | [diff] [blame] | 73 | if not port: |
| 74 | raise Exception("You need to provide port number to connect to") |
| 75 | |
Neels Hofmeyr | 726b58d | 2017-10-15 03:01:09 +0200 | [diff] [blame] | 76 | def connect(self): |
| 77 | assert self.socket is None |
| 78 | retries = 30 |
| 79 | took = 0 |
| 80 | while True: |
| 81 | took += 1 |
| 82 | try: |
| 83 | self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) |
| 84 | self.socket.setblocking(1) |
| 85 | self.socket.connect((self.host, int(self.port))) |
| 86 | except IOError: |
| 87 | retries -= 1 |
| 88 | if retries <= 0: |
| 89 | raise |
| 90 | time.sleep(.1) |
| 91 | continue |
| 92 | break |
| 93 | |
| 94 | def close(self): |
| 95 | if self.socket is None: |
| 96 | return |
| 97 | self.socket.close() |
| 98 | self.socket = None |
| 99 | |
| 100 | def command(self, command): |
| 101 | assert not "implemented separately by InteractVty and InteractCtrl" |
| 102 | |
| 103 | def verify_transcript_file(self, transcript_file): |
| 104 | with open(transcript_file, 'r') as f: |
| 105 | content = f.read() |
| 106 | |
| 107 | try: |
| 108 | result = self.verify_transcript(content) |
| 109 | except: |
| 110 | print('Error while verifying transcript file %r' % transcript_file, file=sys.stderr) |
| 111 | sys.stderr.flush() |
| 112 | raise |
| 113 | |
| 114 | if not self.update: |
| 115 | return |
| 116 | content = '\n'.join(result) |
| 117 | with open(transcript_file, 'w') as f: |
| 118 | f.write(content) |
| 119 | |
| 120 | def verify_transcript(self, transcript): |
| 121 | '''' |
| 122 | transcript is a "screenshot" of a session, a multi-line string |
| 123 | including commands and expected results. |
| 124 | Feed commands to self.command() and verify the expected results. |
| 125 | ''' |
| 126 | |
| 127 | # parse steps |
| 128 | steps = [] |
| 129 | step = None |
| 130 | blank_lines = 0 |
| 131 | for line in transcript.splitlines(): |
| 132 | if not line: |
| 133 | blank_lines += 1 |
| 134 | continue |
| 135 | next_step_started = self.Step.is_next_step(line, self) |
| 136 | if next_step_started: |
| 137 | if step: |
| 138 | steps.append(step) |
| 139 | step = next_step_started |
| 140 | step.leading_blanks = blank_lines |
| 141 | blank_lines = 0 |
| 142 | elif step: |
| 143 | # we only count blank lines directly preceding the start of a |
| 144 | # next step. Insert blank lines in the middle of a response |
| 145 | # back into the response: |
| 146 | if blank_lines: |
| 147 | step.result.extend([''] * blank_lines) |
| 148 | blank_lines = 0 |
| 149 | step.result.append(line) |
| 150 | if step: |
| 151 | steps.append(step) |
| 152 | step = None |
| 153 | |
| 154 | actual_result = [] |
| 155 | |
| 156 | # run steps |
| 157 | step_nr = 0 |
| 158 | for step in steps: |
| 159 | step_nr += 1 |
| 160 | try: |
| 161 | if self.verbose: |
| 162 | if step.leading_blanks: |
| 163 | print('\n' * step.leading_blanks, end='') |
| 164 | print(step.command_str()) |
| 165 | sys.stdout.flush() |
| 166 | |
| 167 | step.verify_interact_state(self) |
| 168 | |
| 169 | res = self.command(step.command) |
| 170 | |
| 171 | if self.verbose: |
| 172 | sys.stderr.flush() |
| 173 | sys.stdout.flush() |
| 174 | print('\n'.join(res)) |
| 175 | sys.stdout.flush() |
| 176 | |
| 177 | if step.leading_blanks: |
| 178 | actual_result.extend([''] * step.leading_blanks) |
| 179 | actual_result.append(step.command_str(self)) |
| 180 | |
| 181 | match_result = self.match_lines(step.result, res) |
| 182 | |
| 183 | if self.update: |
| 184 | if match_result is True: |
| 185 | # preserve any wildcards |
| 186 | actual_result.extend(step.result) |
| 187 | else: |
| 188 | # mismatch, take exactly what came in |
| 189 | actual_result.extend(res) |
| 190 | continue |
| 191 | if match_result is not True: |
| 192 | raise Exception('Result mismatch:\n%s\n\nExpected:\n[\n%s\n]\n\nGot:\n[\n%s\n%s\n]' |
| 193 | % (match_result, step, step.command_str(), '\n'.join(res))) |
| 194 | except: |
| 195 | print('Error during transcript step %d:\n[\n%s\n]' % (step_nr, step), |
| 196 | file=sys.stderr) |
| 197 | sys.stderr.flush() |
| 198 | raise |
| 199 | |
| 200 | # final line ending |
| 201 | actual_result.append('') |
| 202 | return actual_result |
| 203 | |
| 204 | @staticmethod |
| 205 | def match_lines(expect, got): |
| 206 | ''' |
| 207 | Match two lists of strings, allowing certain wildcards: |
| 208 | - In 'expect', if a line is exactly '...', it matches any number of |
| 209 | arbitrary lines in 'got'; the implementation is trivial and skips |
| 210 | lines to the first occurence in 'got' that continues after '...'. |
| 211 | |
| 212 | Return 'True' on match, or a string describing the mismatch. |
| 213 | ''' |
| 214 | def match_line(expect_line, got_line): |
| 215 | return expect_line == got_line |
| 216 | |
| 217 | e = 0 |
| 218 | g = 0 |
| 219 | while e < len(expect): |
| 220 | if expect[e] == '...': |
| 221 | e += 1 |
| 222 | |
| 223 | if e >= len(expect): |
| 224 | # anything left in 'got' is accepted. |
| 225 | return True |
| 226 | |
| 227 | # look for the next occurence of the expected line in 'got' |
| 228 | while g < len(got) and not match_line(expect[e], got[g]): |
| 229 | g += 1 |
| 230 | continue |
| 231 | |
| 232 | if g >= len(got): |
| 233 | return 'Cannot find line %r' % expect[e] |
| 234 | |
| 235 | if not match_line(expect[e], got[g]): |
| 236 | return 'Mismatch:\nExpect:\n%r\nGot:\n%r' % (expect[e], got[g]) |
| 237 | |
| 238 | e += 1 |
| 239 | g += 1 |
| 240 | |
| 241 | if g < len(got): |
| 242 | return 'Did not expect line %r' % got[g] |
| 243 | return True |
| 244 | |
| 245 | def end_process(proc): |
| 246 | if not proc: |
| 247 | return |
| 248 | |
| 249 | rc = proc.poll() |
| 250 | if rc is not None: |
| 251 | print('Process has already terminated with', rc) |
| 252 | proc.wait() |
| 253 | return |
| 254 | |
| 255 | proc.terminate() |
| 256 | time_to_wait_for_term = 5 |
| 257 | wait_step = 0.001 |
| 258 | waited_time = 0 |
| 259 | while True: |
| 260 | # poll returns None if proc is still running |
| 261 | if proc.poll() is not None: |
| 262 | break |
| 263 | waited_time += wait_step |
| 264 | # make wait_step approach 1.0 |
| 265 | wait_step = (1. + 5. * wait_step) / 6. |
| 266 | if waited_time >= time_to_wait_for_term: |
| 267 | break |
| 268 | time.sleep(wait_step) |
| 269 | |
| 270 | if proc.poll() is None: |
| 271 | # termination seems to be slower than that, let's just kill |
| 272 | proc.kill() |
| 273 | print("Killed child process") |
| 274 | elif waited_time > .002: |
| 275 | print("Terminating took %.3fs" % waited_time) |
| 276 | proc.wait() |
| 277 | |
| 278 | class Application: |
| 279 | proc = None |
| 280 | _devnull = None |
| 281 | |
| 282 | @staticmethod |
| 283 | def devnull(): |
| 284 | if Application._devnull is None: |
| 285 | Application._devnull = open(os.devnull, 'w') |
| 286 | return Application._devnull |
| 287 | |
| 288 | def __init__(self, command_tuple, purge_output=True): |
| 289 | self.command_tuple = command_tuple |
| 290 | self.purge_output = purge_output |
| 291 | |
| 292 | def run(self): |
| 293 | out_err = None |
| 294 | if self.purge_output: |
| 295 | out_err = Application.devnull() |
| 296 | |
| 297 | print('Launching: cd %r; %s' % (os.getcwd(), ' '.join(self.command_tuple))) |
| 298 | self.proc = subprocess.Popen(self.command_tuple, stdout=out_err, stderr=out_err) |
| 299 | |
| 300 | def stop(self): |
| 301 | end_process(self.proc) |
| 302 | |
| 303 | def verify_application(command_tuple, interact, transcript_file, verbose): |
| 304 | passed = None |
| 305 | application = None |
| 306 | |
| 307 | sys.stdout.flush() |
| 308 | sys.stderr.flush() |
| 309 | |
| 310 | if command_tuple: |
| 311 | application = Application(command_tuple, purge_output=not verbose) |
| 312 | application.run() |
| 313 | |
| 314 | try: |
| 315 | interact.connect() |
| 316 | interact.verify_transcript_file(transcript_file) |
| 317 | passed = True |
| 318 | except: |
| 319 | traceback.print_exc() |
| 320 | passed = False |
| 321 | interact.close() |
| 322 | |
| 323 | if application: |
| 324 | application.stop() |
| 325 | |
| 326 | sys.stdout.flush() |
| 327 | sys.stderr.flush() |
| 328 | |
| 329 | return passed |
| 330 | |
| 331 | def common_parser(): |
| 332 | parser = argparse.ArgumentParser() |
| 333 | parser.add_argument('-r', '--run', dest='command_str', |
| 334 | help='command to run to launch application to test,' |
| 335 | ' including command line arguments. If omitted, no' |
| 336 | ' application is launched.') |
| 337 | parser.add_argument('-p', '--port', dest='port', |
| 338 | help="Port that the application opens.") |
| 339 | parser.add_argument('-H', '--host', dest='host', default='localhost', |
| 340 | help="Host that the application opens the port on.") |
| 341 | parser.add_argument('-u', '--update', dest='update', action='store_true', |
| 342 | help='Do not verify, but OVERWRITE transcripts based on' |
| 343 | ' the applications current behavior. OVERWRITES TRANSCRIPT' |
| 344 | ' FILES.') |
| 345 | parser.add_argument('-v', '--verbose', action='store_true', |
| 346 | help='Print commands and application output') |
| 347 | parser.add_argument('transcript_files', nargs='*', help='transcript files to verify') |
| 348 | return parser |
| 349 | |
| 350 | def main(command_str, transcript_files, interact, verbose): |
| 351 | |
| 352 | if command_str: |
| 353 | command_tuple = shlex.split(command_str) |
| 354 | else: |
| 355 | command_tuple = None |
| 356 | |
| 357 | results = [] |
| 358 | for t in transcript_files: |
| 359 | passed = verify_application(command_tuple=command_tuple, |
| 360 | interact=interact, |
| 361 | transcript_file=t, |
| 362 | verbose=verbose) |
| 363 | results.append((passed, t)) |
| 364 | |
| 365 | print('\nRESULTS:') |
| 366 | all_passed = True |
| 367 | for passed, t in results: |
| 368 | print('%s: %s' % ('pass' if passed else 'FAIL', t)) |
| 369 | all_passed = all_passed and passed |
| 370 | print() |
| 371 | |
| 372 | if not all_passed: |
| 373 | sys.exit(1) |
| 374 | |
| 375 | # vim: tabstop=4 shiftwidth=4 expandtab nocin ai |