Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 1 | # osmo_gsm_tester: trial: directory of binaries to be tested |
| 2 | # |
| 3 | # Copyright (C) 2016-2017 by sysmocom - s.f.m.c. GmbH |
| 4 | # |
| 5 | # Author: Neels Hofmeyr <neels@hofmeyr.de> |
| 6 | # |
| 7 | # This program is free software: you can redistribute it and/or modify |
Harald Welte | 2720534 | 2017-06-03 09:51:45 +0200 | [diff] [blame] | 8 | # it under the terms of the GNU General Public License as |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 9 | # published by the Free Software Foundation, either version 3 of the |
| 10 | # License, or (at your option) any later version. |
| 11 | # |
| 12 | # This program is distributed in the hope that it will be useful, |
| 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
Harald Welte | 2720534 | 2017-06-03 09:51:45 +0200 | [diff] [blame] | 15 | # GNU General Public License for more details. |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 16 | # |
Harald Welte | 2720534 | 2017-06-03 09:51:45 +0200 | [diff] [blame] | 17 | # You should have received a copy of the GNU General Public License |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 18 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 19 | |
| 20 | import os |
| 21 | import time |
| 22 | import shutil |
| 23 | import tarfile |
| 24 | |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 25 | from . import log, util, suite, report |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 26 | |
| 27 | FILE_MARK_TAKEN = 'taken' |
| 28 | FILE_CHECKSUMS = 'checksums.md5' |
| 29 | TIMESTAMP_FMT = '%Y-%m-%d_%H-%M-%S' |
| 30 | FILE_LAST_RUN = 'last_run' |
Neels Hofmeyr | fd7b9d0 | 2017-05-05 19:51:40 +0200 | [diff] [blame] | 31 | FILE_LOG = 'log' |
Neels Hofmeyr | 4f33dcc | 2017-05-07 00:01:09 +0200 | [diff] [blame] | 32 | FILE_LOG_BRIEF = 'log_brief' |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 33 | |
| 34 | class Trial(log.Origin): |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 35 | UNKNOWN = 'UNKNOWN' |
| 36 | PASS = 'PASS' |
| 37 | FAIL = 'FAIL' |
| 38 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 39 | @staticmethod |
| 40 | def next(trials_dir): |
| 41 | |
| 42 | with trials_dir.lock('Trial.next'): |
| 43 | trials = [e for e in trials_dir.children() |
| 44 | if trials_dir.isdir(e) and not trials_dir.exists(e, FILE_MARK_TAKEN)] |
| 45 | if not trials: |
| 46 | return None |
| 47 | # sort by time to get the one that waited longest |
| 48 | trials.sort(key=lambda e: os.path.getmtime(trials_dir.child(e))) |
| 49 | next_trial = trials[0] |
| 50 | return Trial(trials_dir.child(next_trial)).take() |
| 51 | |
| 52 | def __init__(self, trial_dir): |
Neels Hofmeyr | e44a0cb | 2017-05-14 03:25:45 +0200 | [diff] [blame] | 53 | self.path = os.path.abspath(trial_dir) |
Neels Hofmeyr | 1a7a3f0 | 2017-06-10 01:18:27 +0200 | [diff] [blame] | 54 | super().__init__(log.C_TST, os.path.basename(self.path)) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 55 | self.dir = util.Dir(self.path) |
| 56 | self.inst_dir = util.Dir(self.dir.child('inst')) |
| 57 | self.bin_tars = [] |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 58 | self.suites = [] |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 59 | self.status = Trial.UNKNOWN |
Pau Espin Pedrol | 5860367 | 2018-08-09 13:45:55 +0200 | [diff] [blame] | 60 | self._run_dir = None |
| 61 | self.log_targets = None |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 62 | |
| 63 | def __repr__(self): |
| 64 | return self.name() |
| 65 | |
| 66 | def __enter__(self): |
Neels Hofmeyr | 1a7a3f0 | 2017-06-10 01:18:27 +0200 | [diff] [blame] | 67 | '''add a log target to log to the run dir, write taken marker, log a |
| 68 | starting separator.''' |
Neels Hofmeyr | fd7b9d0 | 2017-05-05 19:51:40 +0200 | [diff] [blame] | 69 | run_dir = self.get_run_dir() |
Neels Hofmeyr | 39b0b89 | 2017-05-14 16:16:31 +0200 | [diff] [blame] | 70 | detailed_log = run_dir.new_child(FILE_LOG) |
Neels Hofmeyr | fd7b9d0 | 2017-05-05 19:51:40 +0200 | [diff] [blame] | 71 | self.log_targets = [ |
Neels Hofmeyr | 39b0b89 | 2017-05-14 16:16:31 +0200 | [diff] [blame] | 72 | log.FileLogTarget(detailed_log) |
Neels Hofmeyr | fd7b9d0 | 2017-05-05 19:51:40 +0200 | [diff] [blame] | 73 | .set_all_levels(log.L_DBG) |
| 74 | .style_change(trace=True), |
Neels Hofmeyr | 4f33dcc | 2017-05-07 00:01:09 +0200 | [diff] [blame] | 75 | log.FileLogTarget(run_dir.new_child(FILE_LOG_BRIEF)) |
Neels Hofmeyr | 9576f5f | 2017-05-24 18:31:01 +0200 | [diff] [blame] | 76 | .style_change(src=False, all_origins_on_levels=(log.L_ERR, log.L_TRACEBACK)) |
Neels Hofmeyr | fd7b9d0 | 2017-05-05 19:51:40 +0200 | [diff] [blame] | 77 | ] |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 78 | log.large_separator(self.name(), sublevel=1) |
Neels Hofmeyr | 39b0b89 | 2017-05-14 16:16:31 +0200 | [diff] [blame] | 79 | self.log('Detailed log at', detailed_log) |
Neels Hofmeyr | 506edbc | 2017-05-06 21:56:27 +0200 | [diff] [blame] | 80 | self.take() |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 81 | return self |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 82 | |
| 83 | def __exit__(self, *exc_info): |
Neels Hofmeyr | 1a7a3f0 | 2017-06-10 01:18:27 +0200 | [diff] [blame] | 84 | '''log a report, then remove log file targets for this trial''' |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 85 | self.log_report() |
Neels Hofmeyr | fd7b9d0 | 2017-05-05 19:51:40 +0200 | [diff] [blame] | 86 | for lt in self.log_targets: |
| 87 | lt.remove() |
| 88 | self.log_targets = None |
| 89 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 90 | def take(self): |
| 91 | self.dir.touch(FILE_MARK_TAKEN) |
| 92 | return self |
| 93 | |
| 94 | def get_run_dir(self): |
| 95 | if self._run_dir is not None: |
| 96 | return self._run_dir |
| 97 | self._run_dir = util.Dir(self.dir.new_child('run.%s' % time.strftime(TIMESTAMP_FMT))) |
| 98 | self._run_dir.mkdir() |
| 99 | |
| 100 | last_run = self.dir.child(FILE_LAST_RUN) |
| 101 | if os.path.islink(last_run): |
| 102 | os.remove(last_run) |
| 103 | if not os.path.exists(last_run): |
| 104 | os.symlink(self.dir.rel_path(self._run_dir.path), last_run) |
| 105 | return self._run_dir |
| 106 | |
| 107 | def verify(self): |
| 108 | "verify checksums" |
| 109 | |
| 110 | if not self.dir.exists(): |
| 111 | raise RuntimeError('Trial dir does not exist: %r' % self.dir) |
| 112 | if not self.dir.isdir(): |
| 113 | raise RuntimeError('Trial dir is not a dir: %r' % self.dir) |
| 114 | |
| 115 | checksums = self.dir.child(FILE_CHECKSUMS) |
| 116 | if not self.dir.isfile(FILE_CHECKSUMS): |
| 117 | raise RuntimeError('No checksums file in trial dir: %r', checksums) |
| 118 | |
| 119 | with open(checksums, 'r') as f: |
| 120 | line_nr = 0 |
| 121 | for line in [l.strip() for l in f.readlines()]: |
| 122 | line_nr += 1 |
| 123 | if not line: |
| 124 | continue |
| 125 | md5, filename = line.split(' ') |
| 126 | file_path = self.dir.child(filename) |
| 127 | |
| 128 | if not self.dir.isfile(filename): |
| 129 | raise RuntimeError('File listed in checksums file but missing in trials dir:' |
| 130 | ' %r vs. %r line %d' % (file_path, checksums, line_nr)) |
| 131 | |
| 132 | if md5 != util.md5_of_file(file_path): |
| 133 | raise RuntimeError('Checksum mismatch for %r vs. %r line %d' |
| 134 | % (file_path, checksums, line_nr)) |
| 135 | |
| 136 | if filename.endswith('.tgz'): |
| 137 | self.bin_tars.append(filename) |
| 138 | |
| 139 | def has_bin_tar(self, bin_name): |
| 140 | bin_tar_start = '%s.' % bin_name |
| 141 | matches = [t for t in self.bin_tars if t.startswith(bin_tar_start)] |
| 142 | self.dbg(bin_name=bin_name, matches=matches) |
| 143 | if not matches: |
| 144 | return None |
| 145 | if len(matches) > 1: |
| 146 | raise RuntimeError('More than one match for bin name %r: %r' % (bin_name, matches)) |
| 147 | bin_tar = matches[0] |
| 148 | bin_tar_path = self.dir.child(bin_tar) |
| 149 | if not os.path.isfile(bin_tar_path): |
| 150 | raise RuntimeError('Not a file or missing: %r' % bin_tar_path) |
| 151 | return bin_tar_path |
| 152 | |
| 153 | def get_inst(self, bin_name): |
| 154 | bin_tar = self.has_bin_tar(bin_name) |
| 155 | if not bin_tar: |
Your Name | 3c6673a | 2017-04-08 18:52:39 +0200 | [diff] [blame] | 156 | raise RuntimeError('No such binary available: %r' % bin_name) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 157 | inst_dir = self.inst_dir.child(bin_name) |
| 158 | |
| 159 | if os.path.isdir(inst_dir): |
| 160 | # already unpacked |
| 161 | return inst_dir |
| 162 | |
| 163 | t = None |
| 164 | try: |
| 165 | os.makedirs(inst_dir) |
| 166 | t = tarfile.open(bin_tar) |
| 167 | t.extractall(inst_dir) |
| 168 | return inst_dir |
| 169 | |
| 170 | except: |
| 171 | shutil.rmtree(inst_dir) |
| 172 | raise |
| 173 | finally: |
| 174 | if t: |
| 175 | try: |
| 176 | t.close() |
| 177 | except: |
| 178 | pass |
| 179 | |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 180 | def add_suite_run(self, suite_scenario_str, suite_def, scenarios): |
| 181 | suite_run = suite.SuiteRun(self, suite_scenario_str, suite_def, scenarios) |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 182 | self.suites.append(suite_run) |
| 183 | |
| 184 | def run_suites(self, names=None): |
| 185 | self.status = Trial.UNKNOWN |
Neels Hofmeyr | f8e6186 | 2017-06-06 23:08:07 +0200 | [diff] [blame] | 186 | try: |
| 187 | for suite_run in self.suites: |
| 188 | try: |
| 189 | suite_run.run_tests(names) |
| 190 | except BaseException as e: |
| 191 | # when the program is aborted by a signal (like Ctrl-C), escalate to abort all. |
| 192 | self.err('TRIAL RUN ABORTED: %s' % type(e).__name__) |
Neels Hofmeyr | 1a7a3f0 | 2017-06-10 01:18:27 +0200 | [diff] [blame] | 193 | # log the traceback before the trial's logging is ended |
| 194 | log.log_exn() |
Neels Hofmeyr | f8e6186 | 2017-06-06 23:08:07 +0200 | [diff] [blame] | 195 | raise |
| 196 | finally: |
| 197 | if suite_run.status != suite.SuiteRun.PASS: |
| 198 | self.status = Trial.FAIL |
| 199 | if self.status == Trial.UNKNOWN: |
| 200 | self.status = Trial.PASS |
| 201 | finally: |
| 202 | junit_path = self.get_run_dir().new_file(self.name()+'.xml') |
| 203 | self.log('Storing JUnit report in', junit_path) |
| 204 | report.trial_to_junit_write(self, junit_path) |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 205 | |
| 206 | def log_report(self): |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 207 | log.large_separator(self.name(), self.status) |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 208 | self.log(report.trial_to_text(self)) |
| 209 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 210 | # vim: expandtab tabstop=4 shiftwidth=4 |