Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 1 | # osmo_gsm_tester: trial: directory of binaries to be tested |
| 2 | # |
| 3 | # Copyright (C) 2016-2017 by sysmocom - s.f.m.c. GmbH |
| 4 | # |
| 5 | # Author: Neels Hofmeyr <neels@hofmeyr.de> |
| 6 | # |
| 7 | # This program is free software: you can redistribute it and/or modify |
Harald Welte | 2720534 | 2017-06-03 09:51:45 +0200 | [diff] [blame] | 8 | # it under the terms of the GNU General Public License as |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 9 | # published by the Free Software Foundation, either version 3 of the |
| 10 | # License, or (at your option) any later version. |
| 11 | # |
| 12 | # This program is distributed in the hope that it will be useful, |
| 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
Harald Welte | 2720534 | 2017-06-03 09:51:45 +0200 | [diff] [blame] | 15 | # GNU General Public License for more details. |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 16 | # |
Harald Welte | 2720534 | 2017-06-03 09:51:45 +0200 | [diff] [blame] | 17 | # You should have received a copy of the GNU General Public License |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 18 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 19 | |
| 20 | import os |
| 21 | import time |
| 22 | import shutil |
| 23 | import tarfile |
Pau Espin Pedrol | ef919c0 | 2020-06-05 13:19:02 +0200 | [diff] [blame] | 24 | import pathlib |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 25 | |
Pau Espin Pedrol | f574a46 | 2020-05-05 12:18:35 +0200 | [diff] [blame] | 26 | from . import log |
| 27 | from . import util |
| 28 | from . import report |
Pau Espin Pedrol | ee217b0 | 2020-05-04 19:06:47 +0200 | [diff] [blame] | 29 | from . import suite |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 30 | |
| 31 | FILE_MARK_TAKEN = 'taken' |
| 32 | FILE_CHECKSUMS = 'checksums.md5' |
| 33 | TIMESTAMP_FMT = '%Y-%m-%d_%H-%M-%S' |
| 34 | FILE_LAST_RUN = 'last_run' |
| 35 | |
| 36 | class Trial(log.Origin): |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 37 | UNKNOWN = 'UNKNOWN' |
| 38 | PASS = 'PASS' |
| 39 | FAIL = 'FAIL' |
| 40 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 41 | @staticmethod |
| 42 | def next(trials_dir): |
| 43 | |
| 44 | with trials_dir.lock('Trial.next'): |
| 45 | trials = [e for e in trials_dir.children() |
| 46 | if trials_dir.isdir(e) and not trials_dir.exists(e, FILE_MARK_TAKEN)] |
| 47 | if not trials: |
| 48 | return None |
| 49 | # sort by time to get the one that waited longest |
| 50 | trials.sort(key=lambda e: os.path.getmtime(trials_dir.child(e))) |
| 51 | next_trial = trials[0] |
| 52 | return Trial(trials_dir.child(next_trial)).take() |
| 53 | |
| 54 | def __init__(self, trial_dir): |
Neels Hofmeyr | e44a0cb | 2017-05-14 03:25:45 +0200 | [diff] [blame] | 55 | self.path = os.path.abspath(trial_dir) |
Neels Hofmeyr | 1a7a3f0 | 2017-06-10 01:18:27 +0200 | [diff] [blame] | 56 | super().__init__(log.C_TST, os.path.basename(self.path)) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 57 | self.dir = util.Dir(self.path) |
| 58 | self.inst_dir = util.Dir(self.dir.child('inst')) |
Pau Espin Pedrol | 6e0b6fb | 2020-05-25 19:49:29 +0200 | [diff] [blame] | 59 | self.bin_tars = {} |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 60 | self.suites = [] |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 61 | self.status = Trial.UNKNOWN |
Pau Espin Pedrol | 5860367 | 2018-08-09 13:45:55 +0200 | [diff] [blame] | 62 | self._run_dir = None |
| 63 | self.log_targets = None |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 64 | |
| 65 | def __repr__(self): |
| 66 | return self.name() |
| 67 | |
| 68 | def __enter__(self): |
Neels Hofmeyr | 1a7a3f0 | 2017-06-10 01:18:27 +0200 | [diff] [blame] | 69 | '''add a log target to log to the run dir, write taken marker, log a |
| 70 | starting separator.''' |
Neels Hofmeyr | fd7b9d0 | 2017-05-05 19:51:40 +0200 | [diff] [blame] | 71 | run_dir = self.get_run_dir() |
Pau Espin Pedrol | ec28572 | 2020-06-11 17:57:43 +0200 | [diff] [blame] | 72 | detailed_log = run_dir.new_child(log.FILE_LOG) |
Neels Hofmeyr | fd7b9d0 | 2017-05-05 19:51:40 +0200 | [diff] [blame] | 73 | self.log_targets = [ |
Neels Hofmeyr | 39b0b89 | 2017-05-14 16:16:31 +0200 | [diff] [blame] | 74 | log.FileLogTarget(detailed_log) |
Neels Hofmeyr | fd7b9d0 | 2017-05-05 19:51:40 +0200 | [diff] [blame] | 75 | .set_all_levels(log.L_DBG) |
| 76 | .style_change(trace=True), |
Pau Espin Pedrol | ec28572 | 2020-06-11 17:57:43 +0200 | [diff] [blame] | 77 | log.FileLogTarget(run_dir.new_child(log.FILE_LOG_BRIEF)) |
Neels Hofmeyr | 9576f5f | 2017-05-24 18:31:01 +0200 | [diff] [blame] | 78 | .style_change(src=False, all_origins_on_levels=(log.L_ERR, log.L_TRACEBACK)) |
Neels Hofmeyr | fd7b9d0 | 2017-05-05 19:51:40 +0200 | [diff] [blame] | 79 | ] |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 80 | log.large_separator(self.name(), sublevel=1) |
Neels Hofmeyr | 39b0b89 | 2017-05-14 16:16:31 +0200 | [diff] [blame] | 81 | self.log('Detailed log at', detailed_log) |
Neels Hofmeyr | 506edbc | 2017-05-06 21:56:27 +0200 | [diff] [blame] | 82 | self.take() |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 83 | return self |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 84 | |
| 85 | def __exit__(self, *exc_info): |
Neels Hofmeyr | 1a7a3f0 | 2017-06-10 01:18:27 +0200 | [diff] [blame] | 86 | '''log a report, then remove log file targets for this trial''' |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 87 | self.log_report() |
Neels Hofmeyr | fd7b9d0 | 2017-05-05 19:51:40 +0200 | [diff] [blame] | 88 | for lt in self.log_targets: |
| 89 | lt.remove() |
| 90 | self.log_targets = None |
| 91 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 92 | def take(self): |
| 93 | self.dir.touch(FILE_MARK_TAKEN) |
| 94 | return self |
| 95 | |
| 96 | def get_run_dir(self): |
| 97 | if self._run_dir is not None: |
| 98 | return self._run_dir |
| 99 | self._run_dir = util.Dir(self.dir.new_child('run.%s' % time.strftime(TIMESTAMP_FMT))) |
| 100 | self._run_dir.mkdir() |
| 101 | |
| 102 | last_run = self.dir.child(FILE_LAST_RUN) |
| 103 | if os.path.islink(last_run): |
| 104 | os.remove(last_run) |
| 105 | if not os.path.exists(last_run): |
| 106 | os.symlink(self.dir.rel_path(self._run_dir.path), last_run) |
| 107 | return self._run_dir |
| 108 | |
| 109 | def verify(self): |
| 110 | "verify checksums" |
| 111 | |
| 112 | if not self.dir.exists(): |
| 113 | raise RuntimeError('Trial dir does not exist: %r' % self.dir) |
| 114 | if not self.dir.isdir(): |
| 115 | raise RuntimeError('Trial dir is not a dir: %r' % self.dir) |
| 116 | |
| 117 | checksums = self.dir.child(FILE_CHECKSUMS) |
| 118 | if not self.dir.isfile(FILE_CHECKSUMS): |
| 119 | raise RuntimeError('No checksums file in trial dir: %r', checksums) |
| 120 | |
| 121 | with open(checksums, 'r') as f: |
| 122 | line_nr = 0 |
| 123 | for line in [l.strip() for l in f.readlines()]: |
| 124 | line_nr += 1 |
| 125 | if not line: |
| 126 | continue |
Pau Espin Pedrol | 6e0b6fb | 2020-05-25 19:49:29 +0200 | [diff] [blame] | 127 | md5, relpath = line.split(' ') |
| 128 | file_path = self.dir.child(relpath) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 129 | |
Pau Espin Pedrol | 6e0b6fb | 2020-05-25 19:49:29 +0200 | [diff] [blame] | 130 | if not self.dir.isfile(relpath): |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 131 | raise RuntimeError('File listed in checksums file but missing in trials dir:' |
| 132 | ' %r vs. %r line %d' % (file_path, checksums, line_nr)) |
| 133 | |
| 134 | if md5 != util.md5_of_file(file_path): |
| 135 | raise RuntimeError('Checksum mismatch for %r vs. %r line %d' |
| 136 | % (file_path, checksums, line_nr)) |
| 137 | |
Pau Espin Pedrol | 6e0b6fb | 2020-05-25 19:49:29 +0200 | [diff] [blame] | 138 | if relpath.endswith('.tgz') or relpath.endswith('.tar.gz'): |
| 139 | (label, name) = os.path.split(relpath) |
| 140 | #print('label: %s, name: %s' % (label, name)) |
| 141 | li = self.bin_tars.get(label, []) |
| 142 | li.append(name) |
| 143 | self.bin_tars[label] = li |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 144 | |
Pau Espin Pedrol | 6e0b6fb | 2020-05-25 19:49:29 +0200 | [diff] [blame] | 145 | def has_bin_tar(self, bin_name, run_label): |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 146 | bin_tar_start = '%s.' % bin_name |
Pau Espin Pedrol | 6e0b6fb | 2020-05-25 19:49:29 +0200 | [diff] [blame] | 147 | matches = [t for t in self.bin_tars[run_label] if t.startswith(bin_tar_start)] |
| 148 | self.dbg('has bin_tar', run_label=run_label, bin_name=bin_name, matches=matches) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 149 | if not matches: |
| 150 | return None |
| 151 | if len(matches) > 1: |
Pau Espin Pedrol | 6e0b6fb | 2020-05-25 19:49:29 +0200 | [diff] [blame] | 152 | raise RuntimeError('More than one match for bin name %r on run_label \'%s\': %r' % (bin_name, run_label, matches)) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 153 | bin_tar = matches[0] |
Pau Espin Pedrol | 6e0b6fb | 2020-05-25 19:49:29 +0200 | [diff] [blame] | 154 | bin_tar_path = self.dir.child(os.path.join(run_label, bin_tar)) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 155 | if not os.path.isfile(bin_tar_path): |
| 156 | raise RuntimeError('Not a file or missing: %r' % bin_tar_path) |
| 157 | return bin_tar_path |
| 158 | |
Pau Espin Pedrol | 6e0b6fb | 2020-05-25 19:49:29 +0200 | [diff] [blame] | 159 | def get_inst(self, bin_name, run_label=None): |
| 160 | if run_label is None: |
| 161 | run_label = '' |
| 162 | bin_tar = self.has_bin_tar(bin_name, run_label) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 163 | if not bin_tar: |
Your Name | 3c6673a | 2017-04-08 18:52:39 +0200 | [diff] [blame] | 164 | raise RuntimeError('No such binary available: %r' % bin_name) |
Pau Espin Pedrol | 6e0b6fb | 2020-05-25 19:49:29 +0200 | [diff] [blame] | 165 | inst_dir = self.inst_dir.child(os.path.join(run_label, bin_name)) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 166 | |
| 167 | if os.path.isdir(inst_dir): |
| 168 | # already unpacked |
| 169 | return inst_dir |
| 170 | |
| 171 | t = None |
| 172 | try: |
| 173 | os.makedirs(inst_dir) |
| 174 | t = tarfile.open(bin_tar) |
| 175 | t.extractall(inst_dir) |
| 176 | return inst_dir |
| 177 | |
| 178 | except: |
| 179 | shutil.rmtree(inst_dir) |
| 180 | raise |
| 181 | finally: |
| 182 | if t: |
| 183 | try: |
| 184 | t.close() |
| 185 | except: |
| 186 | pass |
| 187 | |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 188 | def add_suite_run(self, suite_scenario_str, suite_def, scenarios): |
| 189 | suite_run = suite.SuiteRun(self, suite_scenario_str, suite_def, scenarios) |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 190 | self.suites.append(suite_run) |
| 191 | |
| 192 | def run_suites(self, names=None): |
| 193 | self.status = Trial.UNKNOWN |
Neels Hofmeyr | f8e6186 | 2017-06-06 23:08:07 +0200 | [diff] [blame] | 194 | try: |
| 195 | for suite_run in self.suites: |
| 196 | try: |
| 197 | suite_run.run_tests(names) |
| 198 | except BaseException as e: |
| 199 | # when the program is aborted by a signal (like Ctrl-C), escalate to abort all. |
| 200 | self.err('TRIAL RUN ABORTED: %s' % type(e).__name__) |
Neels Hofmeyr | 1a7a3f0 | 2017-06-10 01:18:27 +0200 | [diff] [blame] | 201 | # log the traceback before the trial's logging is ended |
| 202 | log.log_exn() |
Neels Hofmeyr | f8e6186 | 2017-06-06 23:08:07 +0200 | [diff] [blame] | 203 | raise |
| 204 | finally: |
| 205 | if suite_run.status != suite.SuiteRun.PASS: |
| 206 | self.status = Trial.FAIL |
| 207 | if self.status == Trial.UNKNOWN: |
| 208 | self.status = Trial.PASS |
| 209 | finally: |
| 210 | junit_path = self.get_run_dir().new_file(self.name()+'.xml') |
| 211 | self.log('Storing JUnit report in', junit_path) |
| 212 | report.trial_to_junit_write(self, junit_path) |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 213 | |
Pau Espin Pedrol | ef919c0 | 2020-06-05 13:19:02 +0200 | [diff] [blame] | 214 | def get_all_inst_hash_info(self): |
| 215 | d = {} |
| 216 | pathlist = pathlib.Path(str(self.inst_dir)).glob('**/*git_hashes.txt') |
| 217 | for path in pathlist: |
| 218 | # because path is object not string |
| 219 | abs_path_str = str(path) # because path is object not string |
| 220 | dir, file = os.path.split(abs_path_str) |
| 221 | reldir = os.path.relpath(dir, str(self.inst_dir)).rstrip(os.sep) |
| 222 | with open(abs_path_str, 'r') as f: |
| 223 | for line in [l.strip() for l in f.readlines()]: |
| 224 | if not line: |
| 225 | continue |
| 226 | hash, proj = tuple(line.split(' ', 1)) |
| 227 | d[os.path.join(reldir,proj)] = hash |
| 228 | return d |
| 229 | |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 230 | def log_report(self): |
Neels Hofmeyr | 6ccda11 | 2017-06-06 19:41:17 +0200 | [diff] [blame] | 231 | log.large_separator(self.name(), self.status) |
Pau Espin Pedrol | 0ffb414 | 2017-05-15 18:24:35 +0200 | [diff] [blame] | 232 | self.log(report.trial_to_text(self)) |
| 233 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 234 | # vim: expandtab tabstop=4 shiftwidth=4 |