Daniel Willmann | 7423728 | 2020-08-12 12:44:04 +0200 | [diff] [blame] | 1 | module StatsD_Checker { |
| 2 | |
| 3 | /* Verifies that StatsD metrics in a test match the expected values |
| 4 | * Uses StatsD_CodecPort to receive the statsd messages from the DUT |
| 5 | * and a separate VTY connection to reset and trigger the stats. |
| 6 | * |
| 7 | * When using this you should configure your stats reporter to disable |
| 8 | * interval-based reports and always send all metrics: |
| 9 | * > stats interval 0 |
| 10 | * > stats reporter statsd |
| 11 | * > remote-ip a.b.c.d |
| 12 | * > remote-port 8125 |
| 13 | * > level subscriber |
| 14 | * > flush-period 1 |
| 15 | * > mtu 1024 |
| 16 | * > enable |
| 17 | * |
| 18 | * (C) 2020 by sysmocom s.f.m.c. GmbH <info@sysmocom.de> |
| 19 | * All rights reserved. |
| 20 | * |
| 21 | * Author: Daniel Willmann <dwillmann@sysmocom.de> |
| 22 | * |
| 23 | * Released under the terms of GNU General Public License, Version 2 or |
| 24 | * (at your option) any later version. |
| 25 | * SPDX-License-Identifier: GPL-2.0-or-later |
| 26 | */ |
| 27 | |
Pau Espin Pedrol | fb14690 | 2021-02-05 16:55:18 +0100 | [diff] [blame] | 28 | import from Misc_Helpers all; |
| 29 | import from Socket_API_Definitions all; |
| 30 | |
Daniel Willmann | 7423728 | 2020-08-12 12:44:04 +0200 | [diff] [blame] | 31 | import from StatsD_Types all; |
| 32 | import from StatsD_CodecPort all; |
| 33 | import from StatsD_CodecPort_CtrlFunct all; |
| 34 | |
| 35 | import from Osmocom_Types all; |
| 36 | import from Osmocom_VTY_Functions all; |
| 37 | import from TELNETasp_PortType all; |
| 38 | |
| 39 | modulepar { |
| 40 | /* Whether to test stats values */ |
Pau Espin Pedrol | 4858184 | 2021-02-26 12:40:37 +0100 | [diff] [blame] | 41 | boolean mp_enable_stats := true; |
Daniel Willmann | 7423728 | 2020-08-12 12:44:04 +0200 | [diff] [blame] | 42 | } |
| 43 | |
| 44 | type record StatsDExpect { |
| 45 | MetricName name, |
| 46 | MetricType mtype, |
| 47 | MetricValue min, |
| 48 | MetricValue max |
| 49 | }; |
| 50 | |
| 51 | type set of StatsDExpect StatsDExpects; |
| 52 | |
| 53 | type record StatsDExpectPriv { |
| 54 | StatsDExpect expect, |
| 55 | integer seen |
| 56 | } |
| 57 | |
| 58 | type set of StatsDExpectPriv StatsDExpectPrivs; |
| 59 | |
| 60 | type enumerated StatsDResultType { |
| 61 | e_Matched, |
| 62 | e_Mismatched, |
| 63 | e_NotFound |
| 64 | } |
| 65 | |
| 66 | type record StatsDExpectResult { |
| 67 | StatsDResultType kind, |
| 68 | integer idx |
| 69 | } |
| 70 | |
| 71 | type component StatsD_Checker_CT { |
| 72 | port TELNETasp_PT STATSVTY; |
| 73 | port STATSD_PROC_PT STATSD_PROC; |
| 74 | port STATSD_CODEC_PT STATS; |
| 75 | timer T_statsd := 5.0; |
| 76 | } |
| 77 | |
| 78 | type component StatsD_ConnHdlr { |
| 79 | port STATSD_PROC_PT STATSD_PROC; |
| 80 | } |
| 81 | |
| 82 | signature STATSD_reset(); |
| 83 | signature STATSD_expect(in StatsDExpects expects) return boolean; |
| 84 | |
| 85 | type port STATSD_PROC_PT procedure { |
| 86 | inout STATSD_reset, STATSD_expect; |
| 87 | } with {extension "internal"}; |
| 88 | |
| 89 | /* Expect templates and functions */ |
| 90 | |
| 91 | |
| 92 | /* StatsD checker component */ |
| 93 | function main(charstring statsd_host, integer statsd_port) runs on StatsD_Checker_CT { |
| 94 | var StatsD_ConnHdlr vc_conn; |
| 95 | var StatsDExpects expects; |
Pau Espin Pedrol | fb14690 | 2021-02-05 16:55:18 +0100 | [diff] [blame] | 96 | var Result res; |
Daniel Willmann | 7423728 | 2020-08-12 12:44:04 +0200 | [diff] [blame] | 97 | |
| 98 | while (not mp_enable_stats) { |
| 99 | log("StatsD checker disabled by modulepar"); |
| 100 | f_sleep(3600.0); |
| 101 | } |
| 102 | |
| 103 | map(self:STATS, system:STATS); |
Pau Espin Pedrol | fb14690 | 2021-02-05 16:55:18 +0100 | [diff] [blame] | 104 | res := StatsD_CodecPort_CtrlFunct.f_IPL4_listen(STATS, statsd_host, statsd_port, { udp := {} }, {}); |
| 105 | if (not ispresent(res.connId)) { |
| 106 | Misc_Helpers.f_shutdown(__BFILE__, __LINE__, fail, |
| 107 | "Could not bind StatsD socket, check your configuration"); |
| 108 | } |
Daniel Willmann | 7423728 | 2020-08-12 12:44:04 +0200 | [diff] [blame] | 109 | |
| 110 | /* Connect to VTY and reset stats */ |
| 111 | map(self:STATSVTY, system:STATSVTY); |
| 112 | f_vty_set_prompts(STATSVTY); |
| 113 | f_vty_transceive(STATSVTY, "enable"); |
| 114 | |
| 115 | /* Reset the stats system at start */ |
| 116 | f_vty_transceive(STATSVTY, "stats reset"); |
| 117 | |
| 118 | while (true) { |
| 119 | alt { |
| 120 | [] STATSD_PROC.getcall(STATSD_reset:{}) -> sender vc_conn { |
| 121 | f_vty_transceive(STATSVTY, "stats reset"); |
| 122 | STATSD_PROC.reply(STATSD_reset:{}) to vc_conn; |
| 123 | } |
| 124 | [] STATSD_PROC.getcall(STATSD_expect:{?}) -> param(expects) sender vc_conn { |
| 125 | var boolean success := f_statsd_checker_expect(expects); |
| 126 | STATSD_PROC.reply(STATSD_expect:{expects} value success) to vc_conn; |
| 127 | } |
| 128 | } |
| 129 | } |
| 130 | } |
| 131 | |
| 132 | |
| 133 | /* Return false if the expectation doesn't match the metric, otherwise return true */ |
| 134 | private function f_compare_expect(StatsDMetric metric, StatsDExpect expect) return boolean { |
| 135 | if ((metric.name == expect.name) and (metric.mtype == expect.mtype) |
| 136 | and (metric.val >= expect.min) and (metric.val <= expect.max)) { |
| 137 | return true; |
| 138 | } else { |
| 139 | return false; |
| 140 | } |
| 141 | } |
| 142 | |
| 143 | private function f_statsd_checker_metric_expects(StatsDExpectPrivs exp_seen, StatsDMetric metric) |
| 144 | return StatsDExpectResult { |
| 145 | var StatsDExpectResult result := { |
| 146 | kind := e_NotFound, |
| 147 | idx := -1 |
| 148 | }; |
| 149 | |
| 150 | for (var integer i := 0; i < lengthof(exp_seen); i := i + 1) { |
| 151 | var StatsDExpectPriv exp := exp_seen[i]; |
| 152 | if (exp.expect.name != metric.name) { |
| 153 | continue; |
| 154 | } |
| 155 | if (not f_compare_expect(metric, exp.expect)) { |
| 156 | log("EXP mismatch: ", metric, exp.expect); |
| 157 | result := { |
| 158 | kind := e_Mismatched, |
| 159 | idx := i |
| 160 | }; |
| 161 | break; |
| 162 | } else { |
| 163 | log("EXP match: ", metric, exp.expect); |
| 164 | result := { |
| 165 | kind := e_Matched, |
| 166 | idx := i |
| 167 | }; |
| 168 | break; |
| 169 | } |
| 170 | } |
| 171 | return result; |
| 172 | } |
| 173 | |
| 174 | template StatsDExpectPriv t_statsd_expect_priv(template StatsDExpect expect) := { |
| 175 | expect := expect, |
| 176 | seen := 0 |
| 177 | } |
| 178 | |
| 179 | private function f_statsd_checker_expect(StatsDExpects expects) runs on StatsD_Checker_CT return boolean { |
| 180 | var default t; |
| 181 | var StatsDMessage msg; |
| 182 | var StatsDExpectResult res; |
| 183 | var StatsDExpectPrivs exp_seen := {}; |
| 184 | |
| 185 | for (var integer i := 0; i < lengthof(expects); i := i + 1) { |
| 186 | exp_seen := exp_seen & {valueof(t_statsd_expect_priv(expects[i]))}; |
| 187 | } |
| 188 | |
| 189 | /* Dismiss any messages we might have skipped from the last report */ |
| 190 | STATS.clear; |
| 191 | |
| 192 | f_vty_transceive(STATSVTY, "stats report"); |
| 193 | |
| 194 | var boolean seen_all := false; |
| 195 | T_statsd.start; |
| 196 | while (not seen_all) { |
| 197 | var StatsD_RecvFrom rf; |
| 198 | alt { |
| 199 | [] STATS.receive(tr_StatsD_RecvFrom(?, ?)) -> value rf { |
| 200 | msg := rf.msg; |
| 201 | } |
| 202 | [] T_statsd.timeout { |
| 203 | for (var integer i := 0; i < lengthof(exp_seen); i := i + 1) { |
| 204 | /* We're still missing some expects, keep looking */ |
| 205 | if (exp_seen[i].seen == 0) { |
| 206 | log("Timeout waiting for ", exp_seen[i].expect.name, " (min: ", exp_seen[i].expect.min, |
| 207 | ", max: ", exp_seen[i].expect.max, ")"); |
| 208 | } |
| 209 | } |
| 210 | setverdict(fail, "Timeout waiting for metrics"); |
| 211 | return false; |
| 212 | } |
| 213 | } |
| 214 | |
| 215 | for (var integer i := 0; i < lengthof(msg); i := i + 1) { |
| 216 | var StatsDMetric metric := msg[i]; |
| 217 | |
| 218 | res := f_statsd_checker_metric_expects(exp_seen, metric); |
| 219 | if (res.kind == e_NotFound) { |
| 220 | continue; |
| 221 | } |
| 222 | |
| 223 | if (res.kind == e_Mismatched) { |
| 224 | log("Metric: ", metric); |
| 225 | log("Expect: ", exp_seen[res.idx].expect); |
| 226 | setverdict(fail, "Metric failed expectation ", metric, " vs ", exp_seen[res.idx].expect); |
| 227 | return false; |
| 228 | } else if (res.kind == e_Matched) { |
| 229 | exp_seen[res.idx].seen := exp_seen[res.idx].seen + 1; |
| 230 | } |
| 231 | } |
| 232 | |
| 233 | /* Check if all expected metrics were received */ |
| 234 | seen_all := true; |
| 235 | for (var integer i := 0; i < lengthof(exp_seen); i := i + 1) { |
| 236 | /* We're still missing some expects, keep looking */ |
| 237 | if (exp_seen[i].seen == 0) { |
| 238 | seen_all := false; |
| 239 | break; |
| 240 | } |
| 241 | } |
| 242 | } |
| 243 | |
| 244 | T_statsd.stop; |
| 245 | return seen_all; |
| 246 | } |
| 247 | |
| 248 | function f_init_statsd(charstring id, inout StatsD_Checker_CT vc_STATSD, charstring dst_addr, integer dst_port) { |
| 249 | id := id & "-STATS"; |
| 250 | |
| 251 | vc_STATSD := StatsD_Checker_CT.create(id); |
| 252 | vc_STATSD.start(StatsD_Checker.main(dst_addr, dst_port)); |
| 253 | } |
| 254 | |
| 255 | |
| 256 | /* StatsD connhdlr */ |
| 257 | function f_statsd_reset() runs on StatsD_ConnHdlr { |
| 258 | if (not mp_enable_stats) { |
| 259 | return; |
| 260 | } |
| 261 | |
| 262 | STATSD_PROC.call(STATSD_reset:{}) { |
| 263 | [] STATSD_PROC.getreply(STATSD_reset:{}) {} |
| 264 | } |
| 265 | } |
| 266 | |
| 267 | function f_statsd_expect(StatsDExpects expects) runs on StatsD_ConnHdlr return boolean { |
| 268 | var boolean res; |
| 269 | |
| 270 | if (not mp_enable_stats) { |
| 271 | return true; |
| 272 | } |
| 273 | |
| 274 | STATSD_PROC.call(STATSD_expect:{expects}) { |
| 275 | [] STATSD_PROC.getreply(STATSD_expect:{expects}) -> value res; |
| 276 | } |
| 277 | return res; |
| 278 | } |
| 279 | |
| 280 | } |