Philipp Maier | b1ef8f5 | 2021-12-06 16:31:02 +0100 | [diff] [blame] | 1 | /* |
| 2 | * (C) 2021 by sysmocom - s.f.m.c. GmbH |
| 3 | * Author: Philipp Maier <pmaier@sysmocom.de> |
| 4 | * All Rights Reserved |
| 5 | * |
| 6 | * SPDX-License-Identifier: GPL-2.0+ |
| 7 | * |
| 8 | * This program is free software; you can redistribute it and/or modify |
| 9 | * it under the terms of the GNU General Public License as published by |
| 10 | * the Free Software Foundation; either version 2 of the License, or |
| 11 | * (at your option) any later version. |
| 12 | * |
| 13 | * This program is distributed in the hope that it will be useful, |
| 14 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 16 | * GNU General Public License for more details. |
| 17 | * |
| 18 | */ |
| 19 | |
| 20 | /*! \addtogroup stats |
| 21 | * @{ |
| 22 | * \file stats_tcp.c */ |
| 23 | |
| 24 | #include "config.h" |
| 25 | #if !defined(EMBEDDED) |
| 26 | |
| 27 | #include <sys/types.h> |
| 28 | #include <sys/stat.h> |
| 29 | #include <sys/socket.h> |
| 30 | #include <netinet/in.h> |
| 31 | #include <netinet/ip.h> |
| 32 | #include <linux/tcp.h> |
| 33 | #include <errno.h> |
| 34 | #include <pthread.h> |
| 35 | |
| 36 | #include <osmocom/core/select.h> |
| 37 | #include <osmocom/core/linuxlist.h> |
| 38 | #include <osmocom/core/talloc.h> |
| 39 | #include <osmocom/core/utils.h> |
| 40 | #include <osmocom/core/timer.h> |
| 41 | #include <osmocom/core/stat_item.h> |
| 42 | #include <osmocom/core/stats.h> |
| 43 | #include <osmocom/core/socket.h> |
| 44 | #include <osmocom/core/stats_tcp.h> |
| 45 | |
| 46 | static struct osmo_tcp_stats_config s_tcp_stats_config = { |
| 47 | .interval = TCP_STATS_DEFAULT_INTERVAL, |
| 48 | }; |
| 49 | |
| 50 | struct osmo_tcp_stats_config *osmo_tcp_stats_config = &s_tcp_stats_config; |
| 51 | |
| 52 | static struct osmo_timer_list stats_tcp_poll_timer; |
| 53 | |
| 54 | static LLIST_HEAD(stats_tcp); |
| 55 | static struct stats_tcp_entry *stats_tcp_entry_cur; |
| 56 | pthread_mutex_t stats_tcp_lock; |
| 57 | |
| 58 | struct stats_tcp_entry { |
| 59 | struct llist_head entry; |
| 60 | const struct osmo_fd *fd; |
| 61 | struct osmo_stat_item_group *stats_tcp; |
| 62 | const char *name; |
| 63 | }; |
| 64 | |
| 65 | enum { |
| 66 | STATS_TCP_UNACKED, |
| 67 | STATS_TCP_LOST, |
| 68 | STATS_TCP_RETRANS, |
| 69 | STATS_TCP_RTT, |
| 70 | STATS_TCP_RCV_RTT, |
| 71 | STATS_TCP_NOTSENT_BYTES, |
| 72 | STATS_TCP_RWND_LIMITED, |
| 73 | STATS_TCP_SNDBUF_LIMITED, |
| 74 | STATS_TCP_REORD_SEEN, |
| 75 | }; |
| 76 | |
| 77 | static struct osmo_stat_item_desc stats_tcp_item_desc[] = { |
| 78 | [STATS_TCP_UNACKED] = { "tcp:unacked", "unacknowledged packets", "", 60, 0 }, |
| 79 | [STATS_TCP_LOST] = { "tcp:lost", "lost packets", "", 60, 0 }, |
| 80 | [STATS_TCP_RETRANS] = { "tcp:retrans", "retransmitted packets", "", 60, 0 }, |
| 81 | [STATS_TCP_RTT] = { "tcp:rtt", "roundtrip-time", "", 60, 0 }, |
| 82 | [STATS_TCP_RCV_RTT] = { "tcp:rcv_rtt", "roundtrip-time (receive)", "", 60, 0 }, |
| 83 | [STATS_TCP_NOTSENT_BYTES] = { "tcp:notsent_bytes", "bytes not yet sent", "", 60, 0 }, |
| 84 | [STATS_TCP_RWND_LIMITED] = { "tcp:rwnd_limited", "time (usec) limited by receive window", "", 60, 0 }, |
| 85 | [STATS_TCP_SNDBUF_LIMITED] = { "tcp:sndbuf_limited", "Time (usec) limited by send buffer", "", 60, 0 }, |
Philipp Maier | 42dcbf0 | 2022-01-07 12:00:57 +0100 | [diff] [blame] | 86 | [STATS_TCP_REORD_SEEN] = { "tcp:reord_seen", "reordering events seen", "", 60, 0 }, |
Philipp Maier | b1ef8f5 | 2021-12-06 16:31:02 +0100 | [diff] [blame] | 87 | }; |
| 88 | |
| 89 | static struct osmo_stat_item_group_desc stats_tcp_desc = { |
| 90 | .group_name_prefix = "tcp", |
| 91 | .group_description = "stats tcp", |
| 92 | .class_id = OSMO_STATS_CLASS_GLOBAL, |
| 93 | .num_items = ARRAY_SIZE(stats_tcp_item_desc), |
| 94 | .item_desc = stats_tcp_item_desc, |
| 95 | }; |
| 96 | |
| 97 | static void fill_stats(struct stats_tcp_entry *stats_tcp_entry) |
| 98 | { |
| 99 | int rc; |
| 100 | struct tcp_info tcp_info; |
| 101 | socklen_t tcp_info_len = sizeof(tcp_info); |
| 102 | char stat_name[256]; |
| 103 | |
| 104 | /* Do not fill in anything before the socket is connected to a remote end */ |
| 105 | if (osmo_sock_get_ip_and_port(stats_tcp_entry->fd->fd, NULL, 0, NULL, 0, false) != 0) |
| 106 | return; |
| 107 | |
| 108 | /* Gather TCP statistics and update the stats items */ |
| 109 | rc = getsockopt(stats_tcp_entry->fd->fd, IPPROTO_TCP, TCP_INFO, &tcp_info, &tcp_info_len); |
| 110 | if (rc < 0) |
| 111 | return; |
| 112 | |
| 113 | /* Create stats items if they do not exist yet */ |
| 114 | if (!stats_tcp_entry->stats_tcp) { |
| 115 | stats_tcp_entry->stats_tcp = |
| 116 | osmo_stat_item_group_alloc(stats_tcp_entry, &stats_tcp_desc, stats_tcp_entry->fd->fd); |
| 117 | OSMO_ASSERT(stats_tcp_entry->stats_tcp); |
| 118 | } |
| 119 | |
| 120 | /* Update statistics */ |
| 121 | if (stats_tcp_entry->name) |
Michael Iedema | a6d1ef0 | 2022-02-10 09:30:26 -0800 | [diff] [blame] | 122 | snprintf(stat_name, sizeof(stat_name), "%s", stats_tcp_entry->name); |
Philipp Maier | b1ef8f5 | 2021-12-06 16:31:02 +0100 | [diff] [blame] | 123 | else |
| 124 | snprintf(stat_name, sizeof(stat_name), "%s", osmo_sock_get_name2(stats_tcp_entry->fd->fd)); |
| 125 | osmo_stat_item_group_set_name(stats_tcp_entry->stats_tcp, stat_name); |
| 126 | |
| 127 | osmo_stat_item_set(osmo_stat_item_group_get_item(stats_tcp_entry->stats_tcp, STATS_TCP_UNACKED), |
| 128 | tcp_info.tcpi_unacked); |
| 129 | osmo_stat_item_set(osmo_stat_item_group_get_item(stats_tcp_entry->stats_tcp, STATS_TCP_LOST), |
| 130 | tcp_info.tcpi_lost); |
| 131 | osmo_stat_item_set(osmo_stat_item_group_get_item(stats_tcp_entry->stats_tcp, STATS_TCP_RETRANS), |
| 132 | tcp_info.tcpi_retrans); |
| 133 | osmo_stat_item_set(osmo_stat_item_group_get_item(stats_tcp_entry->stats_tcp, STATS_TCP_RTT), tcp_info.tcpi_rtt); |
| 134 | osmo_stat_item_set(osmo_stat_item_group_get_item(stats_tcp_entry->stats_tcp, STATS_TCP_RCV_RTT), |
| 135 | tcp_info.tcpi_rcv_rtt); |
Harald Welte | c809f4e | 2021-12-24 11:31:12 +0100 | [diff] [blame] | 136 | #if HAVE_TCP_INFO_TCPI_NOTSENT_BYTES == 1 |
Philipp Maier | b1ef8f5 | 2021-12-06 16:31:02 +0100 | [diff] [blame] | 137 | osmo_stat_item_set(osmo_stat_item_group_get_item(stats_tcp_entry->stats_tcp, STATS_TCP_NOTSENT_BYTES), |
| 138 | tcp_info.tcpi_notsent_bytes); |
Harald Welte | c809f4e | 2021-12-24 11:31:12 +0100 | [diff] [blame] | 139 | #else |
| 140 | osmo_stat_item_set(osmo_stat_item_group_get_item(stats_tcp_entry->stats_tcp, STATS_TCP_NOTSENT_BYTES), -1); |
| 141 | #endif |
Philipp Maier | b1ef8f5 | 2021-12-06 16:31:02 +0100 | [diff] [blame] | 142 | |
| 143 | #if HAVE_TCP_INFO_TCPI_RWND_LIMITED == 1 |
| 144 | osmo_stat_item_set(osmo_stat_item_group_get_item(stats_tcp_entry->stats_tcp, STATS_TCP_RWND_LIMITED), |
| 145 | tcp_info.tcpi_rwnd_limited); |
| 146 | #else |
| 147 | osmo_stat_item_set(osmo_stat_item_group_get_item(stats_tcp_entry->stats_tcp, STATS_TCP_RWND_LIMITED), -1); |
| 148 | #endif |
| 149 | |
| 150 | #if STATS_TCP_SNDBUF_LIMITED == 1 |
| 151 | osmo_stat_item_set(osmo_stat_item_group_get_item(stats_tcp_entry->stats_tcp, STATS_TCP_REORD_SEEN), |
| 152 | tcp_info.tcpi_sndbuf_limited); |
| 153 | #else |
| 154 | osmo_stat_item_set(osmo_stat_item_group_get_item(stats_tcp_entry->stats_tcp, STATS_TCP_REORD_SEEN), -1); |
| 155 | #endif |
| 156 | |
| 157 | #if HAVE_TCP_INFO_TCPI_REORD_SEEN == 1 |
| 158 | osmo_stat_item_set(osmo_stat_item_group_get_item(stats_tcp_entry->stats_tcp, STATS_TCP_REORD_SEEN), |
| 159 | tcp_info.tcpi_reord_seen); |
| 160 | #else |
| 161 | osmo_stat_item_set(osmo_stat_item_group_get_item(stats_tcp_entry->stats_tcp, STATS_TCP_REORD_SEEN), -1); |
| 162 | #endif |
| 163 | |
| 164 | } |
| 165 | |
| 166 | static bool is_tcp(const struct osmo_fd *fd) |
| 167 | { |
| 168 | int rc; |
| 169 | struct stat fd_stat; |
| 170 | int so_protocol = 0; |
| 171 | socklen_t so_protocol_len = sizeof(so_protocol); |
| 172 | |
| 173 | /* Is this a socket? */ |
| 174 | rc = fstat(fd->fd, &fd_stat); |
| 175 | if (rc < 0) |
| 176 | return false; |
| 177 | if (!S_ISSOCK(fd_stat.st_mode)) |
| 178 | return false; |
| 179 | |
| 180 | /* Is it a TCP socket? */ |
| 181 | rc = getsockopt(fd->fd, SOL_SOCKET, SO_PROTOCOL, &so_protocol, &so_protocol_len); |
| 182 | if (rc < 0) |
| 183 | return false; |
| 184 | if (so_protocol == IPPROTO_TCP) |
| 185 | return true; |
| 186 | |
| 187 | return false; |
| 188 | } |
| 189 | |
| 190 | /*! Register an osmo_fd for TCP stats monitoring. |
| 191 | * \param[in] fd osmocom file descriptor to be registered. |
| 192 | * \param[in] human readbla name that is used as prefix for the related stats item. |
| 193 | * \returns 0 on success; negative in case of error. */ |
| 194 | int osmo_stats_tcp_osmo_fd_register(const struct osmo_fd *fd, const char *name) |
| 195 | { |
| 196 | struct stats_tcp_entry *stats_tcp_entry; |
| 197 | |
| 198 | /* Only TCP sockets can be registered for monitoring, anything else will fall through. */ |
| 199 | if (!is_tcp(fd)) |
| 200 | return -EINVAL; |
| 201 | |
| 202 | /* When the osmo_fd is registered and unregistered properly there shouldn't be any leftovers from already closed |
| 203 | * osmo_fds in the stats_tcp list. But lets proactively make sure that any leftovers are cleaned up. */ |
| 204 | osmo_stats_tcp_osmo_fd_unregister(fd); |
| 205 | |
| 206 | /* Make a new list object, attach the osmo_fd... */ |
| 207 | stats_tcp_entry = talloc_zero(OTC_GLOBAL, struct stats_tcp_entry); |
| 208 | OSMO_ASSERT(stats_tcp_entry); |
| 209 | stats_tcp_entry->fd = fd; |
| 210 | stats_tcp_entry->name = talloc_strdup(stats_tcp_entry, name); |
| 211 | |
| 212 | pthread_mutex_lock(&stats_tcp_lock); |
| 213 | llist_add_tail(&stats_tcp_entry->entry, &stats_tcp); |
| 214 | pthread_mutex_unlock(&stats_tcp_lock); |
| 215 | |
| 216 | return 0; |
| 217 | } |
| 218 | |
| 219 | static void next_stats_tcp_entry(void) |
| 220 | { |
| 221 | struct stats_tcp_entry *last; |
| 222 | |
| 223 | if (llist_empty(&stats_tcp)) { |
| 224 | stats_tcp_entry_cur = NULL; |
| 225 | return; |
| 226 | } |
| 227 | |
| 228 | last = (struct stats_tcp_entry *)llist_last_entry(&stats_tcp, struct stats_tcp_entry, entry); |
| 229 | |
| 230 | if (!stats_tcp_entry_cur || stats_tcp_entry_cur == last) |
| 231 | stats_tcp_entry_cur = |
| 232 | (struct stats_tcp_entry *)llist_first_entry(&stats_tcp, struct stats_tcp_entry, entry); |
| 233 | else |
| 234 | stats_tcp_entry_cur = |
| 235 | (struct stats_tcp_entry *)llist_entry(stats_tcp_entry_cur->entry.next, struct stats_tcp_entry, |
| 236 | entry); |
| 237 | } |
| 238 | |
| 239 | /*! Register an osmo_fd for TCP stats monitoring. |
| 240 | * \param[in] fd osmocom file descriptor to be unregistered. |
| 241 | * \returns 0 on success; negative in case of error. */ |
| 242 | int osmo_stats_tcp_osmo_fd_unregister(const struct osmo_fd *fd) |
| 243 | { |
| 244 | struct stats_tcp_entry *stats_tcp_entry; |
| 245 | int rc = -EINVAL; |
| 246 | |
| 247 | pthread_mutex_lock(&stats_tcp_lock); |
| 248 | llist_for_each_entry(stats_tcp_entry, &stats_tcp, entry) { |
| 249 | if (fd->fd == stats_tcp_entry->fd->fd) { |
Pau Espin Pedrol | 8d1532a | 2023-03-09 18:38:56 +0100 | [diff] [blame] | 250 | /* In case we want to remove exactly that item which is also |
| 251 | * selected as the current item, we must designate either a |
| 252 | * different item or invalidate the current item. |
| 253 | */ |
Philipp Maier | b1ef8f5 | 2021-12-06 16:31:02 +0100 | [diff] [blame] | 254 | if (stats_tcp_entry == stats_tcp_entry_cur) { |
| 255 | if (llist_count(&stats_tcp) > 2) |
| 256 | next_stats_tcp_entry(); |
| 257 | else |
| 258 | stats_tcp_entry_cur = NULL; |
| 259 | } |
| 260 | |
| 261 | /* Date item from list */ |
| 262 | llist_del(&stats_tcp_entry->entry); |
| 263 | osmo_stat_item_group_free(stats_tcp_entry->stats_tcp); |
| 264 | talloc_free(stats_tcp_entry); |
| 265 | rc = 0; |
| 266 | break; |
| 267 | } |
| 268 | } |
| 269 | pthread_mutex_unlock(&stats_tcp_lock); |
| 270 | |
| 271 | return rc; |
| 272 | } |
| 273 | |
| 274 | static void stats_tcp_poll_timer_cb(void *data) |
| 275 | { |
| 276 | int i; |
| 277 | int batch_size; |
| 278 | int llist_size; |
| 279 | |
| 280 | pthread_mutex_lock(&stats_tcp_lock); |
| 281 | |
| 282 | /* Make sure we do not run over the same sockets multiple times if the |
| 283 | * configured llist_size is larger then the actual list */ |
| 284 | batch_size = osmo_tcp_stats_config->batch_size; |
| 285 | llist_size = llist_count(&stats_tcp); |
| 286 | if (llist_size < batch_size) |
| 287 | batch_size = llist_size; |
| 288 | |
| 289 | /* Process a batch of sockets */ |
| 290 | for (i = 0; i < batch_size; i++) { |
| 291 | next_stats_tcp_entry(); |
| 292 | if (stats_tcp_entry_cur) |
| 293 | fill_stats(stats_tcp_entry_cur); |
| 294 | } |
| 295 | |
| 296 | pthread_mutex_unlock(&stats_tcp_lock); |
| 297 | |
| 298 | if (osmo_tcp_stats_config->interval > 0) |
| 299 | osmo_timer_schedule(&stats_tcp_poll_timer, osmo_tcp_stats_config->interval, 0); |
| 300 | } |
| 301 | |
| 302 | /*! Set the polling interval (common for all sockets) |
| 303 | * \param[in] interval Poll interval in seconds |
| 304 | * \returns 0 on success; negative on error */ |
| 305 | int osmo_stats_tcp_set_interval(int interval) |
| 306 | { |
| 307 | osmo_tcp_stats_config->interval = interval; |
| 308 | if (osmo_tcp_stats_config->interval > 0) |
| 309 | osmo_timer_schedule(&stats_tcp_poll_timer, osmo_tcp_stats_config->interval, 0); |
| 310 | return 0; |
| 311 | } |
| 312 | |
| 313 | static __attribute__((constructor)) |
| 314 | void on_dso_load_stats_tcp(void) |
| 315 | { |
| 316 | stats_tcp_entry_cur = NULL; |
| 317 | pthread_mutex_init(&stats_tcp_lock, NULL); |
| 318 | |
| 319 | osmo_tcp_stats_config->interval = TCP_STATS_DEFAULT_INTERVAL; |
| 320 | osmo_tcp_stats_config->batch_size = TCP_STATS_DEFAULT_BATCH_SIZE; |
| 321 | |
| 322 | osmo_timer_setup(&stats_tcp_poll_timer, stats_tcp_poll_timer_cb, NULL); |
| 323 | } |
| 324 | |
| 325 | #endif /* !EMBEDDED */ |
| 326 | |
| 327 | /* @} */ |