gtphub: improve handling of restarted peer.

Handle peer restart earlier, so that all the tunnels are deleted by the restart
code path, instead of the first one being deleted due to reused TEI. That
caused confusing logging messages.

Also, when receiving Delete confirmations from the peer that didn't restart,
don't complain about unknown peer, but acknowledge and remove the half
invalidated tunnel. This means that the pending delete entry from the restart
code path is not needed / not used, so don't bother to add pending delete
entries upon peer restart.

The test test_peer_restarted_reusing_tei() hits the situation where a tunnel is
removed because of a reused TEI rather than the restart counter. Adjust the
test to expect the "out-of-band" delete request earlier on, and to still see
the half invalidated tunnel around. Enhance the test by adding the delete
response from the peer that didn't restart, and add a final tunnels_are()
verification.

Sponsored-by: On-Waves ehi
diff --git a/openbsc/src/gprs/gtphub.c b/openbsc/src/gprs/gtphub.c
index c8dddc5..a68fa65 100644
--- a/openbsc/src/gprs/gtphub.c
+++ b/openbsc/src/gprs/gtphub.c
@@ -1871,14 +1871,6 @@
 			 * dropped because the tunnel is rendered unusable. */
 			gtphub_send_del_pdp_ctx(hub, tun, other_side_idx(side_idx));
 
-			struct pending_delete *pd;
-			pd = pending_delete_new();
-			pd->tun = tun;
-			pd->teardown_ind = 0xff;
-			pd->nsapi = 0;
-			llist_add(&pd->entry, &hub->pending_deletes);
-			expiry_add(&hub->expire_quickly, &pd->expiry_entry, p->timestamp);
-
 			gtphub_tunnel_endpoint_set_peer(&tun->endpoint[side_idx][GTPH_PLANE_CTRL],
 							NULL);
 			gtphub_tunnel_endpoint_set_peer(&tun->endpoint[side_idx][GTPH_PLANE_USER],
@@ -2228,6 +2220,9 @@
 	LOG(LOGL_DEBUG, "from %s peer: %s\n", gtphub_side_idx_names[side_idx],
 	    gtphub_port_str(from_peer));
 
+	gtphub_check_restart_counter(hub, &p, from_peer);
+	gtphub_map_restart_counter(hub, &p);
+
 	struct gtphub_peer_port *to_peer_from_seq;
 	struct gtphub_peer_port *to_peer;
 	if (gtphub_unmap(hub, &p, from_peer,
@@ -2249,6 +2244,15 @@
 			return -1;
 	}
 
+	if (!to_peer && p.tun && p.type == GTP_DELETE_PDP_RSP) {
+		/* It's a delete confirmation for a tunnel that is partly
+		 * invalid, probably marked unsuable due to a restarted peer.
+		 * Remove the tunnel and be happy without forwarding. */
+		expiring_item_del(&p.tun->expiry_entry);
+		p.tun = NULL;
+		return 0;
+	}
+
 	if (!to_peer) {
 		LOG(LOGL_ERROR, "No %s to send to. Dropping packet%s"
 		    " (type=%" PRIu8 ", header-TEI=%" PRIx32 ", seq=%" PRIx16 ").\n",
@@ -2273,9 +2277,6 @@
 	 * or the tunnel has been deleted due to this message. */
 	OSMO_ASSERT(p.tun || (p.type == GTP_DELETE_PDP_RSP));
 
-	gtphub_check_restart_counter(hub, &p, from_peer);
-	gtphub_map_restart_counter(hub, &p);
-
 	/* If the GGSN is replying to an SGSN request, the sequence nr has
 	 * already been unmapped above (to_peer_from_seq != NULL), and we need not
 	 * create a new mapping. */
diff --git a/openbsc/tests/gtphub/gtphub_test.c b/openbsc/tests/gtphub/gtphub_test.c
index fa9ad8b..8ce83c8 100644
--- a/openbsc/tests/gtphub/gtphub_test.c
+++ b/openbsc/tests/gtphub/gtphub_test.c
@@ -905,9 +905,13 @@
 	send = gtphub_handle_buf(hub, GTPH_SIDE_GGSN, plane_idx, ggsn_sender,
 				 buf, msg(msg_from_ggsn), now,
 				 &reply_buf, &sgsn_ofd, &sgsn_addr);
-	LVL2_ASSERT(send > 0);
-	LVL2_ASSERT(same_addr(&sgsn_addr, sgsn_receiver));
-	LVL2_ASSERT(reply_is(msg_to_sgsn));
+	if (*msg_to_sgsn) {
+		LVL2_ASSERT(send > 0);
+		LVL2_ASSERT(same_addr(&sgsn_addr, sgsn_receiver));
+		LVL2_ASSERT(reply_is(msg_to_sgsn));
+	}
+	else
+		LVL2_ASSERT(send == 0);
 	return 1;
 }
 
@@ -1410,7 +1414,8 @@
 			       );
 	const char *gtp_req_to_ggsn =
 		MSG_PDP_CTX_REQ("0068",
-				"6d32",	/* mapped seq ("1234") */
+				"6d33",	/* seq 6d31 + 2, after "out-of-band" Delete PDP Ctx
+					   due to differing restart counter. */
 				"23",
 				"42000121436587f9",
 				"00000002", /* mapped TEI Data I ("123") */
@@ -1427,7 +1432,31 @@
 	OSMO_ASSERT(was_resolved_for("240010123456789", "internet"));
 
 	OSMO_ASSERT(tunnels_are(
-		"TEI=2:"
+		"TEI=2:" /* being established after restart */
+		" 192.168.42.23 (TEI C=321 U=123)"
+		" <-> 192.168.43.34/(uninitialized) (TEI C=0 U=0)"
+		" @21955\n"
+		"TEI=1:" /* invalidated due to restart */
+		" (uninitialized) (TEI C=321 U=123)"
+		" <-> 192.168.43.34 (TEI C=765 U=567)"
+		" @21945\n"
+		));
+
+	/* An "out-of-band" delete request should have been sent to the GGSN
+	 * (checked by expected log output in gtphub_test.ok), and the GGSN
+	 * will (usually) send a Delete Response like this: */
+	const char *gtp_del_resp_from_ggsn =
+		MSG_DEL_PDP_CTX_RSP("00000001", "6d32");
+
+	/* For this response (due to peer restart) we expect no forwarded
+	 * message. */
+	OSMO_ASSERT(msg_from_ggsn_c(&resolved_ggsn_addr,
+				    &sgsn_sender,
+				    gtp_del_resp_from_ggsn,
+				    ""));
+
+	OSMO_ASSERT(tunnels_are(
+		"TEI=2:" /* still being established after restart */
 		" 192.168.42.23 (TEI C=321 U=123)"
 		" <-> 192.168.43.34/(uninitialized) (TEI C=0 U=0)"
 		" @21955\n"
@@ -1436,7 +1465,7 @@
 	const char *gtp_resp_from_ggsn =
 		MSG_PDP_CTX_RSP("004e",
 				"00000002", /* destination TEI (sent in req above) */
-				"6d32", /* mapped seq */
+				"6d33", /* mapped seq */
 				"01", /* restart */
 				"00000def", /* TEI U */
 				"00000fde", /* TEI C */
@@ -1446,7 +1475,7 @@
 	const char *gtp_resp_to_sgsn =
 		MSG_PDP_CTX_RSP("004e",
 				"00000321", /* unmapped TEI ("005") */
-				"1234", /* unmapped seq ("6d32") */
+				"1234", /* unmapped seq ("6d33") */
 				"23",
 				"00000002", /* mapped TEI from GGSN ("567") */
 				"00000002", /* mapped TEI from GGSN ("765") */
@@ -1460,6 +1489,13 @@
 				    gtp_resp_from_ggsn,
 				    gtp_resp_to_sgsn));
 
+	OSMO_ASSERT(tunnels_are(
+		"TEI=2:" /* still being established after restart */
+		" 192.168.42.23 (TEI C=321 U=123)"
+		" <-> 192.168.43.34 (TEI C=fde U=def)"
+		" @21955\n"
+		));
+
 	OSMO_ASSERT(clear_test_hub());
 }
 
diff --git a/openbsc/tests/gtphub/gtphub_test.ok b/openbsc/tests/gtphub/gtphub_test.ok
index 4a6118f..e60d5f2 100644
--- a/openbsc/tests/gtphub/gtphub_test.ok
+++ b/openbsc/tests/gtphub/gtphub_test.ok
@@ -17,14 +17,17 @@
 test_peer_restarted
 - __wrap_gtphub_resolve_ggsn_addr():
   returning GGSN addr from imsi 240010123456789 ni internet: 192.168.43.34 port 2123
+Out-of-band gtphub_write(16):
+to 192.168.43.34 port 2123
+32 14 00 08 00 00 07 65 6d 32 00 00 13 ff 14 00 
+- __wrap_gtphub_resolve_ggsn_addr():
+  returning GGSN addr from imsi 240010123456789 ni internet: 192.168.43.34 port 2123
+test_peer_restarted_reusing_tei
 - __wrap_gtphub_resolve_ggsn_addr():
   returning GGSN addr from imsi 240010123456789 ni internet: 192.168.43.34 port 2123
 Out-of-band gtphub_write(16):
 to 192.168.43.34 port 2123
 32 14 00 08 00 00 07 65 6d 32 00 00 13 ff 14 00 
-test_peer_restarted_reusing_tei
-- __wrap_gtphub_resolve_ggsn_addr():
-  returning GGSN addr from imsi 240010123456789 ni internet: 192.168.43.34 port 2123
 - __wrap_gtphub_resolve_ggsn_addr():
   returning GGSN addr from imsi 240010123456789 ni internet: 192.168.43.34 port 2123
 test_user_data