libmgcp: Fail if transcoding can't be configured

We want to fail theallocation of an endpoint in case the
transcoding can't be configured.

Manually verified with:

./src/osmo-bsc_mgcp/osmo-bsc_mgcp -c doc/examples/osmo-bsc_mgcp/mgcp.cfg

$ ./contrib/mgcp_server.py
0000   32 30 30 20 33 30 36 39    200 3069
0008   31 20 4F 4B 0D 0A          1 OK.. ('127.0.0.1', 2427)
0000   34 30 30 20 35 39 30 36    400 5906
0008   39 20 46 41 49 4C 0D 0A    9 FAIL.. ('127.0.0.1', 2427)
0000   34 30 30 20 33 35 34 36    400 3546
0008   33 20 46 41 49 4C 0D 0A    3 FAIL.. ('127.0.0.1', 2427)
0000   34 30 30 20 36 32 31 37    400 6217
0008   30 20 46 41 49 4C 0D 0A    0 FAIL.. ('127.0.0.1', 2427)

Verified by not sending L: in the CRCX and then failing on the
MDCX.
diff --git a/openbsc/src/libmgcp/mgcp_protocol.c b/openbsc/src/libmgcp/mgcp_protocol.c
index ccd15e8..62f6974 100644
--- a/openbsc/src/libmgcp/mgcp_protocol.c
+++ b/openbsc/src/libmgcp/mgcp_protocol.c
@@ -111,7 +111,7 @@
 static void create_transcoder(struct mgcp_endpoint *endp);
 static void delete_transcoder(struct mgcp_endpoint *endp);
 
-static void setup_rtp_processing(struct mgcp_endpoint *endp);
+static int setup_rtp_processing(struct mgcp_endpoint *endp);
 
 static int mgcp_analyze_header(struct mgcp_parse_data *parse, char *data);
 
@@ -1050,7 +1050,8 @@
 		endp->bts_end.force_output_ptime = 1;
 	}
 
-	setup_rtp_processing(endp);
+	if (setup_rtp_processing(endp) != 0)
+		goto error2;
 
 	/* policy CB */
 	if (p->cfg->policy_cb) {
@@ -1161,7 +1162,8 @@
 		set_audio_info(p->cfg, &endp->net_end.codec,
 			       PTYPE_UNDEFINED, endp->local_options.codec);
 
-	setup_rtp_processing(endp);
+	if (setup_rtp_processing(endp) != 0)
+		goto error3;
 
 	/* policy CB */
 	if (p->cfg->policy_cb) {
@@ -1659,25 +1661,27 @@
 	return send_agent(endp->cfg, buf, len);
 }
 
-static void setup_rtp_processing(struct mgcp_endpoint *endp)
+static int setup_rtp_processing(struct mgcp_endpoint *endp)
 {
+	int rc = 0;
 	struct mgcp_config *cfg = endp->cfg;
 
 	if (endp->type != MGCP_RTP_DEFAULT)
-		return;
+		return 0;
 
 	if (endp->conn_mode == MGCP_CONN_LOOPBACK)
-		return;
+		return 0;
 
 	if (endp->conn_mode & MGCP_CONN_SEND_ONLY)
-		cfg->setup_rtp_processing_cb(endp, &endp->net_end, &endp->bts_end);
+		rc |= cfg->setup_rtp_processing_cb(endp, &endp->net_end, &endp->bts_end);
 	else
-		cfg->setup_rtp_processing_cb(endp, &endp->net_end, NULL);
+		rc |= cfg->setup_rtp_processing_cb(endp, &endp->net_end, NULL);
 
 	if (endp->conn_mode & MGCP_CONN_RECV_ONLY)
-		cfg->setup_rtp_processing_cb(endp, &endp->bts_end, &endp->net_end);
+		rc |= cfg->setup_rtp_processing_cb(endp, &endp->bts_end, &endp->net_end);
 	else
-		cfg->setup_rtp_processing_cb(endp, &endp->bts_end, NULL);
+		rc |= cfg->setup_rtp_processing_cb(endp, &endp->bts_end, NULL);
+	return rc;
 }
 
 static void create_transcoder(struct mgcp_endpoint *endp)