diff --git a/src/trau/osmo_ortp.c b/src/trau/osmo_ortp.c
index eb78212..cdcc1e7 100644
--- a/src/trau/osmo_ortp.c
+++ b/src/trau/osmo_ortp.c
@@ -392,16 +392,18 @@
 		return 0;
 	}
 
-	rc = rtp_session_set_remote_addr(rs->sess, ip, port);
-	if (rc < 0)
-		return rc;
-
 	/* enable the use of connect() so later getsockname() will
 	 * actually return the IP address that was chosen for the local
 	 * sid of the connection */
 	rtp_session_set_connected_mode(rs->sess, 1);
 	rs->flags &= ~OSMO_RTP_F_DISABLED;
 
+	/* This call attempts to connect to the remote address, so make sure to
+	 * set all other rtp session configuration before this call. */
+	rc = rtp_session_set_remote_addr(rs->sess, ip, port);
+	if (rc < 0)
+		return rc;
+
 	if (rs->flags & OSMO_RTP_F_POLL)
 		return rc;
 	else