diff --git a/src/trau/osmo_ortp.c b/src/trau/osmo_ortp.c
index 5441337..6ce70d2 100644
--- a/src/trau/osmo_ortp.c
+++ b/src/trau/osmo_ortp.c
@@ -389,18 +389,24 @@
 		return 0;
 	}
 
+	/* We don't want the connected mode enabled during
+	 * rtp_session_set_remote_addr(), because that will already setup a
+	 * connection and updating the remote address will no longer have an
+	 * effect. Contrary to what one may expect, this must be 0 at first,
+	 * and we're setting to 1 further down to establish a connection once
+	 * the first RTP packet is received (OS#1661). */
+	rtp_session_set_connected_mode(rs->sess, 0);
+
+	rc = rtp_session_set_remote_addr(rs->sess, ip, port);
+	if (rc < 0)
+		return rc;
+
 	/* enable the use of connect() so later getsockname() will
 	 * actually return the IP address that was chosen for the local
 	 * sid of the connection */
 	rtp_session_set_connected_mode(rs->sess, 1);
 	rs->flags &= ~OSMO_RTP_F_DISABLED;
 
-	/* This call attempts to connect to the remote address, so make sure to
-	 * set all other rtp session configuration before this call. */
-	rc = rtp_session_set_remote_addr(rs->sess, ip, port);
-	if (rc < 0)
-		return rc;
-
 	if (rs->flags & OSMO_RTP_F_POLL)
 		return rc;
 	else