diff --git a/lib/conncache.c b/lib/conncache.c
index 2cdfd34d9a..88463354bc 100644
--- a/lib/conncache.c
+++ b/lib/conncache.c
@@ -163,9 +163,12 @@ static void hashkey(struct connectdata *conn, char *buf,
                     const char **hostp)
 {
   const char *hostname;
+  long port = conn->remote_port;
 
-  if(conn->bits.httpproxy && !conn->bits.tunnel_proxy)
+  if(conn->bits.httpproxy && !conn->bits.tunnel_proxy) {
     hostname = conn->http_proxy.host.name;
+    port = conn->port;
+  }
   else if(conn->bits.conn_to_host)
     hostname = conn->conn_to_host.name;
   else
@@ -178,7 +181,7 @@ static void hashkey(struct connectdata *conn, char *buf,
   DEBUGASSERT(len > 32);
 
   /* put the number first so that the hostname gets cut off if too long */
-  msnprintf(buf, len, "%ld%s", conn->port, hostname);
+  msnprintf(buf, len, "%ld%s", port, hostname);
 }
 
 void Curl_conncache_unlock(struct Curl_easy *data)
diff --git a/lib/url.c b/lib/url.c
index 88d72bfeac..51515c924d 100644
--- a/lib/url.c
+++ b/lib/url.c
@@ -2027,7 +2027,7 @@ static CURLcode parseurlandfillconn(struct Curl_easy *data,
   }
   else {
     unsigned long port = strtoul(data->state.up.port, NULL, 10);
-    conn->remote_port = curlx_ultous(port);
+    conn->port = conn->remote_port = curlx_ultous(port);
   }
 
   (void)curl_url_get(uh, CURLUPART_QUERY, &data->state.up.query, 0);