diff --git a/lib/cf-socket.c b/lib/cf-socket.c
index f449ca36ca..308325ccdc 100644
--- a/lib/cf-socket.c
+++ b/lib/cf-socket.c
@@ -1578,7 +1578,7 @@ static CURLcode cf_socket_recv(struct Curl_cfilter *cf, struct Curl_easy *data,
     *pnread = (size_t)nread;
 
   CURL_TRC_CF(data, cf, "recv(len=%zu) -> %d, %zu", len, result, *pnread);
-  if(!result && !ctx->got_first_byte) {
+  if(!result && !ctx->got_first_byte && nread) {
     ctx->first_byte_at = curlx_now();
     ctx->got_first_byte = TRUE;
   }