content_encoding: Transfer-Encoding parser improvements

- allow and ignore "identity" as an encoding

- fail if any other encoder than chunked follows after chunked

- fail on unsolicited encodings - when the server encodes but curl did
  not ask for it

Add test 1493 to 1496 to verify.

Disable test 319 as that is now broken: issue #16974

Reported-by: Jonathan Rosa
Fixes #16956
Closes #16959

lib/content_encoding.c

@@ -737,6 +737,7 @@ CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
   Curl_cwriter_phase phase = is_transfer ?
     CURL_CW_TRANSFER_DECODE : CURL_CW_CONTENT_DECODE;
   CURLcode result;
+  bool has_chunked = FALSE;
 
   do {
     const char *name;
@@ -765,9 +766,21 @@ CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
        * Exception is "chunked" transfer-encoding which always must happen */
       if((is_transfer && !data->set.http_transfer_encoding && !is_chunked) ||
          (!is_transfer && data->set.http_ce_skip)) {
+        bool is_identity = strncasecompare(name, "identity", 8);
         /* not requested, ignore */
         CURL_TRC_WRITE(data, "decoder not requested, ignored: %.*s",
                        (int)namelen, name);
+        if(is_transfer) {
+          if(has_chunked)
+            failf(data, "A Transfer-Encoding (%.*s) was listed after chunked",
+                  (int)namelen, name);
+          else if(is_identity)
+            continue;
+          else
+            failf(data, "Unsolicited Transfer-Encoding (%.*s) found",
+                  (int)namelen, name);
+          return CURLE_BAD_CONTENT_ENCODING;
+        }
         return CURLE_OK;
       }
 
@@ -818,6 +831,8 @@ CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
         Curl_cwriter_free(data, writer);
         return result;
       }
+      if(is_chunked)
+        has_chunked = TRUE;
     }
   } while(*enclist);
 

tests/data/DISABLED

@@ -27,6 +27,8 @@
 # per line.
 # Lines starting with '#' letters are treated as comments.
 #
+# PR #16959 makes Transfer-Encoding stricer and thus --raw broke
+319
 # Uses SRP to "a server not supporting it" but modern stunnel versions
 # will silently accept it and remain happy
 323

tests/data/Makefile.am

@@ -196,7 +196,7 @@ test1460 test1461 test1462 test1463 test1464 test1465 test1466 test1467 \
 test1468 test1469 test1470 test1471 test1472 test1473 test1474 test1475 \
 test1476 test1477 test1478 test1479 test1480 test1481 test1482 test1483 \
 test1484 test1485 test1486 test1487 test1488 test1489 test1490 test1491 \
-test1492 \
+test1492 test1493 test1494 test1495 test1496 \
 \
 test1500 test1501 test1502 test1503 test1504 test1505 test1506 test1507 \
 test1508 test1509 test1510 test1511 test1512 test1513 test1514 test1515 \

tests/data/test1493

@@ -0,0 +1,78 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+chunked Transfer-Encoding
+DELAY
+</keywords>
+</info>
+#
+# Server-side
+<reply>
+<data>
+HTTP/1.1 200 funky chunky!
+Server: fakeit/0.9 fakeitbad/1.0
+Transfer-Encoding: identity, chunked
+Connection: mooo
+
+40
+aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+30
+bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
+21;heresatest=moooo
+cccccccccccccccccccccccccccccccc
+
+0
+chunky-trailer: header data
+another-header: yes
+
+</data>
+<datacheck>
+HTTP/1.1 200 funky chunky!
+Server: fakeit/0.9 fakeitbad/1.0
+Transfer-Encoding: identity, chunked
+Connection: mooo
+
+aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbcccccccccccccccccccccccccccccccc
+chunky-trailer: header data
+another-header: yes
+</datacheck>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+HTTP GET with identity + chunked in TE header
+</name>
+<command>
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER -D %LOGDIR/heads%TESTNUMBER
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol>
+GET /%TESTNUMBER HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
+</protocol>
+<file name="%LOGDIR/heads%TESTNUMBER">
+HTTP/1.1 200 funky chunky!
+Server: fakeit/0.9 fakeitbad/1.0
+Transfer-Encoding: identity, chunked
+Connection: mooo
+
+chunky-trailer: header data
+another-header: yes
+</file>
+</verify>
+
+</testcase>

tests/data/test1494

@@ -0,0 +1,50 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+chunked Transfer-Encoding
+DELAY
+</keywords>
+</info>
+#
+# Server-side
+<reply>
+<data crlf="yes">
+HTTP/1.1 200 funky chunky!
+Server: fakeit/0.9 fakeitbad/1.0
+Transfer-Encoding: identity, identity
+Content-Length: 19
+
+stuff server sends
+</data>
+
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+HTTP GET with identity twice in TE header
+</name>
+<command>
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER -D %LOGDIR/heads%TESTNUMBER
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol>
+GET /%TESTNUMBER HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
+</protocol>
+</verify>
+
+</testcase>

tests/data/test1495

@@ -0,0 +1,53 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+chunked Transfer-Encoding
+DELAY
+</keywords>
+</info>
+#
+# Server-side
+<reply>
+<data crlf="yes" nocheck="yes">
+HTTP/1.1 200 funky chunky!
+Server: fakeit/0.9 fakeitbad/1.0
+Transfer-Encoding: chunked, identity
+Content-Length: 19
+
+stuff server sends
+</data>
+
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+HTTP GET with chunked + identity in TE header
+</name>
+<command>
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER -D %LOGDIR/heads%TESTNUMBER
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol>
+GET /%TESTNUMBER HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
+</protocol>
+<errorcode>
+61
+</errorcode>
+</verify>
+
+</testcase>

tests/data/test1496

@@ -0,0 +1,53 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+chunked Transfer-Encoding
+DELAY
+</keywords>
+</info>
+#
+# Server-side
+<reply>
+<data crlf="yes" nocheck="yes">
+HTTP/1.1 200 funky chunky!
+Server: fakeit/0.9 fakeitbad/1.0
+Transfer-Encoding: gzip, chunked
+Content-Length: 19
+
+stuff server sends
+</data>
+
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+HTTP GET with gzip + chunked transfer-encoding without being asked
+</name>
+<command>
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER -D %LOGDIR/heads%TESTNUMBER
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol>
+GET /%TESTNUMBER HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
+</protocol>
+<errorcode>
+61
+</errorcode>
+</verify>
+
+</testcase>

tests/data/test319

@@ -3,8 +3,8 @@
 <keywords>
 HTTP
 HTTP GET
-compressed
 Transfer-Encoding
+--raw
 </keywords>
 </info>
 #
@@ -15,7 +15,7 @@ HTTP/1.1 200 OK swsclose
 Date: Mon, 29 Nov 2004 21:56:53 GMT
 Server: Apache/1.3.31 (Debian GNU/Linux) mod_gzip/1.3.26.1a PHP/4.3.9-1 mod_ssl/2.8.20 OpenSSL/0.9.7d mod_perl/1.29
 Content-Type: text/html; charset=ISO-8859-1
-Transfer-Encoding: gzip
+Transfer-Encoding: gobbledigook
 Content-Length: 44
 
 %hex[%1f%8b%08%08%79%9e%ab%41%00%03%6c%61%6c%61%6c%61%00%cb%c9%cc%4b%55%30%e4%52%c8%01%d1%46%5c]hex%
@@ -34,14 +34,11 @@ Content-Length: 44
 #
 # Client-side
 <client>
-<features>
-libz
-</features>
 <server>
 http
 </server>
 <name>
-HTTP GET gzip transfer-encoded data in raw mode
+HTTP GET gobbledigook transfer-encoded data in raw mode
 </name>
  <command option="no-include">
 http://%HOSTIP:%HTTPPORT/%TESTNUMBER --raw