mirror of
https://github.com/curl/curl.git
synced 2026-08-26 04:13:31 +03:00
HTTP: implement Brotli content encoding
This uses the brotli external library (https://github.com/google/brotli). Brotli becomes a feature: additional curl_version_info() bit and structure fields are provided for it and CURLVERSION_NOW bumped. Tests 314 and 315 check Brotli content unencoding with correct and erroneous data. Some tests are updated to accomodate with the now configuration dependent parameters of the Accept-Encoding header.
This commit is contained in:
Patrick Monnerat
parent
dbcced8e32
commit
11bf1796cd
21 changed files with 633 additions and 32 deletions
|
|
@ -37,14 +37,15 @@
|
|||
|
||||
#ifndef CURL_DISABLE_HTTP
|
||||
|
||||
#define DSIZ CURL_MAX_WRITE_SIZE /* buffer size for decompressed data */
|
||||
|
||||
|
||||
#ifdef HAVE_LIBZ
|
||||
|
||||
/* Comment this out if zlib is always going to be at least ver. 1.2.0.4
|
||||
(doing so will reduce code size slightly). */
|
||||
#define OLD_ZLIB_SUPPORT 1
|
||||
|
||||
#define DSIZ CURL_MAX_WRITE_SIZE /* buffer size for decompressed data */
|
||||
|
||||
#define GZIP_MAGIC_0 0x1f
|
||||
#define GZIP_MAGIC_1 0x8b
|
||||
|
||||
|
|
@ -505,6 +506,136 @@ static const content_encoding gzip_encoding = {
|
|||
#endif /* HAVE_LIBZ */
|
||||
|
||||
|
||||
#ifdef HAVE_BROTLI
|
||||
|
||||
/* Writer parameters. */
|
||||
typedef struct {
|
||||
BrotliDecoderState *br; /* State structure for brotli. */
|
||||
} brotli_params;
|
||||
|
||||
|
||||
static CURLcode brotli_map_error(BrotliDecoderErrorCode be)
|
||||
{
|
||||
switch(be) {
|
||||
case BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_NIBBLE:
|
||||
case BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_META_NIBBLE:
|
||||
case BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_ALPHABET:
|
||||
case BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_SAME:
|
||||
case BROTLI_DECODER_ERROR_FORMAT_CL_SPACE:
|
||||
case BROTLI_DECODER_ERROR_FORMAT_HUFFMAN_SPACE:
|
||||
case BROTLI_DECODER_ERROR_FORMAT_CONTEXT_MAP_REPEAT:
|
||||
case BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_1:
|
||||
case BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_2:
|
||||
case BROTLI_DECODER_ERROR_FORMAT_TRANSFORM:
|
||||
case BROTLI_DECODER_ERROR_FORMAT_DICTIONARY:
|
||||
case BROTLI_DECODER_ERROR_FORMAT_WINDOW_BITS:
|
||||
case BROTLI_DECODER_ERROR_FORMAT_PADDING_1:
|
||||
case BROTLI_DECODER_ERROR_FORMAT_PADDING_2:
|
||||
case BROTLI_DECODER_ERROR_COMPOUND_DICTIONARY:
|
||||
case BROTLI_DECODER_ERROR_DICTIONARY_NOT_SET:
|
||||
case BROTLI_DECODER_ERROR_INVALID_ARGUMENTS:
|
||||
return CURLE_BAD_CONTENT_ENCODING;
|
||||
case BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MODES:
|
||||
case BROTLI_DECODER_ERROR_ALLOC_TREE_GROUPS:
|
||||
case BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MAP:
|
||||
case BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_1:
|
||||
case BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_2:
|
||||
case BROTLI_DECODER_ERROR_ALLOC_BLOCK_TYPE_TREES:
|
||||
return CURLE_OUT_OF_MEMORY;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return CURLE_WRITE_ERROR;
|
||||
}
|
||||
|
||||
static CURLcode brotli_init_writer(struct connectdata *conn,
|
||||
contenc_writer *writer)
|
||||
{
|
||||
brotli_params *bp = (brotli_params *) &writer->params;
|
||||
|
||||
(void) conn;
|
||||
|
||||
if(!writer->downstream)
|
||||
return CURLE_WRITE_ERROR;
|
||||
|
||||
bp->br = BrotliDecoderCreateInstance(NULL, NULL, NULL);
|
||||
return bp->br? CURLE_OK: CURLE_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
static CURLcode brotli_unencode_write(struct connectdata *conn,
|
||||
contenc_writer *writer,
|
||||
const char *buf, size_t nbytes)
|
||||
{
|
||||
brotli_params *bp = (brotli_params *) &writer->params;
|
||||
const uint8_t *src = (const uint8_t *) buf;
|
||||
char *decomp;
|
||||
uint8_t *dst;
|
||||
size_t dstleft;
|
||||
CURLcode result = CURLE_OK;
|
||||
|
||||
if(!nbytes)
|
||||
return CURLE_OK;
|
||||
if(!bp->br)
|
||||
return CURLE_WRITE_ERROR; /* Stream already ended. */
|
||||
|
||||
decomp = malloc(DSIZ);
|
||||
if(!decomp)
|
||||
return CURLE_OUT_OF_MEMORY;
|
||||
|
||||
while(nbytes && result == CURLE_OK) {
|
||||
BrotliDecoderResult r;
|
||||
|
||||
dst = (uint8_t *) decomp;
|
||||
dstleft = DSIZ;
|
||||
r = BrotliDecoderDecompressStream(bp->br,
|
||||
&nbytes, &src, &dstleft, &dst, NULL);
|
||||
result = Curl_unencode_write(conn, writer->downstream,
|
||||
decomp, DSIZ - dstleft);
|
||||
if(result)
|
||||
break;
|
||||
switch(r) {
|
||||
case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
|
||||
case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
|
||||
break;
|
||||
case BROTLI_DECODER_RESULT_SUCCESS:
|
||||
BrotliDecoderDestroyInstance(bp->br);
|
||||
bp->br = NULL;
|
||||
if(nbytes)
|
||||
result = CURLE_WRITE_ERROR;
|
||||
break;
|
||||
default:
|
||||
result = brotli_map_error(BrotliDecoderGetErrorCode(bp->br));
|
||||
break;
|
||||
}
|
||||
}
|
||||
free(decomp);
|
||||
return result;
|
||||
}
|
||||
|
||||
static void brotli_close_writer(struct connectdata *conn,
|
||||
contenc_writer *writer)
|
||||
{
|
||||
brotli_params *bp = (brotli_params *) &writer->params;
|
||||
|
||||
(void) conn;
|
||||
|
||||
if(bp->br) {
|
||||
BrotliDecoderDestroyInstance(bp->br);
|
||||
bp->br = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static const content_encoding brotli_encoding = {
|
||||
"br",
|
||||
NULL,
|
||||
brotli_init_writer,
|
||||
brotli_unencode_write,
|
||||
brotli_close_writer,
|
||||
sizeof(brotli_params)
|
||||
};
|
||||
#endif
|
||||
|
||||
|
||||
/* Identity handler. */
|
||||
static CURLcode identity_init_writer(struct connectdata *conn,
|
||||
contenc_writer *writer)
|
||||
|
|
@ -543,6 +674,9 @@ static const content_encoding * const encodings[] = {
|
|||
#ifdef HAVE_LIBZ
|
||||
&deflate_encoding,
|
||||
&gzip_encoding,
|
||||
#endif
|
||||
#ifdef HAVE_BROTLI
|
||||
&brotli_encoding,
|
||||
#endif
|
||||
NULL
|
||||
};
|
||||
|
|
|
|||
|
|
@ -96,6 +96,10 @@
|
|||
#endif
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_BROTLI
|
||||
#include <brotli/decode.h>
|
||||
#endif
|
||||
|
||||
#include <curl/curl.h>
|
||||
|
||||
#include "http_chunks.h" /* for the structs and enum stuff */
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
* | (__| |_| | _ <| |___
|
||||
* \___|\___/|_| \_\_____|
|
||||
*
|
||||
* Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al.
|
||||
* Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al.
|
||||
*
|
||||
* This software is licensed as described in the file COPYING, which
|
||||
* you should have received as part of this distribution. The terms
|
||||
|
|
@ -74,6 +74,18 @@ void Curl_version_init(void)
|
|||
curl_version_info(CURLVERSION_NOW);
|
||||
}
|
||||
|
||||
#ifdef HAVE_BROTLI
|
||||
static size_t brotli_version(char *buf, size_t bufsz)
|
||||
{
|
||||
uint32_t brotli_version = BrotliDecoderVersion();
|
||||
unsigned int major = brotli_version >> 24;
|
||||
unsigned int minor = (brotli_version & 0x00FFFFFF) >> 12;
|
||||
unsigned int patch = brotli_version & 0x00000FFF;
|
||||
|
||||
return snprintf(buf, bufsz, "%u.%u.%u", major, minor, patch);
|
||||
}
|
||||
#endif
|
||||
|
||||
char *curl_version(void)
|
||||
{
|
||||
static bool initialized;
|
||||
|
|
@ -105,6 +117,14 @@ char *curl_version(void)
|
|||
left -= len;
|
||||
ptr += len;
|
||||
#endif
|
||||
#ifdef HAVE_BROTLI
|
||||
len = snprintf(ptr, left, "%s", " brotli/");
|
||||
left -= len;
|
||||
ptr += len;
|
||||
len = brotli_version(ptr, left);
|
||||
left -= len;
|
||||
ptr += len;
|
||||
#endif
|
||||
#ifdef USE_ARES
|
||||
/* this function is only present in c-ares, not in the original ares */
|
||||
len = snprintf(ptr, left, " c-ares/%s", ares_version(NULL));
|
||||
|
|
@ -326,6 +346,9 @@ static curl_version_info_data version_info = {
|
|||
#endif
|
||||
#if defined(CURL_WITH_MULTI_SSL)
|
||||
| CURL_VERSION_MULTI_SSL
|
||||
#endif
|
||||
#if defined(HAVE_BROTLI)
|
||||
| CURL_VERSION_BROTLI
|
||||
#endif
|
||||
,
|
||||
NULL, /* ssl_version */
|
||||
|
|
@ -337,6 +360,8 @@ static curl_version_info_data version_info = {
|
|||
NULL, /* libidn version */
|
||||
0, /* iconv version */
|
||||
NULL, /* ssh lib version */
|
||||
0, /* brotli_ver_num */
|
||||
NULL, /* brotli version */
|
||||
};
|
||||
|
||||
curl_version_info_data *curl_version_info(CURLversion stamp)
|
||||
|
|
@ -348,6 +373,9 @@ curl_version_info_data *curl_version_info(CURLversion stamp)
|
|||
#ifdef USE_SSL
|
||||
static char ssl_buffer[80];
|
||||
#endif
|
||||
#ifdef HAVE_BROTLI
|
||||
static char brotli_buffer[80];
|
||||
#endif
|
||||
|
||||
if(initialized)
|
||||
return &version_info;
|
||||
|
|
@ -396,6 +424,12 @@ curl_version_info_data *curl_version_info(CURLversion stamp)
|
|||
version_info.libssh_version = ssh_buffer;
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_BROTLI
|
||||
version_info.brotli_ver_num = BrotliDecoderVersion();
|
||||
brotli_version(brotli_buffer, sizeof brotli_buffer);
|
||||
version_info.brotli_version = brotli_buffer;
|
||||
#endif
|
||||
|
||||
(void)stamp; /* avoid compiler warnings, we don't use this */
|
||||
|
||||
initialized = true;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue