romenskiy2012/curl
1
0
Fork 0
mirror of https://github.com/curl/curl.git synced 2026-08-25 19:33:32 +03:00
Code Issues Projects Releases Packages Wiki Activity

lib: add bufq and dynhds

Browse source 
Adding `bufq`:
- at init() time configured to hold up to `n` chunks of `m` bytes each.
- various methods for reading from and writing to it.
- `peek` support to get access to buffered data without copy
- `pass` support to allow buffer flushing on write if it becomes full
- use case: IO buffers for dynamic reads and writes that do not blow up
- distinct from `dynbuf` in that:
  - it maintains a read position
  - writes on a full bufq return CURLE_AGAIN instead of nuking itself
- Init options:
  - SOFT_LIMIT: allow writes into a full bufq
  - NO_SPARES: free empty chunks right away
- a `bufc_pool` that can keep a number of spare chunks to
  be shared between different `bufq` instances

Adding `dynhds`:
- a straightforward list of name+value pairs as used for HTTP headers
- headers can be appended dynamically
- headers can be removed again
- headers can be replaced
- headers can be looked up
- http/1.1 formatting into a `dynbuf`
- configured at init() with limits on header counts and total string
  sizes
- use case: pass a HTTP request or response around without being version
  specific
- express a HTTP request without a curl easy handle (used in h2 proxy
  tunnels)
- future extension possibilities:
  - conversions of `dynhds` to nghttp2/nghttp3 name+value arrays

Closes #10720
This commit is contained in:
Stefan Eissing 2023-03-20 14:23:53 +01:00 • committed by Daniel Stenberg
parent 8cabef6fc3
commit 61f52a97e9
No known key found for this signature in database
GPG key ID: 5CC908FDB71E12C2
15 changed files with 2070 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

2
tests/data/Makefile.inc
View file

@ -250,7 +250,7 @@ test2400 test2401 test2402 test2403 \
\
test2500 test2501 test2502 test2503 \
\
test2600 \
test2600 test2601 test2602 \
\
test3000 test3001 test3002 test3003 test3004 test3005 test3006 test3007 \
test3008 test3009 test3010 test3011 test3012 test3013 test3014 test3015 \

22
tests/data/test2601 Normal file
View file

@ -0,0 +1,22 @@
<testcase>
<info>
<keywords>
unittest
bufq
</keywords>
</info>
#
# Client-side
<client>
<server>
none
</server>
<features>
unittest
</features>
<name>
bufq unit tests
</name>
</client>
</testcase>

22
tests/data/test2602 Normal file
View file

@ -0,0 +1,22 @@
<testcase>
<info>
<keywords>
unittest
dynhds
</keywords>
</info>
#
# Client-side
<client>
<server>
none
</server>
<features>
unittest
</features>
<name>
dynhds unit tests
</name>
</client>
</testcase>

2
tests/unit/CMakeLists.txt
View file

@ -38,7 +38,7 @@ include_directories(
# or else they will fail to link. Some of the tests require the special libcurlu
# build, so filter those out until we get libcurlu.
list(FILTER UNITPROGS EXCLUDE REGEX
"unit1394|unit1395|unit1604|unit1608|unit1621|unit1650|unit1653|unit1655|unit1660|unit2600")
"unit1394|unit1395|unit1604|unit1608|unit1621|unit1650|unit1653|unit1655|unit1660|unit2600|unit2601|unit2602")
if(NOT BUILD_SHARED_LIBS)
foreach(_testfile ${UNITPROGS})
add_executable(${_testfile} EXCLUDE_FROM_ALL ${_testfile}.c ${UNITFILES})

4
tests/unit/Makefile.am
View file

@ -154,4 +154,8 @@ unit1661_SOURCES = unit1661.c $(UNITFILES)
unit2600_SOURCES = unit2600.c $(UNITFILES)
unit2601_SOURCES = unit2601.c $(UNITFILES)
unit2602_SOURCES = unit2602.c $(UNITFILES)
unit3200_SOURCES = unit3200.c $(UNITFILES)

2
tests/unit/Makefile.inc
View file

@ -38,5 +38,5 @@ UNITPROGS = unit1300 unit1302 unit1303 unit1304 unit1305 unit1307 \
unit1620 unit1621 \
unit1650 unit1651 unit1652 unit1653 unit1654 unit1655 \
unit1660 unit1661 \
unit2600 \
unit2600 unit2601 unit2602 \
unit3200

246
tests/unit/unit2601.c Normal file
View file

@ -0,0 +1,246 @@
/***************************************************************************
* _ _ ____ _
* Project ___| | | | _ \| |
* / __| | | | |_) | |
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
* are also available at https://curl.se/docs/copyright.html.
*
* You may opt to use, copy, modify, merge, publish, distribute and/or sell
* copies of the Software, and permit persons to whom the Software is
* furnished to do so, under the terms of the COPYING file.
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
* KIND, either express or implied.
*
* SPDX-License-Identifier: curl
*
***************************************************************************/
#include "curlcheck.h"
#include "urldata.h"
#include "bufq.h"
#include "curl_log.h"
static CURLcode unit_setup(void)
{
CURLcode res = CURLE_OK;
return res;
}
static void unit_stop(void)
{
}
static const char *tail_err(struct bufq *q)
{
struct buf_chunk *chunk;
if(!q->tail) {
return q->head? "tail is NULL, but head is not" : NULL;
}
chunk = q->head;
while(chunk) {
if(chunk == q->tail) {
if(chunk->next) {
return "tail points to queue, but not at the end";
}
return NULL;
}
chunk = chunk->next;
}
return "tail not part of queue";
}
static void dump_bufq(struct bufq *q, const char *msg)
{
struct buf_chunk *chunk;
const char *terr;
size_t n;
fprintf(stderr, "bufq[chunk_size=%zu, max_chunks=%zu] %s\n",
q->chunk_size, q->max_chunks, msg);
fprintf(stderr, "- queue[\n");
chunk = q->head;
while(chunk) {
fprintf(stderr, " chunk[len=%zu, roff=%zu, woff=%zu]\n",
chunk->dlen, chunk->r_offset, chunk->w_offset);
chunk = chunk->next;
}
fprintf(stderr, " ]\n");
terr = tail_err(q);
fprintf(stderr, "- tail: %s\n", terr? terr : "ok");
n = 0;
chunk = q->spare;
while(chunk) {
++n;
chunk = chunk->next;
}
fprintf(stderr, "- chunks: %zu\n", q->chunk_count);
fprintf(stderr, "- spares: %zu\n", n);
}
static unsigned char test_data[32*1024];
static void check_bufq(size_t pool_spares,
size_t chunk_size, size_t max_chunks,
size_t wsize, size_t rsize, int opts)
{
struct bufq q;
struct bufc_pool pool;
size_t max_len = chunk_size * max_chunks;
CURLcode result;
ssize_t n, i;
size_t nwritten, nread;
if(pool_spares > 0) {
Curl_bufcp_init(&pool, chunk_size, pool_spares);
Curl_bufq_initp(&q, &pool, max_chunks, opts);
}
else {
Curl_bufq_init2(&q, chunk_size, max_chunks, opts);
}
fail_unless(q.chunk_size == chunk_size, "chunk_size init wrong");
fail_unless(q.max_chunks == max_chunks, "max_chunks init wrong");
fail_unless(q.head == NULL, "init: head not NULL");
fail_unless(q.tail == NULL, "init: tail not NULL");
fail_unless(q.spare == NULL, "init: spare not NULL");
fail_unless(Curl_bufq_len(&q) == 0, "init: bufq length != 0");
n = Curl_bufq_write(&q, test_data, wsize, &result);
fail_unless(n >= 0, "write: negative size returned");
fail_unless((size_t)n <= wsize, "write: wrong size returned");
fail_unless(result == CURLE_OK, "write: wrong result returned");
/* write empty bufq full */
nwritten = 0;
Curl_bufq_reset(&q);
while(!Curl_bufq_is_full(&q)) {
n = Curl_bufq_write(&q, test_data, wsize, &result);
if(n >= 0) {
nwritten += (size_t)n;
}
else if(result != CURLE_AGAIN) {
fail_unless(result == CURLE_AGAIN, "write-loop: unexpected result");
break;
}
}
if(nwritten != max_len) {
fprintf(stderr, "%zu bytes written, but max_len=%zu\n",
nwritten, max_len);
dump_bufq(&q, "after writing full");
fail_if(TRUE, "write: bufq full but nwritten wrong");
}
/* read full bufq empty */
nread = 0;
while(!Curl_bufq_is_empty(&q)) {
n = Curl_bufq_read(&q, test_data, rsize, &result);
if(n >= 0) {
nread += (size_t)n;
}
else if(result != CURLE_AGAIN) {
fail_unless(result == CURLE_AGAIN, "read-loop: unexpected result");
break;
}
}
if(nread != max_len) {
fprintf(stderr, "%zu bytes read, but max_len=%zu\n",
nwritten, max_len);
dump_bufq(&q, "after reading empty");
fail_if(TRUE, "read: bufq empty but nread wrong");
}
if(q.tail) {
dump_bufq(&q, "after reading empty");
fail_if(TRUE, "read empty, but tail is not NULL");
}
for(i = 0; i < 1000; ++i) {
n = Curl_bufq_write(&q, test_data, wsize, &result);
if(n < 0 && result != CURLE_AGAIN) {
fail_unless(result == CURLE_AGAIN, "rw-loop: unexpected write result");
break;
}
n = Curl_bufq_read(&q, test_data, rsize, &result);
if(n < 0 && result != CURLE_AGAIN) {
fail_unless(result == CURLE_AGAIN, "rw-loop: unexpected read result");
break;
}
}
/* Test SOFT_LIMIT option */
Curl_bufq_free(&q);
Curl_bufq_init2(&q, chunk_size, max_chunks, (opts|BUFQ_OPT_SOFT_LIMIT));
nwritten = 0;
while(!Curl_bufq_is_full(&q)) {
n = Curl_bufq_write(&q, test_data, wsize, &result);
if(n < 0 || (size_t)n != wsize) {
fail_unless(n > 0 && (size_t)n == wsize, "write should be complete");
break;
}
nwritten += (size_t)n;
}
if(nwritten < max_len) {
fprintf(stderr, "%zu bytes written, but max_len=%zu\n",
nwritten, max_len);
dump_bufq(&q, "after writing full");
fail_if(TRUE, "write: bufq full but nwritten wrong");
}
/* do one more write on a full bufq, should work */
n = Curl_bufq_write(&q, test_data, wsize, &result);
fail_unless(n > 0 && (size_t)n == wsize, "write should be complete");
nwritten += (size_t)n;
/* see that we get all out again */
nread = 0;
while(!Curl_bufq_is_empty(&q)) {
n = Curl_bufq_read(&q, test_data, rsize, &result);
if(n <= 0) {
fail_unless(n > 0, "read-loop: unexpected fail");
break;
}
nread += (size_t)n;
}
fail_unless(nread == nwritten, "did not get the same out as put in");
dump_bufq(&q, "at end of test");
Curl_bufq_free(&q);
if(pool_spares > 0)
Curl_bufcp_free(&pool);
}
UNITTEST_START
struct bufq q;
ssize_t n;
CURLcode result;
unsigned char buf[16*1024];
Curl_bufq_init(&q, 8*1024, 12);
n = Curl_bufq_read(&q, buf, 128, &result);
fail_unless(n < 0 && result == CURLE_AGAIN, "read empty fail");
Curl_bufq_free(&q);
check_bufq(0, 1024, 4, 128, 128, BUFQ_OPT_NONE);
check_bufq(0, 1024, 4, 129, 127, BUFQ_OPT_NONE);
check_bufq(0, 1024, 4, 2000, 16000, BUFQ_OPT_NONE);
check_bufq(0, 1024, 4, 16000, 3000, BUFQ_OPT_NONE);
check_bufq(0, 8000, 10, 1234, 1234, BUFQ_OPT_NONE);
check_bufq(0, 8000, 10, 8*1024, 4*1024, BUFQ_OPT_NONE);
check_bufq(0, 1024, 4, 128, 128, BUFQ_OPT_NO_SPARES);
check_bufq(0, 1024, 4, 129, 127, BUFQ_OPT_NO_SPARES);
check_bufq(0, 1024, 4, 2000, 16000, BUFQ_OPT_NO_SPARES);
check_bufq(0, 1024, 4, 16000, 3000, BUFQ_OPT_NO_SPARES);
check_bufq(8, 1024, 4, 128, 128, BUFQ_OPT_NONE);
check_bufq(8, 8000, 10, 1234, 1234, BUFQ_OPT_NONE);
check_bufq(8, 1024, 4, 129, 127, BUFQ_OPT_NO_SPARES);
UNITTEST_STOP

125
tests/unit/unit2602.c Normal file
View file

@ -0,0 +1,125 @@
/***************************************************************************
* _ _ ____ _
* Project ___| | | | _ \| |
* / __| | | | |_) | |
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
* are also available at https://curl.se/docs/copyright.html.
*
* You may opt to use, copy, modify, merge, publish, distribute and/or sell
* copies of the Software, and permit persons to whom the Software is
* furnished to do so, under the terms of the COPYING file.
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
* KIND, either express or implied.
*
* SPDX-License-Identifier: curl
*
***************************************************************************/
#include "curlcheck.h"
#include "urldata.h"
#include "dynbuf.h"
#include "dynhds.h"
#include "curl_log.h"
static CURLcode unit_setup(void)
{
return CURLE_OK;
}
static void unit_stop(void)
{
}
UNITTEST_START
struct dynhds hds;
struct dynbuf dbuf;
CURLcode result;
size_t i;
/* add 1 more header than allowed */
Curl_dynhds_init(&hds, 2, 128);
fail_if(Curl_dynhds_count(&hds), "should be empty");
fail_if(Curl_dynhds_add(&hds, "test1", 5, "123", 3), "add failed");
fail_if(Curl_dynhds_add(&hds, "test2", 5, "456", 3), "add failed");
/* remove and add without exceeding limits */
for(i = 0; i < 100; ++i) {
if(Curl_dynhds_remove(&hds, "test2", 5) != 1) {
fail_if(TRUE, "should");
break;
}
if(Curl_dynhds_add(&hds, "test2", 5, "456", 3)) {
fail_if(TRUE, "add failed");
break;
}
}
fail_unless(Curl_dynhds_count(&hds) == 2, "should hold 2");
/* set, replacing previous entry without exceeding limits */
for(i = 0; i < 100; ++i) {
if(Curl_dynhds_set(&hds, "test2", 5, "456", 3)) {
fail_if(TRUE, "add failed");
break;
}
}
fail_unless(Curl_dynhds_count(&hds) == 2, "should hold 2");
/* exceed limit on # of entries */
result = Curl_dynhds_add(&hds, "test3", 5, "789", 3);
fail_unless(result, "add should have failed");
fail_unless(Curl_dynhds_count_name(&hds, "test", 4) == 0, "false positive");
fail_unless(Curl_dynhds_count_name(&hds, "test1", 4) == 0, "false positive");
fail_if(Curl_dynhds_get(&hds, "test1", 4), "false positive");
fail_unless(Curl_dynhds_get(&hds, "test1", 5), "false negative");
fail_unless(Curl_dynhds_count_name(&hds, "test1", 5) == 1, "should");
fail_unless(Curl_dynhds_ccount_name(&hds, "test2") == 1, "should");
fail_unless(Curl_dynhds_cget(&hds, "test2"), "should");
fail_unless(Curl_dynhds_ccount_name(&hds, "TEST2") == 1, "should");
fail_unless(Curl_dynhds_ccontains(&hds, "TesT2"), "should");
fail_unless(Curl_dynhds_contains(&hds, "TeSt2", 5), "should");
Curl_dynhds_free(&hds);
/* add header exceeding max overall length */
Curl_dynhds_init(&hds, 128, 10);
fail_if(Curl_dynhds_add(&hds, "test1", 5, "123", 3), "add failed");
fail_unless(Curl_dynhds_add(&hds, "test2", 5, "456", 3), "should fail");
fail_if(Curl_dynhds_add(&hds, "t", 1, "1", 1), "add failed");
Curl_dynhds_reset(&hds);
Curl_dynhds_free(&hds);
Curl_dynhds_init(&hds, 128, 4*1024);
fail_if(Curl_dynhds_add(&hds, "test1", 5, "123", 3), "add failed");
fail_if(Curl_dynhds_add(&hds, "test1", 5, "123", 3), "add failed");
fail_if(Curl_dynhds_cadd(&hds, "blablabla", "thingies"), "add failed");
fail_if(Curl_dynhds_h1_cadd_line(&hds, "blablabla: thingies"), "add failed");
fail_unless(Curl_dynhds_ccount_name(&hds, "blablabla") == 2, "should");
fail_unless(Curl_dynhds_cremove(&hds, "blablabla") == 2, "should");
fail_if(Curl_dynhds_ccontains(&hds, "blablabla"), "should not");
result = Curl_dynhds_h1_cadd_line(&hds, "blablabla thingies");
fail_unless(result, "add should have failed");
if(!result) {
fail_unless(Curl_dynhds_ccount_name(&hds, "bLABlaBlA") == 0, "should");
fail_if(Curl_dynhds_cadd(&hds, "Bla-Bla", "thingies"), "add failed");
Curl_dyn_init(&dbuf, 32*1024);
fail_if(Curl_dynhds_h1_dprint(&hds, &dbuf), "h1 print failed");
if(Curl_dyn_ptr(&dbuf)) {
fprintf(stderr, "%s", Curl_dyn_ptr(&dbuf));
fail_if(strcmp(Curl_dyn_ptr(&dbuf),
"test1: 123\r\ntest1: 123\r\nBla-Bla: thingies\r\n"),
"h1 format differs");
}
Curl_dyn_free(&dbuf);
}
Curl_dynhds_free(&hds);
UNITTEST_STOP