romenskiy2012/curl
1
0
Fork 0
mirror of https://github.com/curl/curl.git synced 2026-08-26 23:04:24 +03:00
Code Issues Projects Releases Packages Wiki Activity

websockets: remodeled API to support 63 bit frame sizes

Browse source 
curl_ws_recv() now receives data to fill up the provided buffer, but can
return a partial fragment. The function now also get a pointer to a
curl_ws_frame struct with metadata that also mentions the offset and
total size of the fragment (of which you might be receiving a smaller
piece). This way, large incoming fragments will be "streamed" to the
application. When the curl_ws_frame struct field 'bytesleft' is 0, the
final fragment piece has been delivered.

curl_ws_recv() was also adjusted to work with a buffer size smaller than
the fragment size. (Possibly needless to say as the fragment size can
now be 63 bit large).

curl_ws_send() now supports sending a piece of a fragment, in a
streaming manner, in addition to sending the entire fragment in a single
call if it is small enough. To send a huge fragment, curl_ws_send() can
be used to send it in many small calls by first telling libcurl about
the total expected fragment size, and then send the payload in N number
of separate invokes and libcurl will stream those over the wire.

The struct curl_ws_meta() returns is now called 'curl_ws_frame' and it
has been extended with two new fields: *offset* and *bytesleft*. To help
describe the passed on data chunk when a fragment is delivered in many
smaller pieces.

The documentation has been updated accordingly.

Closes #9636
This commit is contained in:
Daniel Stenberg 2022-10-03 17:40:02 +02:00
parent 83de62babc
commit e3f335148a
No known key found for this signature in database
GPG key ID: 5CC908FDB71E12C2
12 changed files with 325 additions and 172 deletions
Download patch file Download diff file Expand all files Collapse all files

4
lib/easy.c
View file

@ -1235,7 +1235,7 @@ CURLcode curl_easy_recv(struct Curl_easy *data, void *buffer, size_t buflen,
* This is the private internal version of curl_easy_send()
*/
CURLcode Curl_senddata(struct Curl_easy *data, const void *buffer,
size_t buflen, size_t *n)
size_t buflen, ssize_t *n)
{
curl_socket_t sfd;
CURLcode result;
@ -1279,7 +1279,7 @@ CURLcode curl_easy_send(struct Curl_easy *data, const void *buffer,
if(Curl_is_in_callback(data))
return CURLE_RECURSIVE_API_CALL;
return Curl_senddata(data, buffer, buflen, n);
return Curl_senddata(data, buffer, buflen, (ssize_t *)n);
}
/*

2
lib/easyif.h
View file

@ -28,7 +28,7 @@
* Prototypes for library-wide functions provided by easy.c
*/
CURLcode Curl_senddata(struct Curl_easy *data, const void *buffer,
size_t buflen, size_t *n);
size_t buflen, ssize_t *n);
#ifdef CURLDEBUG
CURL_EXTERN CURLcode curl_easy_perform_ev(struct Curl_easy *easy);

13
lib/http.h
View file

@ -202,17 +202,6 @@ struct h3out; /* see ngtcp2 */
#endif /* _WIN32 */
#endif /* USE_MSH3 */
struct websockets {
bool contfragment; /* set TRUE if the previous fragment sent was not final */
unsigned char mask[4]; /* 32 bit mask for this connection */
struct Curl_easy *data; /* used for write callback handling */
struct dynbuf buf;
size_t usedbuf; /* number of leading bytes in 'buf' the most recent complete
websocket frame uses */
struct curl_ws_metadata handout; /* the struct storage used for
curl_ws_meta() returns */
};
/****************************************************************************
* HTTP unique setup
***************************************************************************/
@ -240,7 +229,7 @@ struct HTTP {
} sending;
#ifdef USE_WEBSOCKETS
struct websockets ws;
struct websocket ws;
#endif
#ifndef CURL_DISABLE_HTTP

323
lib/ws.c
View file

@ -120,6 +120,7 @@ CURLcode Curl_ws_accept(struct Curl_easy *data)
struct SingleRequest *k = &data->req;
struct HTTP *ws = data->req.p.http;
struct connectdata *conn = data->conn;
struct websocket *wsp = &data->req.p.http->ws;
CURLcode result;
/* Verify the Sec-WebSocket-Accept response.
@ -146,7 +147,7 @@ CURLcode Curl_ws_accept(struct Curl_easy *data)
if(result)
return result;
infof(data, "Received 101, switch to WebSockets; mask %02x%02x%02x%02x",
infof(data, "Received 101, switch to WebSocket; mask %02x%02x%02x%02x",
ws->ws.mask[0], ws->ws.mask[1], ws->ws.mask[2], ws->ws.mask[3]);
k->upgr101 = UPGR101_RECEIVED;
@ -154,6 +155,7 @@ CURLcode Curl_ws_accept(struct Curl_easy *data)
/* switch off non-blocking sockets */
(void)curlx_nonblock(conn->sock[FIRSTSOCKET], FALSE);
wsp->oleft = 0;
return result;
}
@ -172,7 +174,7 @@ CURLcode Curl_ws_accept(struct Curl_easy *data)
now been delivered to the application */
static void ws_decode_clear(struct Curl_easy *data)
{
struct websockets *wsp = &data->req.p.http->ws;
struct websocket *wsp = &data->req.p.http->ws;
size_t spent = wsp->usedbuf;
size_t len = Curl_dyn_len(&wsp->buf);
size_t keep = len - spent;
@ -186,6 +188,7 @@ static void ws_decode_clear(struct Curl_easy *data)
ilen - the size of the provided data, perhaps too little, perhaps too much
out - stored pointed to extracted data
olen - stored length of the extracted data
oleft - number of unread bytes pending to that belongs to this frame
endp - stored pointer to data immediately following the parsed data, if
there is more data in there. NULL if there's no more data.
flags - stored bitmask about the frame
@ -198,16 +201,17 @@ static void ws_decode_clear(struct Curl_easy *data)
static CURLcode ws_decode(struct Curl_easy *data,
unsigned char *wpkt, size_t ilen,
unsigned char **out, size_t *olen,
curl_off_t *oleft,
unsigned char **endp,
unsigned int *flags)
{
bool fin;
unsigned char opcode;
size_t total;
curl_off_t total;
size_t dataindex = 2;
size_t plen; /* size of data in the buffer */
size_t payloadssize;
struct websockets *wsp = &data->req.p.http->ws;
curl_off_t plen; /* size of data in the buffer */
curl_off_t payloadsize;
struct websocket *wsp = &data->req.p.http->ws;
unsigned char *p;
CURLcode result;
@ -266,36 +270,52 @@ static CURLcode ws_decode(struct Curl_easy *data,
failf(data, "WS: masked input frame");
return CURLE_RECV_ERROR;
}
payloadssize = p[1];
if(payloadssize == 126) {
payloadsize = p[1];
if(payloadsize == 126) {
if(plen < 4) {
infof(data, "WS:%d plen == %u, EAGAIN", __LINE__, (int)plen);
return CURLE_AGAIN; /* not enough data available */
}
payloadssize = (p[2] << 8) | p[3];
payloadsize = (p[2] << 8) | p[3];
dataindex += 2;
}
else if(payloadssize == 127) {
failf(data, "WS: too large frame received");
return CURLE_RECV_ERROR;
else if(payloadsize == 127) {
/* 64 bit payload size */
if(plen < 10)
return CURLE_AGAIN;
if(p[2] & 80) {
failf(data, "WS: too large frame");
return CURLE_RECV_ERROR;
}
dataindex += 8;
payloadsize = ((curl_off_t)p[2] << 56) |
(curl_off_t)p[3] << 48 |
(curl_off_t)p[4] << 40 |
(curl_off_t)p[5] << 32 |
(curl_off_t)p[6] << 24 |
(curl_off_t)p[7] << 16 |
(curl_off_t)p[8] << 8 |
p[9];
}
total = dataindex + payloadssize;
total = dataindex + payloadsize;
if(total > plen) {
/* not enough data in buffer yet */
infof(data, "WS:%d plen == %u (%u), EAGAIN", __LINE__, (int)plen,
(int)total);
return CURLE_AGAIN;
/* deliver a partial frame */
*oleft = total - dataindex;
payloadsize = total - dataindex;
}
else
*oleft = 0;
/* point to the payload */
*out = &p[dataindex];
/* return the payload length */
*olen = payloadssize;
*olen = payloadsize;
wsp->usedbuf = total; /* number of bytes "used" from the buffer */
*endp = &p[total];
infof(data, "WS: received %zu bytes payload", payloadssize);
infof(data, "WS: received %zu bytes payload (%zu left)",
payloadsize, *oleft);
return CURLE_OK;
}
@ -312,40 +332,52 @@ size_t Curl_ws_writecb(char *buffer, size_t size /* 1 */,
if(data->set.ws_raw_mode)
return data->set.fwrite_func(buffer, size, nitems, writebody_ptr);
else if(nitems) {
unsigned char *wsp;
size_t wslen;
unsigned char *frame;
size_t flen;
unsigned int recvflags;
CURLcode result;
unsigned char *endp;
curl_off_t oleft;
decode:
result = ws_decode(data, (unsigned char *)buffer, nitems,
&wsp, &wslen, &endp, &recvflags);
if(result == CURLE_AGAIN)
/* insufficient amount of data, keep it for later */
return nitems;
else if(result) {
infof(data, "WS: decode error %d", (int)result);
return nitems - 1;
oleft = ws->ws.frame.bytesleft;
if(!oleft) {
result = ws_decode(data, (unsigned char *)buffer, nitems,
&frame, &flen, &oleft, &endp, &recvflags);
if(result == CURLE_AGAIN)
/* insufficient amount of data, keep it for later */
return nitems;
else if(result) {
infof(data, "WS: decode error %d", (int)result);
return nitems - 1;
}
/* Store details about the frame to be reachable with curl_ws_meta()
from within the write callback */
ws->ws.frame.age = 0;
ws->ws.frame.offset = 0;
ws->ws.frame.flags = recvflags;
ws->ws.frame.bytesleft = oleft;
}
else {
ws->ws.frame.bytesleft -= nitems;
}
/* auto-respond to PINGs */
if(recvflags & CURLWS_PING) {
if((recvflags & CURLWS_PING) && !oleft) {
size_t bytes;
infof(data, "WS: auto-respond to PING with a PONG");
/* send back the exact same content as a PONG */
result = curl_ws_send(data, wsp, wslen, &bytes, CURLWS_PONG);
result = curl_ws_send(data, frame, flen, &bytes, 0, CURLWS_PONG);
if(result)
return result;
}
else {
/* Store details about the frame to be reachable with curl_ws_meta()
from within the write callback */
ws->ws.handout.age = 0;
ws->ws.handout.recvflags = recvflags;
/* deliver the decoded frame to the user callback */
if(data->set.fwrite_func((char *)wsp, 1, wslen, writebody_ptr) != wslen)
if(data->set.fwrite_func((char *)frame, 1, flen, writebody_ptr) != flen)
return 0;
}
if(oleft)
ws->ws.frame.offset += flen;
/* the websocket frame has been delivered */
ws_decode_clear(data);
if(endp) {
@ -359,43 +391,70 @@ size_t Curl_ws_writecb(char *buffer, size_t size /* 1 */,
CURL_EXTERN CURLcode curl_ws_recv(struct Curl_easy *data, void *buffer,
size_t buflen,
size_t *nread, unsigned int *recvflags)
size_t buflen, size_t *nread,
struct curl_ws_frame **metap)
{
size_t bytes;
CURLcode result;
struct websocket *wsp = &data->req.p.http->ws;
*nread = 0;
*recvflags = 0;
*metap = NULL;
/* get a download buffer */
result = Curl_preconnect(data);
if(result)
return result;
do {
result = curl_easy_recv(data, data->state.buffer,
data->set.buffer_size, &bytes);
if(result)
return result;
bool drain = FALSE; /* if there is pending buffered data to drain */
char *inbuf = data->state.buffer;
bytes = wsp->stillbuffer;
if(!bytes) {
result = curl_easy_recv(data, data->state.buffer,
data->set.buffer_size, &bytes);
if(result)
return result;
}
else {
/* the pending bytes can be found here */
inbuf = wsp->stillb;
drain = TRUE;
}
if(bytes) {
unsigned char *out;
size_t olen;
unsigned char *endp;
unsigned int recvflags;
curl_off_t oleft = wsp->frame.bytesleft;
infof(data, "WS: got %u websocket bytes to decode", (int)bytes);
result = ws_decode(data, (unsigned char *)data->state.buffer,
bytes, &out, &olen, &endp, recvflags);
if(result == CURLE_AGAIN)
/* a packet fragment only */
break;
else if(result)
return result;
if(!oleft && !drain) {
result = ws_decode(data, (unsigned char *)inbuf, bytes,
&out, &olen, &oleft, &endp, &recvflags);
if(result == CURLE_AGAIN)
/* a packet fragment only */
break;
else if(result)
return result;
wsp->frame.offset = 0;
wsp->frame.bytesleft = oleft;
wsp->frame.flags = recvflags;
}
else {
olen = oleft;
recvflags = wsp->frame.flags;
if((curl_off_t)buflen < oleft)
/* there is still data left after this */
wsp->frame.bytesleft -= buflen;
else
wsp->frame.bytesleft = 0;
}
/* auto-respond to PINGs */
if(*recvflags & CURLWS_PING) {
if((recvflags & CURLWS_PING) && !oleft) {
infof(data, "WS: auto-respond to PING with a PONG");
/* send back the exact same content as a PONG */
result = curl_ws_send(data, out, olen, &bytes, CURLWS_PONG);
result = curl_ws_send(data, out, olen, &bytes, 0, CURLWS_PONG);
if(result)
return result;
}
@ -404,24 +463,44 @@ CURL_EXTERN CURLcode curl_ws_recv(struct Curl_easy *data, void *buffer,
/* copy the payload to the user buffer */
memcpy(buffer, out, olen);
*nread = olen;
if(!oleft)
/* websocket frame has been delivered */
ws_decode_clear(data);
}
else {
/* Received a larger websocket frame than what could fit in the user
provided buffer! */
infof(data, "WS: too large websocket frame received");
return CURLE_RECV_ERROR;
/* copy a partial payload */
memcpy(buffer, out, buflen);
*nread = buflen;
/* remember what is left and where */
wsp->stillbuffer = olen - buflen;
wsp->stillb = (char *)buffer + buflen;
}
wsp->frame.offset += *nread;
}
/* the websocket frame has been delivered */
ws_decode_clear(data);
}
else
*nread = bytes;
break;
} while(1);
*metap = &wsp->frame;
return CURLE_OK;
}
static void ws_xor(struct Curl_easy *data,
const unsigned char *source,
unsigned char *dest,
size_t len)
{
struct websocket *wsp = &data->req.p.http->ws;
size_t i;
/* append payload after the mask, XOR appropriately */
for(i = 0; i < len; i++) {
dest[i] = source[i] ^ wsp->mask[wsp->xori];
wsp->xori++;
wsp->xori &= 3;
}
}
/***
RFC 6455 Section 5.2
@ -445,17 +524,14 @@ CURL_EXTERN CURLcode curl_ws_recv(struct Curl_easy *data, void *buffer,
+---------------------------------------------------------------+
*/
static size_t ws_packet(struct Curl_easy *data,
const unsigned char *payload, size_t len,
unsigned int flags)
static size_t ws_packethead(struct Curl_easy *data,
size_t len, unsigned int flags)
{
struct HTTP *ws = data->req.p.http;
unsigned char *out = (unsigned char *)data->state.ulbuf;
unsigned char firstbyte = 0;
int outi;
unsigned char opcode;
unsigned int xori;
unsigned int i;
if(flags & CURLWS_TEXT) {
opcode = WSBIT_OPCODE_TEXT;
infof(data, "WS: send OPCODE TEXT");
@ -491,8 +567,13 @@ static size_t ws_packet(struct Curl_easy *data,
ws->ws.contfragment = TRUE;
}
out[0] = firstbyte;
if(len > 65535) {
out[1] = 127 | WSBIT_MASK;
out[2] = (len >> 8) & 0xff;
out[3] = len & 0xff;
outi = 10;
}
if(len > 126) {
/* no support for > 16 bit fragment sizes */
out[1] = 126 | WSBIT_MASK;
out[2] = (len >> 8) & 0xff;
out[3] = len & 0xff;
@ -517,112 +598,138 @@ static size_t ws_packet(struct Curl_easy *data,
/* pass over the mask */
outi += 4;
/* append payload after the mask, XOR appropriately */
for(i = 0, xori = 0; i < len; i++, outi++) {
out[outi] = payload[i] ^ ws->ws.mask[xori];
xori++;
xori &= 3;
}
ws->ws.xori = 0;
/* return packet size */
return outi;
}
CURL_EXTERN CURLcode curl_ws_send(struct Curl_easy *data, const void *buffer,
size_t buflen, size_t *sent,
curl_off_t totalsize,
unsigned int sendflags)
{
size_t bytes;
CURLcode result;
size_t plen;
size_t headlen;
char *out;
if(buflen > MAX_WS_SIZE) {
failf(data, "too large packet");
return CURLE_BAD_FUNCTION_ARGUMENT;
}
ssize_t written;
struct websocket *wsp = &data->req.p.http->ws;
if(!data->set.ws_raw_mode) {
result = Curl_get_upload_buffer(data);
if(result)
return result;
}
else {
if(totalsize || sendflags)
return CURLE_BAD_FUNCTION_ARGUMENT;
}
if(Curl_is_in_callback(data)) {
ssize_t written;
if(data->set.ws_raw_mode) {
/* raw mode sends exactly what was requested, and this is from within
the write callback */
if(data->set.ws_raw_mode) {
if(!buflen)
/* nothing to do */
return CURLE_OK;
/* raw mode sends exactly what was requested, and this is from within
the write callback */
if(Curl_is_in_callback(data))
result = Curl_write(data, data->conn->writesockfd, buffer, buflen,
&written);
infof(data, "WS: wanted to send %u bytes, sent %u bytes",
(int)buflen, (int)written);
else
result = Curl_senddata(data, buffer, buflen, &written);
infof(data, "WS: wanted to send %zu bytes, sent %zu bytes",
buflen, written);
*sent = written;
return result;
}
if(buflen > (data->set.upload_buffer_size - 10))
/* don't do more than this in one go */
buflen = data->set.upload_buffer_size - 10;
if(sendflags & CURLWS_OFFSET) {
if(totalsize) {
/* a frame series 'totalsize' bytes big, this is the first */
headlen = ws_packethead(data, totalsize, sendflags);
wsp->sleft = totalsize - buflen;
}
else {
plen = ws_packet(data, buffer, buflen, sendflags);
out = data->state.ulbuf;
result = Curl_write(data, data->conn->writesockfd, out, plen,
&written);
infof(data, "WS: wanted to send %u bytes, sent %u bytes",
(int)plen, (int)written);
headlen = 0;
if((curl_off_t)buflen > wsp->sleft) {
infof(data, "WS: unaligned frame size (sending %zu instead of %zu)",
buflen, wsp->sleft);
wsp->sleft = 0;
}
else
wsp->sleft -= buflen;
}
bytes = written;
}
else {
plen = ws_packet(data, buffer, buflen, sendflags);
else
headlen = ws_packethead(data, buflen, sendflags);
out = data->state.ulbuf;
result = Curl_senddata(data, out, plen, &bytes);
(void)sendflags;
}
*sent = bytes;
/* headlen is the size of the frame header */
out = data->state.ulbuf;
if(buflen)
/* for PING and PONG etc there might not be a payload */
ws_xor(data, buffer, (unsigned char *)out + headlen, buflen - headlen);
if(Curl_is_in_callback(data))
result = Curl_write(data, data->conn->writesockfd, out,
buflen + headlen, &written);
else
result = Curl_senddata(data, out, buflen + headlen, &written);
infof(data, "WS: wanted to send %zu bytes, sent %zu bytes",
headlen + buflen, written);
*sent = written;
return result;
}
void Curl_ws_done(struct Curl_easy *data)
{
struct websockets *wsp = &data->req.p.http->ws;
struct websocket *wsp = &data->req.p.http->ws;
DEBUGASSERT(wsp);
Curl_dyn_free(&wsp->buf);
}
CURL_EXTERN struct curl_ws_metadata *curl_ws_meta(struct Curl_easy *data)
CURL_EXTERN struct curl_ws_frame *curl_ws_meta(struct Curl_easy *data)
{
/* we only return something for websockets, called from within the callback
/* we only return something for websocket, called from within the callback
when not using raw mode */
if(GOOD_EASY_HANDLE(data) && Curl_is_in_callback(data) && data->req.p.http &&
!data->set.ws_raw_mode)
return &data->req.p.http->ws.handout;
return &data->req.p.http->ws.frame;
return NULL;
}
#else
CURL_EXTERN CURLcode curl_ws_recv(CURL *curl, void *buffer, size_t buflen,
size_t *nread, unsigned int *recvflags)
size_t *nread,
struct curl_ws_frame **metap)
{
(void)curl;
(void)buffer;
(void)buflen;
(void)nread;
(void)recvflags;
(void)metap;
return CURLE_OK;
}
CURL_EXTERN CURLcode curl_ws_send(CURL *curl, const void *buffer,
size_t buflen, size_t *sent,
curl_off_t framesize,
unsigned int sendflags)
{
(void)curl;
(void)buffer;
(void)buflen;
(void)sent;
(void)framesize;
(void)sendflags;
return CURLE_OK;
}
CURL_EXTERN struct curl_ws_metadata *curl_ws_meta(struct Curl_easy *data)
CURL_EXTERN struct curl_ws_frame *curl_ws_meta(struct Curl_easy *data)
{
(void)data;
return NULL;

19
lib/ws.h
View file

@ -36,6 +36,25 @@
/* this is the largest single fragment size we support */
#define MAX_WS_SIZE 65535
/* part of 'struct HTTP', when used in the 'struct SingleRequest' in the
Curl_easy struct */
struct websocket {
bool contfragment; /* set TRUE if the previous fragment sent was not final */
unsigned char mask[4]; /* 32 bit mask for this connection */
struct Curl_easy *data; /* used for write callback handling */
struct dynbuf buf;
size_t usedbuf; /* number of leading bytes in 'buf' the most recent complete
websocket frame uses */
struct curl_ws_frame frame; /* the struct used for frame state */
curl_off_t oleft; /* outstanding number of payload bytes left from the
server */
curl_off_t stillbuffer; /* number of bytes left in the buffer to deliver in
the next curl_ws_recv() call */
char *stillb; /* the stillbuffer pending bytes are here */
curl_off_t sleft; /* outstanding number of payload bytes left to send */
unsigned int xori; /* xor index */
};
CURLcode Curl_ws_request(struct Curl_easy *data, REQTYPE *req);
CURLcode Curl_ws_accept(struct Curl_easy *data);