romenskiy2012/curl
1
0
Fork 0
mirror of https://github.com/curl/curl.git synced 2026-08-02 09:30:29 +03:00
Code Issues Projects Releases Packages Wiki Activity

tool_urlglob: add named globs

Browse source 
Idea-by: Bastian Jesuiter

Verified by test 2408 - 2411

Closes #21409
This commit is contained in:
Daniel Stenberg 2026-04-22 11:38:02 +02:00
parent 2a2104f3cf
commit cb0636980b
No known key found for this signature in database
GPG key ID: 5CC908FDB71E12C2
13 changed files with 432 additions and 58 deletions
Download patch file Download diff file Expand all files Collapse all files

13
docs/cmdline-opts/_GLOBBING.md
View file

@ -39,4 +39,17 @@ probably have to put the full URL within double quotes to avoid the shell from
interfering with it. This also goes for other characters treated special, like
for example '&', '?' and '*'.
The separate globbing components can be referenced in the --output option to
allow pieces to be reused in the target filename.
Starting in curl 8.21.0, the separate globbing parts can be named and
referenced by their names. The case sensitive alphanumeric name is set
enclosed within angle brackets after the opening character. Examples:
https://fun.example/{<number>one,two,three}.jpg
ftp://ftp.example.com/file[<range>1-100].txt
Setting the same glob name twice is an error.
Switch off globbing with --globoff.

11
docs/cmdline-opts/output.md
View file

@ -69,3 +69,14 @@ override curl's internal binary output in terminal prevention:
Note that the binary output may be caused by the response being compressed, in
which case you may want to use the --compressed option.
Starting in curl 8.21.0, the separate globbing parts can be named and
referenced by their names. The case sensitive alphanumeric name is set
enclosed within angle brackets after the opening character. Examples:
curl "https://fun.example/{<num>one,two}.jpg" -o "save-#<num>"
curl "ftp://ftp.example/file[<range>1-100].txt" \
-o "save-#<range>.txt"
Referencing a named glob that is not set, causes an error.

7
src/tool_operate.c
View file

@ -1067,7 +1067,12 @@ static CURLcode setup_outfile(struct OperationConfig *config,
}
else if(result) {
/* bad globbing */
warnf("bad output glob");
if(state->urlglob.error) {
glob_show_error(&state->urlglob, u->outfile, tool_stderr, result);
config->synthetic_error = TRUE;
}
else
warnf("bad output glob");
return result;
}
if(!*per->outfile) {

188
src/tool_urlglob.c
View file

@ -56,6 +56,7 @@ static CURLcode glob_fixed(struct URLGlob *glob, char *fixed, size_t len)
pat->c.set.palloc = 1;
pat->c.set.size = 1;
pat->name = NULL; /* unnamed */
return CURLE_OK;
}
@ -89,7 +90,7 @@ static int multiply(curl_off_t *amount, curl_off_t with)
static CURLcode glob_set(struct URLGlob *glob, const char **patternp,
size_t *posp, curl_off_t *amount,
int globindex)
int globindex, const struct Curl_str *name)
{
/* processes a set expression with the point behind the opening '{'
','-separated elements are collected until the next closing '}'
@ -103,6 +104,7 @@ static CURLcode glob_set(struct URLGlob *glob, const char **patternp,
size_t size = 0;
char **elem = NULL;
size_t palloc = 0; /* start with this */
DEBUGASSERT(name);
while(!done) {
switch(*pattern) {
@ -198,6 +200,15 @@ static CURLcode glob_set(struct URLGlob *glob, const char **patternp,
pat->c.set.size = size;
pat->c.set.idx = 0;
pat->c.set.palloc = palloc;
if(curlx_strlen(name)) {
pat->name = curlx_memdup0(curlx_str(name), curlx_strlen(name));
if(!pat->name) {
result = CURLE_OUT_OF_MEMORY;
goto error;
}
}
else
pat->name = NULL; /* no name */
return CURLE_OK;
error:
@ -212,7 +223,7 @@ error:
static CURLcode glob_range(struct URLGlob *glob, const char **patternp,
size_t *posp, curl_off_t *amount,
int globindex)
int globindex, const struct Curl_str *name)
{
/* processes a range expression with the point behind the opening '['
- char range: e.g. "a-z]", "B-Q]"
@ -224,8 +235,10 @@ static CURLcode glob_range(struct URLGlob *glob, const char **patternp,
const char *pattern = *patternp;
const char *c;
DEBUGASSERT(name);
pat = &glob->pattern[glob->pnum];
pat->globindex = globindex;
pat->name = NULL; /* no name (so far) */
if(ISALPHA(*pattern)) {
/* character range detected */
@ -340,6 +353,11 @@ static CURLcode glob_range(struct URLGlob *glob, const char **patternp,
return globerror(glob, "bad range specification", *posp,
CURLE_URL_MALFORMAT);
if(curlx_strlen(name)) {
pat->name = curlx_memdup0(curlx_str(name), curlx_strlen(name));
if(!pat->name)
return CURLE_OUT_OF_MEMORY;
}
*patternp = pattern;
return CURLE_OK;
}
@ -407,12 +425,31 @@ static CURLcode add_glob(struct URLGlob *glob, size_t pos)
return CURLE_OK;
}
/* returns the named glob pattern (case sensitively) if it exists, otherwise
NULL
*/
static struct URLPattern *glob_find_name(struct URLGlob *glob,
struct Curl_str *name)
{
size_t i;
/* find the correct glob entry */
for(i = 0; i < glob->pnum; i++) {
if(glob->pattern[i].name &&
curlx_str_cmp(name, glob->pattern[i].name))
return &glob->pattern[i];
}
return NULL; /* no match */
}
#define MAX_GLOBNAME_LEN 64
static CURLcode glob_parse(struct URLGlob *glob, const char *pattern,
size_t pos, curl_off_t *amount)
{
/* processes a literal string component of a URL
special characters '{' and '[' branch to set/range processing functions
*/
const char *ipattern = pattern; /* start position */
CURLcode result = CURLE_OK;
int globindex = 0; /* count "actual" globs */
@ -464,21 +501,39 @@ static CURLcode glob_parse(struct URLGlob *glob, const char *pattern,
curlx_dyn_reset(&glob->buf);
}
else {
struct Curl_str name;
if(!*pattern) /* done */
break;
else if(*pattern == '{') {
/* process set pattern */
else if((*pattern == '{') || (*pattern == '[')) {
bool set = (*pattern == '{');
const char *start;
pattern++;
pos++;
result = glob_set(glob, &pattern, &pos, amount, globindex++);
if(!result)
result = add_glob(glob, pos);
}
else if(*pattern == '[') {
/* process range pattern */
pattern++;
pos++;
result = glob_range(glob, &pattern, &pos, amount, globindex++);
start = pattern;
/* fetch the name, if provided */
if(curlx_str_single(&pattern, '<') ||
curlx_str_until(&pattern, &name, MAX_GLOBNAME_LEN, '>') ||
curlx_str_single(&pattern, '>')) {
/* Not a proper name. This is not reporting errors on syntax errors
on purpose: it means that if there is an existing use case that
uses what looks like a broken named-glob syntax (now introduced)
we let that function like before. */
curlx_str_init(&name);
pattern = start; /* reset any partial patch */
}
else {
/* check that the name is not already used */
struct URLPattern *p = glob_find_name(glob, &name);
if(p)
return globerror(glob, "Duplicate glob name", 2 + start - ipattern,
CURLE_URL_MALFORMAT);
}
if(set)
result = glob_set(glob, &pattern, &pos, amount, globindex++, &name);
else
result = glob_range(glob, &pattern, &pos, amount, globindex++,
&name);
if(!result)
result = add_glob(glob, pos);
}
@ -492,6 +547,26 @@ bool glob_inuse(struct URLGlob *glob)
return glob->palloc ? TRUE : FALSE;
}
/* a glob error has been confirmed, this outputs details about it to the set
error stream */
void glob_show_error(struct URLGlob *glob, const char *url, FILE *error,
CURLcode result)
{
char text[512];
const char *t;
if(glob->pos) {
curl_msnprintf(text, sizeof(text), "%s in position %zu:\n%s\n%*s^",
glob->error,
glob->pos, url, (int)glob->pos - 1, " ");
t = text;
}
else
t = glob->error;
/* send error description to the error-stream */
curl_mfprintf(error, "curl: (%d) %s\n", result, t);
}
CURLcode glob_url(struct URLGlob *glob, const char *url, curl_off_t *urlnum,
FILE *error)
{
@ -511,21 +586,8 @@ CURLcode glob_url(struct URLGlob *glob, const char *url, curl_off_t *urlnum,
result = glob_parse(glob, url, 1, &amount);
if(result) {
if(error && glob->error) {
char text[512];
const char *t;
if(glob->pos) {
curl_msnprintf(text, sizeof(text), "%s in URL position %zu:\n%s\n%*s^",
glob->error,
glob->pos, url, (int)glob->pos - 1, " ");
t = text;
}
else
t = glob->error;
/* send error description to the error-stream */
curl_mfprintf(error, "curl: (%d) %s\n", result, t);
}
if(error && glob->error)
glob_show_error(glob, url, error, result);
*urlnum = 1;
return result;
}
@ -547,6 +609,7 @@ void glob_cleanup(struct URLGlob *glob)
curlx_safefree(glob->pattern[i].c.set.elem[elem]);
curlx_safefree(glob->pattern[i].c.set.elem);
}
curlx_safefree(glob->pattern[i].name);
}
curlx_safefree(glob->pattern);
glob->palloc = 0;
@ -643,6 +706,7 @@ CURLcode glob_match_url(char **output, const char *filename,
struct URLGlob *glob, SANITIZEcode *sc)
{
struct dynbuf dyn;
const char *ifilename = filename;
*output = NULL;
*sc = SANITIZE_ERR_OK;
@ -650,11 +714,11 @@ CURLcode glob_match_url(char **output, const char *filename,
while(*filename) {
CURLcode result = CURLE_OK;
struct URLPattern *pat = NULL;
if(*filename == '#' && ISDIGIT(filename[1])) {
const char *ptr = filename;
/* a numbered glob reference */
const char *ptr = filename++;
curl_off_t num;
struct URLPattern *pat = NULL;
filename++;
if(!curlx_str_number(&filename, &num, glob->pnum) && num) {
size_t i;
num--; /* make it zero based */
@ -666,31 +730,49 @@ CURLcode glob_match_url(char **output, const char *filename,
}
}
}
if(pat) {
switch(pat->type) {
case GLOB_SET:
if(pat->c.set.elem)
result = curlx_dyn_add(&dyn, pat->c.set.elem[pat->c.set.idx]);
break;
case GLOB_ASCII: {
char letter = (char)pat->c.ascii.letter;
result = curlx_dyn_addn(&dyn, &letter, 1);
break;
}
case GLOB_NUM:
result = curlx_dyn_addf(&dyn, "%0*" CURL_FORMAT_CURL_OFF_T,
pat->c.num.npad, pat->c.num.idx);
break;
default:
DEBUGASSERT(0);
if(!pat)
filename = ptr;
}
else if(*filename == '#' && (filename[1] == '<')) {
/* a named glob reference */
struct Curl_str name;
const char *ptr = filename;
filename += 2; /* pass both leading bytes */
if(!curlx_str_until(&filename, &name, MAX_GLOBNAME_LEN, '>') &&
!curlx_str_single(&filename, '>')) {
/* find the correct glob entry */
pat = glob_find_name(glob, &name);
if(!pat) {
/* when the name is given correctly, it needs to be an existing glob
name, which makes this an error */
curlx_dyn_free(&dyn);
return CURLE_FAILED_INIT;
return globerror(glob, "no glob exists with this name",
filename - ifilename, CURLE_BAD_FUNCTION_ARGUMENT);
}
}
else
/* #[num] out of range, use the #[num] in the output */
result = curlx_dyn_addn(&dyn, ptr, filename - ptr);
if(!pat)
filename = ptr;
}
if(pat) {
switch(pat->type) {
case GLOB_SET:
if(pat->c.set.elem)
result = curlx_dyn_add(&dyn, pat->c.set.elem[pat->c.set.idx]);
break;
case GLOB_ASCII: {
char letter = (char)pat->c.ascii.letter;
result = curlx_dyn_addn(&dyn, &letter, 1);
break;
}
case GLOB_NUM:
result = curlx_dyn_addf(&dyn, "%0*" CURL_FORMAT_CURL_OFF_T,
pat->c.num.npad, pat->c.num.idx);
break;
default:
DEBUGASSERT(0);
curlx_dyn_free(&dyn);
return CURLE_FAILED_INIT;
}
}
else
result = curlx_dyn_addn(&dyn, filename++, 1);

4
src/tool_urlglob.h
View file

@ -33,6 +33,7 @@ typedef enum {
struct URLPattern {
globtype type;
char *name; /* if not NULL */
int globindex; /* the number of this particular glob or -1 if not used
within {} or [] */
union {
@ -71,6 +72,9 @@ struct URLGlob {
size_t pos; /* column position of error or 0 */
};
void glob_show_error(struct URLGlob *glob, const char *url, FILE *error,
CURLcode result);
CURLcode glob_url(struct URLGlob *glob, const char *url, curl_off_t *urlnum,
FILE *error);
CURLcode glob_next_url(char **globbed, struct URLGlob *glob);

1
tests/data/Makefile.am
View file

@ -265,6 +265,7 @@ test2300 test2301 test2302 test2303 test2304 test2306 test2307 test2308 \
test2309 \
\
test2400 test2401 test2402 test2403 test2404 test2405 test2406 test2407 \
test2408 test2409 test2410 test2411 \
\
test2500 test2501 test2502 test2503 test2504 test2505 test2506 \
\

92
tests/data/test2408 Normal file
View file

@ -0,0 +1,92 @@
<?xml version="1.0" encoding="US-ASCII"?>
<testcase>
<info>
<keywords>
HTTP
HTTP GET
globbing
{} list
</keywords>
</info>
# Server-side
<reply>
<data crlf="headers" nocheck="yes">
HTTP/1.1 200 OK
Funny-head: yesyes
Content-Length: 4
moo
</data>
<data2 crlf="headers">
HTTP/1.1 200 OK
Funny-head: yesyes
Content-Length: 4
foo
</data2>
<data3 crlf="headers">
HTTP/1.1 200 OK
Funny-head: yesyes
Content-Length: 4
hoo
</data3>
</reply>
# Client-side
<client>
<server>
http
</server>
<name>
multiple requests using named {} globs in URL
</name>
<command option="no-output">
"%HOSTIP:%HTTPPORT/{%LTtest%GT%TESTNUMBER,%TESTNUMBER0002,%TESTNUMBER0003}" -o "%LOGDIR/dump-#%LTtest%GT"
</command>
</client>
# Verify data after the test has been "shot"
<verify>
<protocol crlf="headers">
GET /%TESTNUMBER HTTP/1.1
Host: %HOSTIP:%HTTPPORT
User-Agent: curl/%VERSION
Accept: */*
GET /%TESTNUMBER0002 HTTP/1.1
Host: %HOSTIP:%HTTPPORT
User-Agent: curl/%VERSION
Accept: */*
GET /%TESTNUMBER0003 HTTP/1.1
Host: %HOSTIP:%HTTPPORT
User-Agent: curl/%VERSION
Accept: */*
</protocol>
<file name="%LOGDIR/dump-%TESTNUMBER" crlf="headers">
HTTP/1.1 200 OK
Funny-head: yesyes
Content-Length: 4
moo
</file>
<file2 name="%LOGDIR/dump-%TESTNUMBER0002" crlf="headers">
HTTP/1.1 200 OK
Funny-head: yesyes
Content-Length: 4
foo
</file2>
<file3 name="%LOGDIR/dump-%TESTNUMBER0003" crlf="headers">
HTTP/1.1 200 OK
Funny-head: yesyes
Content-Length: 4
hoo
</file3>
</verify>
</testcase>

92
tests/data/test2409 Normal file
View file

@ -0,0 +1,92 @@
<?xml version="1.0" encoding="US-ASCII"?>
<testcase>
<info>
<keywords>
HTTP
HTTP GET
globbing
{} list
</keywords>
</info>
# Server-side
<reply>
<data crlf="headers" nocheck="yes">
HTTP/1.1 200 swsbounce
Funny-head: yesyes
Content-Length: 4
moo
</data>
<data1 crlf="headers">
HTTP/1.1 200 swsbounce
Funny-head: yesyes
Content-Length: 4
foo
</data1>
<data2 crlf="headers">
HTTP/1.1 200 OK
Funny-head: yesyes
Content-Length: 4
hoo
</data2>
</reply>
# Client-side
<client>
<server>
http
</server>
<name>
multiple requests using named [] globs in URL
</name>
<command option="no-output">
"%HOSTIP:%HTTPPORT/hello[%LTtest%GT7-9]" -o "%LOGDIR/dump-#%LTtest%GT"
</command>
</client>
# Verify data after the test has been "shot"
<verify>
<protocol crlf="headers">
GET /hello7 HTTP/1.1
Host: %HOSTIP:%HTTPPORT
User-Agent: curl/%VERSION
Accept: */*
GET /hello8 HTTP/1.1
Host: %HOSTIP:%HTTPPORT
User-Agent: curl/%VERSION
Accept: */*
GET /hello9 HTTP/1.1
Host: %HOSTIP:%HTTPPORT
User-Agent: curl/%VERSION
Accept: */*
</protocol>
<file name="%LOGDIR/dump-7" crlf="headers">
HTTP/1.1 200 swsbounce
Funny-head: yesyes
Content-Length: 4
moo
</file>
<file2 name="%LOGDIR/dump-8" crlf="headers">
HTTP/1.1 200 swsbounce
Funny-head: yesyes
Content-Length: 4
foo
</file2>
<file3 name="%LOGDIR/dump-9" crlf="headers">
HTTP/1.1 200 OK
Funny-head: yesyes
Content-Length: 4
hoo
</file3>
</verify>
</testcase>

37
tests/data/test2410 Normal file
View file

@ -0,0 +1,37 @@
<?xml version="1.0" encoding="US-ASCII"?>
<testcase>
<info>
<keywords>
HTTP
HTTP GET
globbing
{} list
</keywords>
</info>
# Server-side
# Client-side
<client>
<server>
http
</server>
<name>
duplicate named glob
</name>
<command option="no-output">
"%HOSTIP:%HTTPPORT/{%LTtest%GTA,B}{%LTtest%GTC,D}" -o "%LOGDIR/dump"
</command>
</client>
# Verify data after the test has been "shot"
<verify>
<stderr mode="text">
curl: (3) Duplicate glob name in position 30:
%HOSTIP:%HTTPPORT/{%LTtest%GTA,B}{%LTtest%GTC,D}
^
</stderr>
<errorcode>
3
</errorcode>
</verify>
</testcase>

37
tests/data/test2411 Normal file
View file

@ -0,0 +1,37 @@
<?xml version="1.0" encoding="US-ASCII"?>
<testcase>
<info>
<keywords>
HTTP
HTTP GET
globbing
{} list
</keywords>
</info>
# Server-side
# Client-side
<client>
<server>
http
</server>
<name>
reference a named glob not set
</name>
<command option="no-output">
"%HOSTIP:%HTTPPORT/{%LTtest%GTA,B}{%LTmoo%GTC,D}" -o "somewhere/#%LTfoo%GT"
</command>
</client>
# Verify data after the test has been "shot"
<verify>
<stderr mode="text">
curl: (43) no glob exists with this name in position 16:
somewhere/#%LTfoo%GT
^
</stderr>
<errorcode>
43
</errorcode>
</verify>
</testcase>

2
tests/data/test75
View file

@ -31,7 +31,7 @@ HTTP, urlglob retrieval with bad range
3
</errorcode>
<stdout mode="text">
curl: (3) bad range in URL position 47:
curl: (3) bad range in position 47:
http://a-site-never-accessed.example.org/[2-1]
^
</stdout>

2
tests/data/test759
View file

@ -18,7 +18,7 @@ glob '{,'
# Verify data after the test has been "shot"
<verify>
# curl: (3) unmatched brace in URL position 1:
# curl: (3) unmatched brace in position 1:
<errorcode>
3
</errorcode>

4
tests/data/test761
View file

@ -22,8 +22,8 @@ http://testingthis/%repeat[201 x {a}b]%
3
</errorcode>
<stderr mode="text">
curl: (3) too many {} sets in URL position 403:
http://testingthis/%repeat[113 x {a}b]%{a
curl: (3) too many {} sets in position 403:
http://testingthis/%repeat[114 x {a}b]%{a
</stderr>
</verify>
</testcase>