diff --git a/docs/cmdline-opts/_GLOBBING.md b/docs/cmdline-opts/_GLOBBING.md index 37c8d43069..b801adb4d1 100644 --- a/docs/cmdline-opts/_GLOBBING.md +++ b/docs/cmdline-opts/_GLOBBING.md @@ -39,4 +39,17 @@ probably have to put the full URL within double quotes to avoid the shell from interfering with it. This also goes for other characters treated special, like for example '&', '?' and '*'. +The separate globbing components can be referenced in the --output option to +allow pieces to be reused in the target filename. + +Starting in curl 8.21.0, the separate globbing parts can be named and +referenced by their names. The case sensitive alphanumeric name is set +enclosed within angle brackets after the opening character. Examples: + + https://fun.example/{one,two,three}.jpg + + ftp://ftp.example.com/file[1-100].txt + +Setting the same glob name twice is an error. + Switch off globbing with --globoff. diff --git a/docs/cmdline-opts/output.md b/docs/cmdline-opts/output.md index 0c4f7f9fac..c1d823e324 100644 --- a/docs/cmdline-opts/output.md +++ b/docs/cmdline-opts/output.md @@ -69,3 +69,14 @@ override curl's internal binary output in terminal prevention: Note that the binary output may be caused by the response being compressed, in which case you may want to use the --compressed option. + +Starting in curl 8.21.0, the separate globbing parts can be named and +referenced by their names. The case sensitive alphanumeric name is set +enclosed within angle brackets after the opening character. Examples: + + curl "https://fun.example/{one,two}.jpg" -o "save-#" + + curl "ftp://ftp.example/file[1-100].txt" \ + -o "save-#.txt" + +Referencing a named glob that is not set, causes an error. diff --git a/src/tool_operate.c b/src/tool_operate.c index 3ad30ca4c4..62d40afd55 100644 --- a/src/tool_operate.c +++ b/src/tool_operate.c @@ -1067,7 +1067,12 @@ static CURLcode setup_outfile(struct OperationConfig *config, } else if(result) { /* bad globbing */ - warnf("bad output glob"); + if(state->urlglob.error) { + glob_show_error(&state->urlglob, u->outfile, tool_stderr, result); + config->synthetic_error = TRUE; + } + else + warnf("bad output glob"); return result; } if(!*per->outfile) { diff --git a/src/tool_urlglob.c b/src/tool_urlglob.c index d2249980e2..305efdeecc 100644 --- a/src/tool_urlglob.c +++ b/src/tool_urlglob.c @@ -56,6 +56,7 @@ static CURLcode glob_fixed(struct URLGlob *glob, char *fixed, size_t len) pat->c.set.palloc = 1; pat->c.set.size = 1; + pat->name = NULL; /* unnamed */ return CURLE_OK; } @@ -89,7 +90,7 @@ static int multiply(curl_off_t *amount, curl_off_t with) static CURLcode glob_set(struct URLGlob *glob, const char **patternp, size_t *posp, curl_off_t *amount, - int globindex) + int globindex, const struct Curl_str *name) { /* processes a set expression with the point behind the opening '{' ','-separated elements are collected until the next closing '}' @@ -103,6 +104,7 @@ static CURLcode glob_set(struct URLGlob *glob, const char **patternp, size_t size = 0; char **elem = NULL; size_t palloc = 0; /* start with this */ + DEBUGASSERT(name); while(!done) { switch(*pattern) { @@ -198,6 +200,15 @@ static CURLcode glob_set(struct URLGlob *glob, const char **patternp, pat->c.set.size = size; pat->c.set.idx = 0; pat->c.set.palloc = palloc; + if(curlx_strlen(name)) { + pat->name = curlx_memdup0(curlx_str(name), curlx_strlen(name)); + if(!pat->name) { + result = CURLE_OUT_OF_MEMORY; + goto error; + } + } + else + pat->name = NULL; /* no name */ return CURLE_OK; error: @@ -212,7 +223,7 @@ error: static CURLcode glob_range(struct URLGlob *glob, const char **patternp, size_t *posp, curl_off_t *amount, - int globindex) + int globindex, const struct Curl_str *name) { /* processes a range expression with the point behind the opening '[' - char range: e.g. "a-z]", "B-Q]" @@ -224,8 +235,10 @@ static CURLcode glob_range(struct URLGlob *glob, const char **patternp, const char *pattern = *patternp; const char *c; + DEBUGASSERT(name); pat = &glob->pattern[glob->pnum]; pat->globindex = globindex; + pat->name = NULL; /* no name (so far) */ if(ISALPHA(*pattern)) { /* character range detected */ @@ -340,6 +353,11 @@ static CURLcode glob_range(struct URLGlob *glob, const char **patternp, return globerror(glob, "bad range specification", *posp, CURLE_URL_MALFORMAT); + if(curlx_strlen(name)) { + pat->name = curlx_memdup0(curlx_str(name), curlx_strlen(name)); + if(!pat->name) + return CURLE_OUT_OF_MEMORY; + } *patternp = pattern; return CURLE_OK; } @@ -407,12 +425,31 @@ static CURLcode add_glob(struct URLGlob *glob, size_t pos) return CURLE_OK; } +/* returns the named glob pattern (case sensitively) if it exists, otherwise + NULL +*/ +static struct URLPattern *glob_find_name(struct URLGlob *glob, + struct Curl_str *name) +{ + size_t i; + /* find the correct glob entry */ + for(i = 0; i < glob->pnum; i++) { + if(glob->pattern[i].name && + curlx_str_cmp(name, glob->pattern[i].name)) + return &glob->pattern[i]; + } + return NULL; /* no match */ +} + +#define MAX_GLOBNAME_LEN 64 + static CURLcode glob_parse(struct URLGlob *glob, const char *pattern, size_t pos, curl_off_t *amount) { /* processes a literal string component of a URL special characters '{' and '[' branch to set/range processing functions */ + const char *ipattern = pattern; /* start position */ CURLcode result = CURLE_OK; int globindex = 0; /* count "actual" globs */ @@ -464,21 +501,39 @@ static CURLcode glob_parse(struct URLGlob *glob, const char *pattern, curlx_dyn_reset(&glob->buf); } else { + struct Curl_str name; if(!*pattern) /* done */ break; - else if(*pattern == '{') { - /* process set pattern */ + else if((*pattern == '{') || (*pattern == '[')) { + bool set = (*pattern == '{'); + const char *start; pattern++; pos++; - result = glob_set(glob, &pattern, &pos, amount, globindex++); - if(!result) - result = add_glob(glob, pos); - } - else if(*pattern == '[') { - /* process range pattern */ - pattern++; - pos++; - result = glob_range(glob, &pattern, &pos, amount, globindex++); + start = pattern; + /* fetch the name, if provided */ + if(curlx_str_single(&pattern, '<') || + curlx_str_until(&pattern, &name, MAX_GLOBNAME_LEN, '>') || + curlx_str_single(&pattern, '>')) { + /* Not a proper name. This is not reporting errors on syntax errors + on purpose: it means that if there is an existing use case that + uses what looks like a broken named-glob syntax (now introduced) + we let that function like before. */ + curlx_str_init(&name); + pattern = start; /* reset any partial patch */ + } + else { + /* check that the name is not already used */ + struct URLPattern *p = glob_find_name(glob, &name); + if(p) + return globerror(glob, "Duplicate glob name", 2 + start - ipattern, + CURLE_URL_MALFORMAT); + } + if(set) + result = glob_set(glob, &pattern, &pos, amount, globindex++, &name); + else + result = glob_range(glob, &pattern, &pos, amount, globindex++, + &name); + if(!result) result = add_glob(glob, pos); } @@ -492,6 +547,26 @@ bool glob_inuse(struct URLGlob *glob) return glob->palloc ? TRUE : FALSE; } +/* a glob error has been confirmed, this outputs details about it to the set + error stream */ +void glob_show_error(struct URLGlob *glob, const char *url, FILE *error, + CURLcode result) +{ + char text[512]; + const char *t; + if(glob->pos) { + curl_msnprintf(text, sizeof(text), "%s in position %zu:\n%s\n%*s^", + glob->error, + glob->pos, url, (int)glob->pos - 1, " "); + t = text; + } + else + t = glob->error; + + /* send error description to the error-stream */ + curl_mfprintf(error, "curl: (%d) %s\n", result, t); +} + CURLcode glob_url(struct URLGlob *glob, const char *url, curl_off_t *urlnum, FILE *error) { @@ -511,21 +586,8 @@ CURLcode glob_url(struct URLGlob *glob, const char *url, curl_off_t *urlnum, result = glob_parse(glob, url, 1, &amount); if(result) { - if(error && glob->error) { - char text[512]; - const char *t; - if(glob->pos) { - curl_msnprintf(text, sizeof(text), "%s in URL position %zu:\n%s\n%*s^", - glob->error, - glob->pos, url, (int)glob->pos - 1, " "); - t = text; - } - else - t = glob->error; - - /* send error description to the error-stream */ - curl_mfprintf(error, "curl: (%d) %s\n", result, t); - } + if(error && glob->error) + glob_show_error(glob, url, error, result); *urlnum = 1; return result; } @@ -547,6 +609,7 @@ void glob_cleanup(struct URLGlob *glob) curlx_safefree(glob->pattern[i].c.set.elem[elem]); curlx_safefree(glob->pattern[i].c.set.elem); } + curlx_safefree(glob->pattern[i].name); } curlx_safefree(glob->pattern); glob->palloc = 0; @@ -643,6 +706,7 @@ CURLcode glob_match_url(char **output, const char *filename, struct URLGlob *glob, SANITIZEcode *sc) { struct dynbuf dyn; + const char *ifilename = filename; *output = NULL; *sc = SANITIZE_ERR_OK; @@ -650,11 +714,11 @@ CURLcode glob_match_url(char **output, const char *filename, while(*filename) { CURLcode result = CURLE_OK; + struct URLPattern *pat = NULL; if(*filename == '#' && ISDIGIT(filename[1])) { - const char *ptr = filename; + /* a numbered glob reference */ + const char *ptr = filename++; curl_off_t num; - struct URLPattern *pat = NULL; - filename++; if(!curlx_str_number(&filename, &num, glob->pnum) && num) { size_t i; num--; /* make it zero based */ @@ -666,31 +730,49 @@ CURLcode glob_match_url(char **output, const char *filename, } } } - - if(pat) { - switch(pat->type) { - case GLOB_SET: - if(pat->c.set.elem) - result = curlx_dyn_add(&dyn, pat->c.set.elem[pat->c.set.idx]); - break; - case GLOB_ASCII: { - char letter = (char)pat->c.ascii.letter; - result = curlx_dyn_addn(&dyn, &letter, 1); - break; - } - case GLOB_NUM: - result = curlx_dyn_addf(&dyn, "%0*" CURL_FORMAT_CURL_OFF_T, - pat->c.num.npad, pat->c.num.idx); - break; - default: - DEBUGASSERT(0); + if(!pat) + filename = ptr; + } + else if(*filename == '#' && (filename[1] == '<')) { + /* a named glob reference */ + struct Curl_str name; + const char *ptr = filename; + filename += 2; /* pass both leading bytes */ + if(!curlx_str_until(&filename, &name, MAX_GLOBNAME_LEN, '>') && + !curlx_str_single(&filename, '>')) { + /* find the correct glob entry */ + pat = glob_find_name(glob, &name); + if(!pat) { + /* when the name is given correctly, it needs to be an existing glob + name, which makes this an error */ curlx_dyn_free(&dyn); - return CURLE_FAILED_INIT; + return globerror(glob, "no glob exists with this name", + filename - ifilename, CURLE_BAD_FUNCTION_ARGUMENT); } } - else - /* #[num] out of range, use the #[num] in the output */ - result = curlx_dyn_addn(&dyn, ptr, filename - ptr); + if(!pat) + filename = ptr; + } + if(pat) { + switch(pat->type) { + case GLOB_SET: + if(pat->c.set.elem) + result = curlx_dyn_add(&dyn, pat->c.set.elem[pat->c.set.idx]); + break; + case GLOB_ASCII: { + char letter = (char)pat->c.ascii.letter; + result = curlx_dyn_addn(&dyn, &letter, 1); + break; + } + case GLOB_NUM: + result = curlx_dyn_addf(&dyn, "%0*" CURL_FORMAT_CURL_OFF_T, + pat->c.num.npad, pat->c.num.idx); + break; + default: + DEBUGASSERT(0); + curlx_dyn_free(&dyn); + return CURLE_FAILED_INIT; + } } else result = curlx_dyn_addn(&dyn, filename++, 1); diff --git a/src/tool_urlglob.h b/src/tool_urlglob.h index ad0f144fd2..abc279de73 100644 --- a/src/tool_urlglob.h +++ b/src/tool_urlglob.h @@ -33,6 +33,7 @@ typedef enum { struct URLPattern { globtype type; + char *name; /* if not NULL */ int globindex; /* the number of this particular glob or -1 if not used within {} or [] */ union { @@ -71,6 +72,9 @@ struct URLGlob { size_t pos; /* column position of error or 0 */ }; +void glob_show_error(struct URLGlob *glob, const char *url, FILE *error, + CURLcode result); + CURLcode glob_url(struct URLGlob *glob, const char *url, curl_off_t *urlnum, FILE *error); CURLcode glob_next_url(char **globbed, struct URLGlob *glob); diff --git a/tests/data/Makefile.am b/tests/data/Makefile.am index 85ea4bcd1d..12f6bfbc0c 100644 --- a/tests/data/Makefile.am +++ b/tests/data/Makefile.am @@ -265,6 +265,7 @@ test2300 test2301 test2302 test2303 test2304 test2306 test2307 test2308 \ test2309 \ \ test2400 test2401 test2402 test2403 test2404 test2405 test2406 test2407 \ +test2408 test2409 test2410 test2411 \ \ test2500 test2501 test2502 test2503 test2504 test2505 test2506 \ \ diff --git a/tests/data/test2408 b/tests/data/test2408 new file mode 100644 index 0000000000..edc83d48a4 --- /dev/null +++ b/tests/data/test2408 @@ -0,0 +1,92 @@ + + + + +HTTP +HTTP GET +globbing +{} list + + +# Server-side + + +HTTP/1.1 200 OK +Funny-head: yesyes +Content-Length: 4 + +moo + + +HTTP/1.1 200 OK +Funny-head: yesyes +Content-Length: 4 + +foo + + +HTTP/1.1 200 OK +Funny-head: yesyes +Content-Length: 4 + +hoo + + + +# Client-side + + +http + + +multiple requests using named {} globs in URL + + +"%HOSTIP:%HTTPPORT/{%LTtest%GT%TESTNUMBER,%TESTNUMBER0002,%TESTNUMBER0003}" -o "%LOGDIR/dump-#%LTtest%GT" + + + +# Verify data after the test has been "shot" + + +GET /%TESTNUMBER HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +User-Agent: curl/%VERSION +Accept: */* + +GET /%TESTNUMBER0002 HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +User-Agent: curl/%VERSION +Accept: */* + +GET /%TESTNUMBER0003 HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +User-Agent: curl/%VERSION +Accept: */* + + + + +HTTP/1.1 200 OK +Funny-head: yesyes +Content-Length: 4 + +moo + + +HTTP/1.1 200 OK +Funny-head: yesyes +Content-Length: 4 + +foo + + +HTTP/1.1 200 OK +Funny-head: yesyes +Content-Length: 4 + +hoo + + + + diff --git a/tests/data/test2409 b/tests/data/test2409 new file mode 100644 index 0000000000..4c9e9c57a1 --- /dev/null +++ b/tests/data/test2409 @@ -0,0 +1,92 @@ + + + + +HTTP +HTTP GET +globbing +{} list + + +# Server-side + + +HTTP/1.1 200 swsbounce +Funny-head: yesyes +Content-Length: 4 + +moo + + +HTTP/1.1 200 swsbounce +Funny-head: yesyes +Content-Length: 4 + +foo + + +HTTP/1.1 200 OK +Funny-head: yesyes +Content-Length: 4 + +hoo + + + +# Client-side + + +http + + +multiple requests using named [] globs in URL + + +"%HOSTIP:%HTTPPORT/hello[%LTtest%GT7-9]" -o "%LOGDIR/dump-#%LTtest%GT" + + + +# Verify data after the test has been "shot" + + +GET /hello7 HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +User-Agent: curl/%VERSION +Accept: */* + +GET /hello8 HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +User-Agent: curl/%VERSION +Accept: */* + +GET /hello9 HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +User-Agent: curl/%VERSION +Accept: */* + + + + +HTTP/1.1 200 swsbounce +Funny-head: yesyes +Content-Length: 4 + +moo + + +HTTP/1.1 200 swsbounce +Funny-head: yesyes +Content-Length: 4 + +foo + + +HTTP/1.1 200 OK +Funny-head: yesyes +Content-Length: 4 + +hoo + + + + diff --git a/tests/data/test2410 b/tests/data/test2410 new file mode 100644 index 0000000000..fdec8c2e5d --- /dev/null +++ b/tests/data/test2410 @@ -0,0 +1,37 @@ + + + + +HTTP +HTTP GET +globbing +{} list + + +# Server-side + +# Client-side + + +http + + +duplicate named glob + + +"%HOSTIP:%HTTPPORT/{%LTtest%GTA,B}{%LTtest%GTC,D}" -o "%LOGDIR/dump" + + + +# Verify data after the test has been "shot" + + +curl: (3) Duplicate glob name in position 30: +%HOSTIP:%HTTPPORT/{%LTtest%GTA,B}{%LTtest%GTC,D} + ^ + + +3 + + + diff --git a/tests/data/test2411 b/tests/data/test2411 new file mode 100644 index 0000000000..5f45ac6ad8 --- /dev/null +++ b/tests/data/test2411 @@ -0,0 +1,37 @@ + + + + +HTTP +HTTP GET +globbing +{} list + + +# Server-side + +# Client-side + + +http + + +reference a named glob not set + + +"%HOSTIP:%HTTPPORT/{%LTtest%GTA,B}{%LTmoo%GTC,D}" -o "somewhere/#%LTfoo%GT" + + + +# Verify data after the test has been "shot" + + +curl: (43) no glob exists with this name in position 16: +somewhere/#%LTfoo%GT + ^ + + +43 + + + diff --git a/tests/data/test75 b/tests/data/test75 index 40c0d3b4de..0ba5e782c5 100644 --- a/tests/data/test75 +++ b/tests/data/test75 @@ -31,7 +31,7 @@ HTTP, urlglob retrieval with bad range 3 -curl: (3) bad range in URL position 47: +curl: (3) bad range in position 47: http://a-site-never-accessed.example.org/[2-1] ^ diff --git a/tests/data/test759 b/tests/data/test759 index 9c67e30f2d..7ad896bbcc 100644 --- a/tests/data/test759 +++ b/tests/data/test759 @@ -18,7 +18,7 @@ glob '{,' # Verify data after the test has been "shot" -# curl: (3) unmatched brace in URL position 1: +# curl: (3) unmatched brace in position 1: 3 diff --git a/tests/data/test761 b/tests/data/test761 index eec55e297f..2772a8f78f 100644 --- a/tests/data/test761 +++ b/tests/data/test761 @@ -22,8 +22,8 @@ http://testingthis/%repeat[201 x {a}b]% 3 -curl: (3) too many {} sets in URL position 403: -http://testingthis/%repeat[113 x {a}b]%{a +curl: (3) too many {} sets in position 403: +http://testingthis/%repeat[114 x {a}b]%{a