tool: add "variable" support

Add support for command line variables. Set variables with --variable
name=content or --variable name@file (where "file" can be stdin if set
to a single dash (-)).

Variable content is expanded in option parameters using "{{name}}"
(without the quotes) if the option name is prefixed with
"--expand-". This gets the contents of the variable "name" inserted, or
a blank if the name does not exist as a variable. Insert "{{" verbatim
in the string by prefixing it with a backslash, like "\\{{".

Import an environment variable with --variable %name. It makes curl exit
with an error if the environment variable is not set. It can also rather
get a default value if the variable does not exist, using =content or
@file like shown above.

Example: get the USER environment variable into the URL:

 --variable %USER
 --expand-url = "https://example.com/api/{{USER}}/method"

When expanding variables, curl supports a set of functions that can make
the variable contents more convenient to use. It can trim leading and
trailing white space with "trim", output the contents as a JSON quoted
string with "json", URL encode it with "url" and base 64 encode it with
"b64". To apply functions to a variable expansion, add them colon
separated to the right side of the variable. They are then performed in
a left to right order.

Example: get the contents of a file called $HOME/.secret into a variable
called "fix". Make sure that the content is trimmed and percent-encoded
sent as POST data:

  --variable %HOME=/home/default
  --expand-variable fix@{{HOME}}/.secret
  --expand-data "{{fix:trim:url}}"
  https://example.com/

Documented. Many new test cases.

Co-brainstormed-by: Emanuele Torre
Assisted-by: Jat Satiro
Closes #11346
This commit is contained in:
Daniel Stenberg 2023-07-31 11:50:28 +02:00
parent 47a3e6e577
commit 2e160c9c65
No known key found for this signature in database
GPG key ID: 5CC908FDB71E12C2
32 changed files with 1562 additions and 258 deletions

View file

@ -69,10 +69,11 @@ test390 test391 test392 test393 test394 test395 test396 test397 test398 \
test399 test400 test401 test402 test403 test404 test405 test406 test407 \
test408 test409 test410 test411 test412 test413 test414 test415 test416 \
test417 test418 test419 test420 test421 test422 test423 test424 test425 \
test426 test427 \
test430 test431 test432 test433 test434 test435 test436 \
test426 test427 test428 test429 test430 test431 test432 test433 test434 \
test435 test436 \
\
test440 test441 test442 test443 test444 test445 test446 test447 \
test440 test441 test442 test443 test444 test445 test446 test447 test448 \
test449 test450 test451 test452 test453 test454 test455 \
\
test490 test491 test492 test493 test494 test495 test496 \
\

68
tests/data/test428 Normal file
View file

@ -0,0 +1,68 @@
<testcase>
<info>
<keywords>
HTTP
variables
</keywords>
</info>
#
# Server-side
<reply>
<data crlf="yes">
HTTP/1.1 200 OK
Date: Tue, 09 Nov 2010 14:49:00 GMT
Server: test-server/fake
Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
ETag: "21025-dc7-39462498"
Accept-Ranges: bytes
Content-Length: 6
Connection: close
Content-Type: text/html
Funny-head: yesyes
-foo-
</data>
</reply>
#
# Client-side
<client>
<server>
http
</server>
<setenv>
FUNVALUE=contents
VALUE2=curl
BLANK=
</setenv>
<name>
Expand environment variables within config file
</name>
<file name="%LOGDIR/cmd">
--variable %FUNVALUE
--variable %VALUE2
--variable %BLANK
--variable %curl_NOT_SET=default
--expand-data 1{{FUNVALUE}}2{{VALUE2}}3{{curl_NOT_SET}}4{{BLANK}}5\{{verbatim}}6{{not.good}}7{{}}
</file>
<command>
http://%HOSTIP:%HTTPPORT/%TESTNUMBER -K %LOGDIR/cmd
</command>
</client>
#
# Verify data after the test has been "shot"
<verify>
<protocol crlf="yes" nonewline="yes">
POST /%TESTNUMBER HTTP/1.1
Host: %HOSTIP:%HTTPPORT
User-Agent: curl/%VERSION
Accept: */*
Content-Length: 54
Content-Type: application/x-www-form-urlencoded
1contents2curl3default45{{verbatim}}6{{not.good}}7{{}}
</protocol>
</verify>
</testcase>

63
tests/data/test429 Normal file
View file

@ -0,0 +1,63 @@
<testcase>
<info>
<keywords>
HTTP
HTTP POST
variables
</keywords>
</info>
#
# Server-side
<reply>
<data crlf="yes">
HTTP/1.1 200 OK
Date: Tue, 09 Nov 2010 14:49:00 GMT
Server: test-server/fake
Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
ETag: "21025-dc7-39462498"
Accept-Ranges: bytes
Content-Length: 6
Connection: close
Content-Type: text/html
Funny-head: yesyes
-foo-
</data>
</reply>
#
# Client-side
<client>
<server>
http
</server>
<setenv>
FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF=contents2023
</setenv>
<name>
Expand environment variable in config file - too long name
</name>
<file name="%LOGDIR/cmd">
--expand-data {{FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF}}
</file>
<command>
http://%HOSTIP:%HTTPPORT/%TESTNUMBER -K %LOGDIR/cmd
</command>
</client>
#
# Verify data after the test has been "shot"
<verify>
<protocol crlf="yes" nonewline="yes">
POST /%TESTNUMBER HTTP/1.1
Host: %HOSTIP:%HTTPPORT
User-Agent: curl/%VERSION
Accept: */*
Content-Length: 133
Content-Type: application/x-www-form-urlencoded
{{FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF}}
</protocol>
</verify>
</testcase>

67
tests/data/test448 Normal file
View file

@ -0,0 +1,67 @@
<testcase>
<info>
<keywords>
HTTP
variables
--config
</keywords>
</info>
#
# Server-side
<reply>
<data crlf="yes">
HTTP/1.1 200 OK
Date: Tue, 09 Nov 2010 14:49:00 GMT
Server: test-server/fake
Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
ETag: "21025-dc7-39462498"
Accept-Ranges: bytes
Content-Length: 6
Connection: close
Content-Type: text/html
Funny-head: yesyes
-foo-
</data>
</reply>
#
# Client-side
<client>
<server>
http
</server>
<setenv>
FUNVALUE=contents
VALUE2=curl
BLANK=
</setenv>
<name>
Environment variables within config file, unbalanced braces
</name>
<file name="%LOGDIR/cmd">
--variable %FUNVALUE
--variable %VALUE2
--expand-data 1{{FUNVALUE}}2{{VALUE2}}3{{curl_NOT_SET}}4{{AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA}}5{{broken
</file>
<command>
http://%HOSTIP:%HTTPPORT/%TESTNUMBER -K %LOGDIR/cmd
</command>
</client>
#
# Verify data after the test has been "shot"
<verify>
<protocol crlf="yes" nonewline="yes">
POST /%TESTNUMBER HTTP/1.1
Host: %HOSTIP:%HTTPPORT
User-Agent: curl/%VERSION
Accept: */*
Content-Length: 157
Content-Type: application/x-www-form-urlencoded
1contents2curl34{{AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA}}5{{broken
</protocol>
</verify>
</testcase>

65
tests/data/test449 Normal file
View file

@ -0,0 +1,65 @@
<testcase>
<info>
<keywords>
HTTP
variables
--config
</keywords>
</info>
#
# Server-side
<reply>
<data crlf="yes">
HTTP/1.1 200 OK
Date: Tue, 09 Nov 2010 14:49:00 GMT
Server: test-server/fake
Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
ETag: "21025-dc7-39462498"
Accept-Ranges: bytes
Content-Length: 6
Connection: close
Content-Type: text/html
Funny-head: yesyes
-foo-
</data>
</reply>
#
# Client-side
<client>
<server>
http
</server>
<setenv>
FUNVALUE=contents
VALUE2=curl
BLANK=
</setenv>
<name>
Environment variables in config file w/o [expand]
</name>
<file name="%LOGDIR/cmd">
-d 1{{FUNVALUE}}2{{VALUE2}}3{{CURL_NOT_SET}}4{{BLANK}}5\{{verbatim}}6{{not.good}}7{{}}
</file>
<command>
http://%HOSTIP:%HTTPPORT/%TESTNUMBER -K %LOGDIR/cmd
</command>
</client>
#
# Verify data after the test has been "shot"
<verify>
<protocol crlf="yes" nonewline="yes">
POST /%TESTNUMBER HTTP/1.1
Host: %HOSTIP:%HTTPPORT
User-Agent: curl/%VERSION
Accept: */*
Content-Length: 83
Content-Type: application/x-www-form-urlencoded
1{{FUNVALUE}}2{{VALUE2}}3{{CURL_NOT_SET}}4{{BLANK}}5\{{verbatim}}6{{not.good}}7{{}}
</protocol>
</verify>
</testcase>

60
tests/data/test450 Normal file
View file

@ -0,0 +1,60 @@
<testcase>
<info>
<keywords>
HTTP
--config
variables
</keywords>
</info>
#
# Server-side
<reply>
<data crlf="yes">
HTTP/1.1 200 OK
Date: Tue, 09 Nov 2010 14:49:00 GMT
Server: test-server/fake
Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
ETag: "21025-dc7-39462498"
Accept-Ranges: bytes
Content-Length: 6
Connection: close
Content-Type: text/html
Funny-head: yesyes
-foo-
</data>
</reply>
#
# Client-side
<client>
<server>
http
</server>
<name>
Variable from file that is trimmed and URL encoded
</name>
<file name="%LOGDIR/junk">
space with space
</file>
<command>
http://%HOSTIP:%HTTPPORT/%TESTNUMBER --variable what@%LOGDIR/junk --expand-data "{{what:trim:url}}"
</command>
</client>
#
# Verify data after the test has been "shot"
<verify>
<protocol crlf="yes" nonewline="yes">
POST /%TESTNUMBER HTTP/1.1
Host: %HOSTIP:%HTTPPORT
User-Agent: curl/%VERSION
Accept: */*
Content-Length: 20
Content-Type: application/x-www-form-urlencoded
space%20with%20space
</protocol>
</verify>
</testcase>

59
tests/data/test451 Normal file
View file

@ -0,0 +1,59 @@
<testcase>
<info>
<keywords>
HTTP
variables
</keywords>
</info>
#
# Server-side
<reply>
<data crlf="yes">
HTTP/1.1 200 OK
Date: Tue, 09 Nov 2010 14:49:00 GMT
Server: test-server/fake
Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
ETag: "21025-dc7-39462498"
Accept-Ranges: bytes
Content-Length: 6
Connection: close
Content-Type: text/html
Funny-head: yesyes
-foo-
</data>
</reply>
#
# Client-side
<client>
<server>
http
</server>
<name>
Variable from file that is JSON and URL encoded (with null byte)
</name>
<file name="%LOGDIR/junk">
%hex[%01%02%03%00%04%05%06]hex%
</file>
<command>
http://%HOSTIP:%HTTPPORT/%TESTNUMBER --variable what@%LOGDIR/junk --variable second=hello --variable second=again --expand-data "--{{what:trim:json}}22{{none}}--{{second}}{{what:trim:url}}"
</command>
</client>
#
# Verify data after the test has been "shot"
<verify>
<protocol crlf="yes" nonewline="yes">
POST /%TESTNUMBER HTTP/1.1
Host: %HOSTIP:%HTTPPORT
User-Agent: curl/%VERSION
Accept: */*
Content-Length: 74
Content-Type: application/x-www-form-urlencoded
--\u0001\u0002\u0003\u0000\u0004\u0005\u000622--again%01%02%03%00%04%05%06
</protocol>
</verify>
</testcase>

34
tests/data/test452 Normal file
View file

@ -0,0 +1,34 @@
<testcase>
<info>
<keywords>
variables
</keywords>
</info>
#
# Server-side
<reply>
</reply>
#
# Client-side
<client>
<server>
http
</server>
<name>
Variable using illegal function in expansion
</name>
<command>
http://%HOSTIP:%HTTPPORT/%TESTNUMBER --variable what=hello --expand-data "--{{what:trim:super}}"
</command>
</client>
#
# Verify data after the test has been "shot"
<verify>
<errorcode>
2
</errorcode>
</verify>
</testcase>

33
tests/data/test453 Normal file
View file

@ -0,0 +1,33 @@
<testcase>
<info>
<keywords>
HTTP
variables
</keywords>
</info>
#
# Client-side
<client>
<server>
http
</server>
<name>
Variable output containing null byte
</name>
<file name="%LOGDIR/junk">
%hex[%01%02%03%00%04%05%06]hex%
</file>
<command>
http://%HOSTIP:%HTTPPORT/%TESTNUMBER --variable what@%LOGDIR/junk --expand-data "{{what}}"
</command>
</client>
#
# Verify data after the test has been "shot"
<verify>
<errorcode>
2
</errorcode>
</verify>
</testcase>

34
tests/data/test454 Normal file
View file

@ -0,0 +1,34 @@
<testcase>
<info>
<keywords>
variables
</keywords>
</info>
#
# Server-side
<reply>
</reply>
#
# Client-side
<client>
<server>
http
</server>
<name>
Variable using illegal function separator
</name>
<command>
http://%HOSTIP:%HTTPPORT/%TESTNUMBER --variable what=hello --expand-data "--{{what:trim,url}}"
</command>
</client>
#
# Verify data after the test has been "shot"
<verify>
<errorcode>
2
</errorcode>
</verify>
</testcase>

52
tests/data/test455 Normal file
View file

@ -0,0 +1,52 @@
<testcase>
<info>
<keywords>
variables
</keywords>
</info>
#
# Server-side
<reply>
<data crlf="yes">
HTTP/1.1 200 OK
Date: Tue, 09 Nov 2010 14:49:00 GMT
Server: test-server/fake
Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
ETag: "21025-dc7-39462498"
Accept-Ranges: bytes
Content-Length: 6
Connection: close
Content-Type: text/html
Funny-head: yesyes
-foo-
</data>
</reply>
#
# Client-side
<client>
<server>
http
</server>
<name>
Variable using base64
</name>
<command>
--variable moby="Call me Ishmael" --expand-url "http://%HOSTIP:%HTTPPORT/{{moby:b64}}/%TESTNUMBER"
</command>
</client>
#
# Verify data after the test has been "shot"
<verify>
<protocol crlf="yes">
GET /%b64[Call me Ishmael]b64%/%TESTNUMBER HTTP/1.1
Host: %HOSTIP:%HTTPPORT
User-Agent: curl/%VERSION
Accept: */*
</protocol>
</verify>
</testcase>