tests: replace significant invisible spaces with macros

To make them explicit, visible, avoid being accidentally trimmed.
Also prevents Git warnings, e.g. on `git am`.

Also:
- runtests: add support for `%spc%` and `%tab%` macros.
- test59: delete non-significant line-ending space.
- spacecheck.pl: drop line-ending whitespace exception for tests.

Closes #19300

.github/scripts/spacecheck.pl

@@ -42,10 +42,6 @@ my @need_crlf = (
     "\\.(bat|sln)\$",
 );
 
-my @space_at_eol = (
-    "^tests/data/test",
-);
-
 my @non_ascii_allowed = (
     '\xC3\xB6',  # UTF-8 for https://codepoints.net/U+00F6 LATIN SMALL LETTER O WITH DIAERESIS
 );
@@ -129,8 +125,7 @@ while(my $filename = <$git_ls_files>) {
         push @err, "content: must use LF EOL for this file type";
     }
 
-    if(!fn_match($filename, @space_at_eol) &&
-       $content =~ /[ \t]\n/) {
+    if($content =~ /[ \t]\n/) {
         my $line;
         for my $l (split(/\n/, $content)) {
             $line++;

docs/tests/FILEFORMAT.md

@@ -76,6 +76,14 @@ For example, to insert the word hello 100 times:
 
     %repeat[100 x hello]%
 
+## Whitespace
+
+To add significant whitespace characters at the end of the line, or to empty
+lines:
+
+    %spc%
+    %tab%
+
 ## Insert capped epoch days
 
 Mostly to test capped cookie expire dates: `%days[NUM]` inserts the number of

tests/data/test1029

@@ -50,7 +50,7 @@ Content-Length: 62
 Connection: close
 
 This server reply is for testing a simple Location: following
-http://%HOSTIP:%HTTPPORT/we/want/our/data/%TESTNUMBER0002.txt?coolsite=yes http://%HOSTIP:%HTTPPORT/we/want/our/%TESTNUMBER 0 
+http://%HOSTIP:%HTTPPORT/we/want/our/data/%TESTNUMBER0002.txt?coolsite=yes http://%HOSTIP:%HTTPPORT/we/want/our/%TESTNUMBER 0%spc%
 </stdout>
 </verify>
 </testcase>

tests/data/test1070

@@ -57,7 +57,7 @@ Expect: 100-continue
 Content-Length: 2313
 Content-Type: application/x-www-form-urlencoded
 
-This creates 
+This creates%spc%
 </protocol>
 </verify>
 </testcase>

tests/data/test1185

@@ -91,7 +91,7 @@ void startfunc(int a, int b) {
 ./%LOGDIR/code1185.c:4:82: warning: Longer than 79 columns (LONGLINE)
  int hello; /*------------------------------------------------------------------*/
 ./%LOGDIR/code1185.c:5:4: error: Contains TAB character (TABS)
- int	tab;
+ int%tab%tab;
     ^
 ./%LOGDIR/code1185.c:7:13: warning:  func with space (SPACEBEFOREPAREN)
  int a = func ();
@@ -118,7 +118,7 @@ void startfunc(int a, int b) {
  } else {
  ^
 ./%LOGDIR/code1185.c:24:11: warning: missing space after close paren (PARENBRACE)
- if(a == 2){  
+ if(a == 2){%spc%%spc%
            ^
 ./%LOGDIR/code1185.c:28:14: warning: no space before semicolon (SPACESEMICOLON)
  func_return() ;
@@ -205,10 +205,10 @@ void startfunc(int a, int b) {
   // CPP comment ?
   ^
 ./%LOGDIR/code1185.c:1:1: error: Missing copyright statement (COPYRIGHT)
- 
+%spc%
  ^
 ./%LOGDIR/code1185.c:1:1: error: Missing closing comment (OPENCOMMENT)
- 
+%spc%
  ^
 checksrc: 0 errors and 41 warnings
 </stdout>

tests/data/test136

@@ -30,7 +30,7 @@ FTP with user and no password
 <verify>
 <protocol crlf="yes">
 USER user
-PASS 
+PASS%spc%
 PWD
 EPSV
 TYPE I

tests/data/test1461

@@ -47,8 +47,8 @@ Usage: curl [options...] <url>
 
 This is not the full help; this menu is split into categories.
 Use "--help category" to get an overview of all categories, which are:
-auth, connection, curl, deprecated, dns, file, ftp, global, http, imap, ldap, 
-output, pop3, post, proxy, scp, sftp, smtp, ssh, telnet, tftp, timeout, tls, 
+auth, connection, curl, deprecated, dns, file, ftp, global, http, imap, ldap,%spc%
+output, pop3, post, proxy, scp, sftp, smtp, ssh, telnet, tftp, timeout, tls,%spc%
 upload, verbose.
 Use "--help all" to list all options
 Use "--help [option]" to view documentation for a given option

tests/data/test1654

@@ -27,7 +27,7 @@ h2 example.com 443 h3 shiny.example.com 8443 "20191231 00:00:00" 0 0
 # a comment
 h2 foo.example.com 443 h3 shiny.example.com 8443 "20291231 23:30:00" 0 0
   h1 example.com 443 h3 shiny.example.com 8443 "20121231 00:00:01" 0 0
-	h3 example.com 443 h3 shiny.example.com 8443 "20131231 00:00:00" 0 0
+%tab%h3 example.com 443 h3 shiny.example.com 8443 "20131231 00:00:00" 0 0
     # also a comment
 bad example.com 443 h3 shiny.example.com 8443 "20191231 00:00:00" 0 0
 rubbish

tests/data/test1664

@@ -68,7 +68,7 @@ curlx_str_singlespace
 3: ("b") 5, line 0
 4: ("\") 5, line 0
 5: (" ") 0, line 1
-6: ("	") 5, line 0
+6: ("%tab%") 5, line 0
 7: ("
 ") 5, line 0
 8: ("") 5, line 0

tests/data/test17

@@ -46,7 +46,7 @@ request MOOO
 <protocol crlf="yes">
 MOOO /that.site.com/%TESTNUMBER HTTP/1.1
 Host: %HOSTIP:%HTTPPORT
-User-Agent: agent007 license to drill	
+User-Agent: agent007 license to drill%tab%
 Accept: */*
 
 </protocol>

tests/data/test188

@@ -19,7 +19,7 @@ Content-Length: 3
 OK
 </data>
 <data1>
-HTTP/1.1 200 OK 
+HTTP/1.1 200 OK%spc%
 Connection: close
 Content-Length: 15
 Content-Range: bytes 50-
@@ -32,7 +32,7 @@ HTTP/1.1 301 OK swsbounce
 Location: /%TESTNUMBER
 Content-Length: 3
 
-HTTP/1.1 200 OK 
+HTTP/1.1 200 OK%spc%
 Connection: close
 Content-Length: 15
 Content-Range: bytes 50-

tests/data/test189

@@ -16,7 +16,7 @@ Content-Length: 3
 OK
 </data>
 <data1>
-HTTP/1.1 200 OK 
+HTTP/1.1 200 OK%spc%
 Connection: close
 Content-Length: 15
 
@@ -28,7 +28,7 @@ HTTP/1.1 301 OK swsbounce
 Location: /%TESTNUMBER
 Content-Length: 3
 
-HTTP/1.1 200 OK 
+HTTP/1.1 200 OK%spc%
 Connection: close
 Content-Length: 15
 

tests/data/test1940

@@ -10,10 +10,10 @@ curl_easy_header
 <data nocheck="yes">
 HTTP/1.1 200 OK
 Date: Thu, 09 Nov 2010 14:49:00 GMT
-Server:       test with trailing space     
+Server:       test with trailing space%repeat[5 x  ]%
 Content-Type: text/html
 Fold: is
- folding a     
+ folding a%repeat[5 x  ]%
    line
 Content-Length: 0
 Set-Cookie: onecookie=data;
@@ -59,8 +59,8 @@ http://%HOSTIP:%HTTPPORT/%TESTNUMBER
 - Set-Cookie == secondcookie=2data; (1/3)
 - Set-Cookie == cookie3=data3; (2/3)
  Fold == is folding a line
- Blank == 
- Blank2 == 
+ Blank ==%spc%
+ Blank2 ==%spc%
 </stdout>
 </verify>
 </testcase>

tests/data/test1941

@@ -11,7 +11,7 @@ CONNECT
 <data nocheck="yes">
 HTTP/1.1 200 OK
 Date: Thu, 09 Nov 2010 14:49:00 GMT
-Server:       test with trailing space     
+Server:       test with trailing space%repeat[5 x  ]%
 Content-Type: text/html
 Content-Length: 0
 Set-Cookie: onecookie=data;

tests/data/test1942

@@ -15,7 +15,7 @@ Server: maybe different
 
 HTTP/1.1 200 OK
 Date: Thu, 09 Nov 2010 14:49:00 GMT
-Server:       test with trailing space     
+Server:       test with trailing space%repeat[5 x  ]%
 Content-Type: text/html
 Content-Length: 0
 Set-Cookie: onecookie=data;

tests/data/test1944

@@ -10,7 +10,7 @@ curl_easy_header
 <data nocheck="yes">
 HTTP/1.1 302 OK
 Date: Thu, 01 Nov 2001 14:49:00 GMT
-Server:       test with trailing space     
+Server:       test with trailing space%repeat[5 x  ]%
 Content-Type: text/html
 Content-Length: 0
 Set-Cookie: onecookie=data;

tests/data/test1945

@@ -11,7 +11,7 @@ CONNECT
 <data nocheck="yes">
 HTTP/1.1 200 OK
 Date: Thu, 09 Nov 2010 14:49:00 GMT
-Server:       test with trailing space     
+Server:       test with trailing space%repeat[5 x  ]%
 Content-Type: text/html
 Content-Length: 0
 Set-Cookie: onecookie=data;

tests/data/test1946

@@ -10,7 +10,7 @@ curl_easy_header
 <data nocheck="yes">
 HTTP/1.1 302 OK
 Date: Thu, 01 Nov 2001 14:49:00 GMT
-Server:       test with trailing space     
+Server:       test with trailing space%repeat[5 x  ]%
 Content-Type: text/html
 Content-Length: 0
 Set-Cookie: onecookie=data;

tests/data/test1947

@@ -10,7 +10,7 @@ curl_easy_nextheader
 <data nocheck="yes">
 HTTP/1.1 302 OK
 Date: Thu, 01 Nov 2001 14:49:00 GMT
-Server:       test with trailing space     
+Server:       test with trailing space%repeat[5 x  ]%
 Content-Type: text/html
 Content-Length: 0
 Set-Cookie: onecookie=data;

tests/data/test1955

@@ -66,7 +66,7 @@ Authorization: XXX4-HMAC-SHA256 Credential=xxx/19700101/ple/exam/xxx4_request, S
 X-Xxx-Date: 19700101T000000Z
 test3: 1234
 test2:
-test_space: t	s  m	   end    
+test_space: t%tab%s  m%tab%   end%repeat[4 x  ]%
 tesMixCase: MixCase
 
 </protocol>

tests/data/test1958

@@ -64,7 +64,7 @@ GET /aws_sigv4/testapi/test HTTP/1.1
 Host: exam.ple.com:9000
 Authorization: XXX4-HMAC-SHA256 Credential=xxx/19700101/ple/exam/xxx4_request, SignedHeaders=content-type;host;x-xxx-content-sha256;x-xxx-date, Signature=25b4cac711ea8f65010c485d3778885f5f3870d0b8ff0b3ab58a8d7eeab991ff
 X-Xxx-Date: 19700101T000000Z
-X-Xxx-Content-Sha256: 	arbitrary 
+X-Xxx-Content-Sha256: %tab%arbitrary%spc%
 
 </protocol>
 </verify>

tests/data/test2302

@@ -66,7 +66,7 @@ Connection: Upgrade
 %hex[%8a%808321]hex%
 </protocol>
 <stdout mode="text">
-68 65 6c 6c 6f 
+68 65 6c 6c 6f%spc%
 RECFLAGS: 1
 </stdout>
 <limits>

tests/data/test2306

@@ -24,7 +24,7 @@ Funny-head: yesyes
 </data>
 <data2 nocheck="yes">
 HTTP/1.1 200 OK swsclose
-	Access-Control-Allow-Origin: *
+%tab%Access-Control-Allow-Origin: *
 Connection: Keep-Alive
 Content-Type: text/html; charset=utf-8
 Date: Wed, 10 May 2023 14:58:08 GMT

tests/data/test2400

@@ -50,7 +50,7 @@ HTTP/2 GET over HTTPS
 # Verify data after the test has been "shot"
 <verify>
 <stdout crlf="yes">
-HTTP/2 200 
+HTTP/2 200%spc%
 date: Tue, 09 Nov 2010 14:49:00 GMT
 content-length: 6
 content-type: text/html

tests/data/test2403

@@ -51,7 +51,7 @@ HTTP/2 GET using %{header_json}
 # Verify data after the test has been "shot"
 <verify>
 <stdout crlf="yes">
-HTTP/2 200 
+HTTP/2 200%spc%
 date: Tue, 09 Nov 2010 14:49:00 GMT
 content-length: 6
 content-type: text/html

tests/data/test2406

@@ -49,7 +49,7 @@ HTTP/2 over HTTPS with -f
 # Verify data after the test has been "shot"
 <verify>
 <stdout crlf="yes">
-HTTP/2 404 
+HTTP/2 404%spc%
 date: Tue, 09 Nov 2010 14:49:00 GMT
 content-length: 6
 content-type: text/html

tests/data/test2500

@@ -58,7 +58,7 @@ Accept: */*
 
 </protocol>
 <stdout crlf="yes">
-HTTP/3 200 
+HTTP/3 200%spc%
 date: Tue, 09 Nov 2010 14:49:00 GMT
 last-modified: Tue, 13 Jun 2000 12:10:00 GMT
 etag: "21025-dc7-39462498"

tests/data/test2503

@@ -50,7 +50,7 @@ HTTP/3 header-api
 # Verify data after the test has been "shot"
 <verify>
 <stdout crlf="yes">
-HTTP/3 200 
+HTTP/3 200%spc%
 date: Tue, 09 Nov 2010 14:49:00 GMT
 content-length: 6
 content-type: text/html

tests/data/test378

@@ -29,7 +29,7 @@ Reject using -T and -d at once
 2
 </errorcode>
 <stderr mode="text">
-Warning: You can only select one HTTP request method! You asked for both PUT 
+Warning: You can only select one HTTP request method! You asked for both PUT%spc%
 Warning: (-T, --upload-file) and POST (-d, --data).
 </stderr>
 </verify>

tests/data/test4

@@ -44,7 +44,7 @@ User-Agent: curl/%VERSION
 extra-header: here
 Accept: replaced
 X-Custom-Header:
-X-Test: foo; 
+X-Test: foo;%spc%
 X-Test2: foo;
 
 GET /%TESTNUMBER HTTP/1.1
@@ -53,7 +53,7 @@ User-Agent: curl/%VERSION
 extra-header: here
 Accept: replaced
 X-Custom-Header:
-X-Test: foo; 
+X-Test: foo;%spc%
 X-Test2: foo;
 
 </protocol>

tests/data/test421

@@ -21,7 +21,7 @@ referrer-policy: strict-origin-when-cross-origin
 access-control-allow-methods: GET, POST, PUT, DELETE, OPTIONS
 access-control-max-age: 1728000
 access-control-allow-headers: Authorization, Content-Type, AuthorizationOauth, X-EARLY-ACCESS
-access-control-expose-headers: 
+access-control-expose-headers:%spc%
 vary: Accept
 etag: W/"2678f9ab2ba550d164e7cc014aefd31e"
 cache-control: max-age=0, private, must-revalidate

tests/data/test459

@@ -56,7 +56,7 @@ Content-Type: application/x-www-form-urlencoded
 arg
 </protocol>
 <stderr mode="text">
-Warning: %LOGDIR/config:1: warning: 'data' uses unquoted whitespace. This may 
+Warning: %LOGDIR/config:1: warning: 'data' uses unquoted whitespace. This may%spc%
 Warning: cause side-effects. Consider double quotes.
 </stderr>
 </verify>

tests/data/test46

@@ -65,7 +65,7 @@ www.loser.com	FALSE	/	FALSE	2139150993	UID	99
 domain..tld	FALSE	/	FALSE	2139150993	mooo	indeed
 #HttpOnly_domain..tld	FALSE	/want	FALSE	2139150993	mooo2	indeed2
 %endif
-domain..tld	FALSE	/want	FALSE	0	empty	
+domain..tld	FALSE	/want	FALSE	0	empty%tab%
 </file>
 <features>
 cookies
@@ -88,18 +88,18 @@ Cookie: empty=; mooo2=indeed2; mooo=indeed
 # This file was generated by libcurl! Edit at your own risk.
 
 domain..tld	FALSE	/want/	FALSE	0	simplyhuge	%repeat[3998 x z]%
-domain..tld	FALSE	/	FALSE	0	justaname	
+domain..tld	FALSE	/	FALSE	0	justaname%tab%
 domain..tld	FALSE	/	FALSE	0	ASPSESSIONIDQGGQQSJJ	GKNBDIFAAOFDPDAIEAKDIBKE
 domain..tld	FALSE	/	FALSE	0	ckySession	temporary
 domain..tld	FALSE	/	FALSE	%days[400]	ckyPersistent	permanent
 %if large-time
-domain..tld	FALSE	/want	FALSE	0	empty	
+domain..tld	FALSE	/want	FALSE	0	empty%tab%
 #HttpOnly_domain..tld	FALSE	/want	FALSE	22139150993	mooo2	indeed2
 domain..tld	FALSE	/	FALSE	22139150993	mooo	indeed
 www.loser.com	FALSE	/	FALSE	22139150993	UID	99
 www.fake.come	FALSE	/	FALSE	22147483647	cookiecliente	si
 %else
-domain..tld	FALSE	/want	FALSE	0	empty	
+domain..tld	FALSE	/want	FALSE	0	empty%tab%
 #HttpOnly_domain..tld	FALSE	/want	FALSE	2139150993	mooo2	indeed2
 domain..tld	FALSE	/	FALSE	2139150993	mooo	indeed
 www.loser.com	FALSE	/	FALSE	2139150993	UID	99

tests/data/test54

@@ -11,7 +11,7 @@ followlocation
 <data>
 HTTP/1.1 302 This is a weirdo text message swsclose
 Connection: close
-Location: 
+Location:%spc%
 
 This server reply is for testing
 </data>

tests/data/test57

@@ -12,7 +12,7 @@ HTTP GET
 <data nocheck="yes">
 HTTP/1.1 200 OK swsclose
 Funny-head: yesyes
-Content-Type: text/html; charset=ISO-8859-4   
+Content-Type: text/html; charset=ISO-8859-4%repeat[3 x  ]%
 
 </data>
 </reply>

tests/data/test59

@@ -13,7 +13,7 @@ HTTP/1.0 200 OK swsclose
 Date: Tue, 09 Nov 2010 14:49:00 GMT
 Server: test-server/fake
 
-hej 
+hej
 </data>
 </reply>
 

tests/data/test689

@@ -11,9 +11,9 @@ OPTIONS
 # Server-side
 <reply>
 <data>
-RTSP/1.0 786          
+RTSP/1.0 786%repeat[10 x  ]%
 
-RTSP/          
+RTSP/%repeat[10 x  ]%
 </data>
 <datacheck>
 </datacheck>

tests/data/test769

@@ -16,7 +16,7 @@ Server: test-server/fake
 Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
 ETag: "21025-dc7-39462498"
 Accept-Ranges: bytes
-Content-Length: 6     
+Content-Length: 6%repeat[5 x  ]%
 Connection: close
 Content-Type: text/html
 Funny-head: yesyes

tests/data/test792

@@ -36,7 +36,7 @@ machine %HOSTIP login username password%hex[%00]hex% hello
 <verify>
 <protocol crlf="yes">
 USER username
-PASS 
+PASS%spc%
 PWD
 EPSV
 TYPE I

tests/data/test793

@@ -36,7 +36,7 @@ machine %HOSTIP login username "password"%hex[%00]hex% hello
 <verify>
 <protocol crlf="yes">
 USER username
-PASS 
+PASS%spc%
 PWD
 EPSV
 TYPE I

tests/testutil.pm

@@ -147,6 +147,9 @@ sub subbase64 {
         $$thing =~ s/%%DAYS%%/%alternatives[$d,$d2]/;
     }
 
+    $$thing =~ s/%spc%/ /g;   # space
+    $$thing =~ s/%tab%/\t/g;  # horizontal tab
+
     # include a file
     $$thing =~ s/%include ([^%]*)%[\n\r]+/includefile($1)/ge;
 }