tool: improve error/warning messages when output filename sanitization fails

On MS-DOS (OOM and bad filename) and Windows (OOM only).

Given the rarity of both platform and error, we make a compromise and
return an unrelated libcurl error (43) in case of a bad output filename
on MS-DOS.

After:
```
$ CURL_FN_SANITIZE_OOM=1 wine curl.exe https://curl.se/ --output out.txt
curl: (27) Out of memory

$ CURL_FN_SANITIZE_BAD=1 wine curl.exe https://curl.se/ --output out.txt
Warning: bad output filename
curl: (43) A libcurl function was given a bad argument

$ CURL_FN_SANITIZE_OOM=1 wine curl.exe https://curl.se/index.html --globoff -O
curl: (27) Out of memory

$ CURL_FN_SANITIZE_BAD=1 wine curl.exe https://curl.se/index.html --globoff -O
curl: bad output filename
curl: (43) A libcurl function was given a bad argument
```

Before:
```
$ CURL_FN_SANITIZE_OOM=1 wine curl.exe https://curl.se/ --output out.txt
Warning: bad output glob
curl: (27) Out of memory

$ CURL_FN_SANITIZE_BAD=1 wine curl.exe https://curl.se/ --output out.txt
Warning: bad output glob
curl: (3) URL using bad/illegal format or missing URL

$ CURL_FN_SANITIZE_OOM=1 wine curl.exe https://curl.se/index.html --globoff -O
curl: Failed to extract a filename from the URL to use for storage
curl: (27) Out of memory

$ CURL_FN_SANITIZE_BAD=1 wine curl.exe https://curl.se/index.html --globoff -O
curl: Failed to extract a filename from the URL to use for storage
curl: (3) URL using bad/illegal format or missing URL
```

Ref: #20116 (simpler reboot of)
Ref: #20113 #20121
Ref: 40c1748af5 #20198
Ref: eb7f5b71e5 #20143
Ref: 8c02407bef #20125
Fixes #20044
Closes #20199

src/tool_doswin.c

@@ -32,9 +32,9 @@
 #ifdef _WIN32
 #  include <stdlib.h>
 #  include <tlhelp32.h>
-#  include "tool_cfgable.h"
 #endif
 
+#include "tool_cfgable.h"
 #include "tool_bname.h"
 #include "tool_doswin.h"
 #include "tool_msgs.h"

src/tool_doswin.h

@@ -30,14 +30,6 @@
 #define SANITIZE_ALLOW_PATH     (1 << 1) /* Allow path separators and colons */
 #define SANITIZE_ALLOW_RESERVED (1 << 2) /* Allow reserved device names */
 
-typedef enum {
-  SANITIZE_ERR_OK = 0,           /* 0 - OK */
-  SANITIZE_ERR_INVALID_PATH,     /* 1 - the path is invalid */
-  SANITIZE_ERR_BAD_ARGUMENT,     /* 2 - bad function parameter */
-  SANITIZE_ERR_OUT_OF_MEMORY,    /* 3 - out of memory */
-  SANITIZE_ERR_LAST /* never use! */
-} SANITIZEcode;
-
 SANITIZEcode sanitize_file_name(char ** const sanitized, const char *file_name,
                                 int flags);
 

src/tool_operate.c

@@ -979,9 +979,17 @@ static CURLcode setup_outfile(struct OperationConfig *config,
   struct State *state = &global->state;
 
   if(!per->outfile) {
+    SANITIZEcode sc;
     /* extract the filename from the URL */
-    CURLcode result = get_url_file_name(&per->outfile, per->url);
-    if(result) {
+    CURLcode result = get_url_file_name(&per->outfile, per->url, &sc);
+    if(sc) {
+      if(sc == SANITIZE_ERR_OUT_OF_MEMORY)
+        return CURLE_OUT_OF_MEMORY;
+      if(sc == SANITIZE_ERR_INVALID_PATH)
+        errorf("bad output filename");
+      return result;
+    }
+    else if(result) {
       errorf("Failed to extract a filename"
              " from the URL to use for storage");
       return result;
@@ -990,10 +998,18 @@ static CURLcode setup_outfile(struct OperationConfig *config,
   else if(glob_inuse(&state->urlglob)) {
     /* fill '#1' ... '#9' terms from URL pattern */
     char *storefile = per->outfile;
+    SANITIZEcode sc;
     CURLcode result =
-      glob_match_url(&per->outfile, storefile, &state->urlglob);
+      glob_match_url(&per->outfile, storefile, &state->urlglob, &sc);
     tool_safefree(storefile);
-    if(result) {
+    if(sc) {
+      if(sc == SANITIZE_ERR_OUT_OF_MEMORY)
+        return CURLE_OUT_OF_MEMORY;
+      if(sc == SANITIZE_ERR_INVALID_PATH)
+        warnf("bad output filename");
+      return result;
+    }
+    else if(result) {
       /* bad globbing */
       warnf("bad output glob");
       return result;

src/tool_operhlp.c

@@ -172,12 +172,14 @@ fail:
  * Returns a pointer to a heap-allocated string or NULL if
  * no name part, at location indicated by first argument.
  */
-CURLcode get_url_file_name(char **filename, const char *url)
+CURLcode get_url_file_name(char **filename, const char *url, SANITIZEcode *sc)
 {
   CURLU *uh = curl_url();
   char *path = NULL;
   CURLUcode uerr;
 
+  *sc = SANITIZE_ERR_OK;
+
   if(!uh)
     return CURLE_OUT_OF_MEMORY;
 
@@ -220,13 +222,10 @@ CURLcode get_url_file_name(char **filename, const char *url)
 #if defined(_WIN32) || defined(MSDOS)
       {
         char *sanitized;
-        SANITIZEcode sc = sanitize_file_name(&sanitized, *filename, 0);
+        *sc = sanitize_file_name(&sanitized, *filename, 0);
         tool_safefree(*filename);
-        if(sc) {
-          if(sc == SANITIZE_ERR_OUT_OF_MEMORY)
-            return CURLE_OUT_OF_MEMORY;
-          return CURLE_URL_MALFORMAT;
-        }
+        if(*sc)
+          return CURLE_BAD_FUNCTION_ARGUMENT;
         *filename = sanitized;
       }
 #endif /* _WIN32 || MSDOS */

src/tool_operhlp.h

@@ -31,7 +31,7 @@ void clean_getout(struct OperationConfig *config);
 bool output_expected(const char *url, const char *uploadfile);
 bool stdin_upload(const char *uploadfile);
 CURLcode add_file_name_to_url(CURL *curl, char **inurlp, const char *filename);
-CURLcode get_url_file_name(char **filename, const char *url);
+CURLcode get_url_file_name(char **filename, const char *url, SANITIZEcode *sc);
 CURLcode urlerr_cvt(CURLUcode ucode);
 
 #endif /* HEADER_CURL_TOOL_OPERHLP_H */

src/tool_sdecls.h

@@ -122,6 +122,14 @@ typedef enum {
   TOOL_HTTPREQ_PUT
 } HttpReq;
 
+typedef enum {
+  SANITIZE_ERR_OK = 0,           /* 0 - OK */
+  SANITIZE_ERR_INVALID_PATH,     /* 1 - the path is invalid */
+  SANITIZE_ERR_BAD_ARGUMENT,     /* 2 - bad function parameter */
+  SANITIZE_ERR_OUT_OF_MEMORY,    /* 3 - out of memory */
+  SANITIZE_ERR_LAST /* never use! */
+} SANITIZEcode;
+
 /*
  * Complete struct declarations which have OperationConfig struct members,
  * just in case this header is directly included in some source file.

src/tool_urlglob.c

@@ -637,10 +637,11 @@ CURLcode glob_next_url(char **globbed, struct URLGlob *glob)
 #define MAX_OUTPUT_GLOB_LENGTH (1024 * 1024)
 
 CURLcode glob_match_url(char **output, const char *filename,
-                        struct URLGlob *glob)
+                        struct URLGlob *glob, SANITIZEcode *sc)
 {
   struct dynbuf dyn;
   *output = NULL;
+  *sc = SANITIZE_ERR_OK;
 
   curlx_dyn_init(&dyn, MAX_OUTPUT_GLOB_LENGTH);
 
@@ -700,15 +701,11 @@ CURLcode glob_match_url(char **output, const char *filename,
 #if defined(_WIN32) || defined(MSDOS)
   {
     char *sanitized;
-    SANITIZEcode sc = sanitize_file_name(&sanitized, curlx_dyn_ptr(&dyn),
-                                         (SANITIZE_ALLOW_PATH |
-                                          SANITIZE_ALLOW_RESERVED));
+    *sc = sanitize_file_name(&sanitized, curlx_dyn_ptr(&dyn),
+                             SANITIZE_ALLOW_PATH | SANITIZE_ALLOW_RESERVED);
     curlx_dyn_free(&dyn);
-    if(sc) {
-      if(sc == SANITIZE_ERR_OUT_OF_MEMORY)
-        return CURLE_OUT_OF_MEMORY;
-      return CURLE_URL_MALFORMAT;
-    }
+    if(*sc)
+      return CURLE_BAD_FUNCTION_ARGUMENT;
     *output = sanitized;
     return CURLE_OK;
   }

src/tool_urlglob.h

@@ -75,7 +75,7 @@ CURLcode glob_url(struct URLGlob *glob, const char *url, curl_off_t *urlnum,
                   FILE *error);
 CURLcode glob_next_url(char **globbed, struct URLGlob *glob);
 CURLcode glob_match_url(char **output, const char *filename,
-                        struct URLGlob *glob);
+                        struct URLGlob *glob, SANITIZEcode *sc);
 void glob_cleanup(struct URLGlob *glob);
 bool glob_inuse(struct URLGlob *glob);