tests: strict torture mode, warn on alloc after limit

In order to better track down cases where we don't exit cleanly and
directly on OOM errors, this mode warns if there is another memory
operation found *after* the torture limit was reached.

runtests sets this strict mode.

memdebug adds two new helper functions needed to make this work.

curl_dbg_overlook() lets the code mark a memory allocation as fine to
not notice and yet do another one after even if this one returns error.

curl_dbg_restart() marks the code as - at this point the code ignores
the OOM status and it is fine to do another memory call afterwards
without it being considered a problem.

lib/curl_setup.h

@@ -1005,6 +1005,8 @@ CURL_EXTERN ALLOC_FUNC
 CURL_EXTERN void curl_dbg_memdebug(const char *logname);
 CURL_EXTERN void curl_dbg_memlimit(long limit);
 CURL_EXTERN void curl_dbg_log(const char *format, ...) CURL_PRINTF(1, 2);
+CURL_EXTERN void curl_dbg_allow(int line, const char *source, int index);
+CURL_EXTERN void curl_dbg_clear(int line, const char *source);
 
 /* file descriptor manipulators */
 CURL_EXTERN curl_socket_t curl_dbg_socket(int domain, int type, int protocol,
@@ -1091,6 +1093,9 @@ CURL_EXTERN ALLOC_FUNC
 #define CURL_SEND send
 #define CURL_RECV recv
 
+#define curl_dbg_overlook(x)
+#define curl_dbg_restart()
+
 #endif /* CURLDEBUG */
 
 /* Some versions of the Android NDK is missing the declaration */

lib/easy.c

@@ -196,9 +196,11 @@ static CURLcode global_init(long flags, bool memoryfuncs)
   easy_init_flags = flags;
 
 #ifdef DEBUGBUILD
-  if(getenv("CURL_GLOBAL_INIT"))
+  if(getenv("CURL_GLOBAL_INIT")) {
     /* alloc data that will leak if *cleanup() is not called! */
     leakpointer = malloc(1);
+    curl_dbg_restart();
+  }
 #endif
 
   return CURLE_OK;

lib/getenv.c

@@ -70,6 +70,7 @@ static char *GetEnv(const char *variable)
   }
 #else
   char *env = getenv(variable);
+  curl_dbg_overlook(1);
   return (env && env[0]) ? strdup(env) : NULL;
 #endif
 }

lib/memdebug.c

@@ -110,8 +110,7 @@ static bool countcheck(const char *func, int line, const char *source)
   if(memlimit && source) {
     if(!memsize) {
       /* log to file */
-      curl_dbg_log("LIMIT %s:%d %s reached memlimit\n",
-                   source, line, func);
+      curl_dbg_log("LIMIT %s:%d %s reached memlimit\n", source, line, func);
       /* log to stderr also */
       curl_mfprintf(stderr, "LIMIT %s:%d %s reached memlimit\n",
                     source, line, func);
@@ -135,8 +134,11 @@ void *curl_dbg_malloc(size_t wantedsize, int line, const char *source)
 
   DEBUGASSERT(wantedsize != 0);
 
-  if(countcheck("malloc", line, source))
+  if(countcheck("malloc", line, source)) {
+    curl_dbg_log("FAIL %s:%d malloc(%zu) = NULL\n",
+                 source, line, wantedsize);
     return NULL;
+  }
 
   /* alloc at least 64 bytes */
   size = sizeof(struct memdebug) + wantedsize;
@@ -164,8 +166,11 @@ void *curl_dbg_calloc(size_t wanted_elements, size_t wanted_size,
   DEBUGASSERT(wanted_elements != 0);
   DEBUGASSERT(wanted_size != 0);
 
-  if(countcheck("calloc", line, source))
+  if(countcheck("calloc", line, source)) {
+    curl_dbg_log("FAIL %s:%d calloc(%zu,%zu) = NULL\n",
+                 source, line, wanted_elements, wanted_size);
     return NULL;
+  }
 
   /* alloc at least 64 bytes */
   user_size = wanted_size * wanted_elements;
@@ -191,11 +196,14 @@ char *curl_dbg_strdup(const char *str, int line, const char *source)
 
   DEBUGASSERT(str != NULL);
 
-  if(countcheck("strdup", line, source))
-    return NULL;
-
   len = strlen(str) + 1;
 
+  if(countcheck("strdup", line, source)) {
+    curl_dbg_log("FAIL %s:%d strdup(%zu bytes) = NULL\n", source, line,
+                 len);
+    return NULL;
+  }
+
   mem = curl_dbg_malloc(len, 0, NULL); /* NULL prevents logging */
   if(mem)
     memcpy(mem, str, len);
@@ -216,8 +224,10 @@ wchar_t *curl_dbg_wcsdup(const wchar_t *str, int line, const char *source)
 
   DEBUGASSERT(str != NULL);
 
-  if(countcheck("wcsdup", line, source))
+  if(countcheck("wcsdup", line, source)) {
+    curl_dbg_log("FAIL %s:%d wcdup() = NULL\n", source, line);
     return NULL;
+  }
 
   wsiz = wcslen(str) + 1;
   bsiz = wsiz * sizeof(wchar_t);
@@ -245,8 +255,11 @@ void *curl_dbg_realloc(void *ptr, size_t wantedsize,
 
   DEBUGASSERT(wantedsize != 0);
 
-  if(countcheck("realloc", line, source))
+  if(countcheck("realloc", line, source)) {
+    curl_dbg_log("FAIL %s:%d realloc(%zu) = NULL\n",
+                 source, line, wantedsize);
     return NULL;
+  }
 
 #ifdef __INTEL_COMPILER
 #  pragma warning(push)
@@ -305,8 +318,11 @@ curl_socket_t curl_dbg_socket(int domain, int type, int protocol,
 {
   curl_socket_t sockfd;
 
-  if(countcheck("socket", line, source))
+  if(countcheck("socket", line, source)) {
+    curl_dbg_log("FAIL %s:%d socket() = [bad]\n",
+                 source, line);
     return CURL_SOCKET_BAD;
+  }
 
   /* !checksrc! disable BANNEDFUNC 1 */
   sockfd = socket(domain, type, protocol);
@@ -324,8 +340,10 @@ SEND_TYPE_RETV curl_dbg_send(SEND_TYPE_ARG1 sockfd,
                              int line, const char *source)
 {
   SEND_TYPE_RETV rc;
-  if(countcheck("send", line, source))
+  if(countcheck("send", line, source)) {
+    curl_dbg_log("FAIL %s:%d send() = -1\n", source, line);
     return -1;
+  }
   /* !checksrc! disable BANNEDFUNC 1 */
   rc = send(sockfd, buf, len, flags);
   if(source)
@@ -339,8 +357,10 @@ RECV_TYPE_RETV curl_dbg_recv(RECV_TYPE_ARG1 sockfd, RECV_TYPE_ARG2 buf,
                              int line, const char *source)
 {
   RECV_TYPE_RETV rc;
-  if(countcheck("recv", line, source))
+  if(countcheck("recv", line, source)) {
+    curl_dbg_log("FAIL %s:%d recv() = -1\n", source, line);
     return -1;
+  }
   /* !checksrc! disable BANNEDFUNC 1 */
   rc = recv(sockfd, buf, len, flags);
   if(source)
@@ -490,4 +510,15 @@ void curl_dbg_log(const char *format, ...)
     (fwrite)(buf, 1, (size_t)nchars, curl_dbg_logfile);
 }
 
+void curl_dbg_allow(int line, const char *source, int index)
+{
+  curl_dbg_log("OVERLOOK %s:%d\n", source, line + index);
+}
+
+void curl_dbg_clear(int line, const char *source)
+{
+  curl_dbg_log("RESTART %s:%d\n", source, line);
+}
+
+
 #endif /* CURLDEBUG */

lib/memdebug.h

@@ -52,5 +52,8 @@
 #endif
 #endif /* _WIN32 */
 
+#define curl_dbg_overlook(x) curl_dbg_allow(__LINE__, __FILE__, x)
+#define curl_dbg_restart() curl_dbg_clear(__LINE__, __FILE__)
+
 #endif /* CURLDEBUG */
 #endif /* HEADER_CURL_MEMDEBUG_H */

src/tool_operate.c

@@ -776,6 +776,7 @@ skip:
   free(per->uploadfile);
   curl_slist_free_all(per->hdrcbdata.headlist);
   per->hdrcbdata.headlist = NULL;
+  curl_dbg_restart();
   return result;
 }
 

src/tool_writeout.c

@@ -733,6 +733,7 @@ void ourWriteOut(struct OperationConfig *config, struct per_transfer *per,
   if(!writeinfo)
     return;
 
+  curl_dbg_restart();
   curlx_dyn_init(&name, MAX_WRITEOUT_NAME_LENGTH);
   while(ptr && *ptr && !done) {
     if('%' == *ptr && ptr[1]) {

tests/data/test558

@@ -31,6 +31,7 @@ libtest memory tracking operational
 <verify>
 <file name="%LOGDIR/memdump">
 MEM easy.c: malloc()
+RESTART easy.c:
 MEM lib%TESTNUMBER.c: malloc()
 MEM lib%TESTNUMBER.c: free()
 MEM dynbuf.c: realloc()
@@ -43,10 +44,11 @@ s/^MEM escape.c:\d+ free\(\(nil\)\)[\n]$//
 s/ =.*//
 s/\(.*\)/()/
 s/:\d+/:/
-s:^(MEM |FD )(.*[/\\])(.*):$1$3:
+s:^(MEM |FD |OVERLOOK |RESTART )(.*[/\\])(.*):$1$3:
 s/\r\n/\n/
 s/^MEM getenv.c: realloc\(\)[\n]$//
 s/^MEM getenv.c: free\(\)[\n]$//
+s/^OVERLOOK getenv.c:[\n]$//
 </stripfile>
 </verify>
 

tests/memanalyze.pl

@@ -43,6 +43,7 @@ my $recvs=0;
 my $sockets=0;
 my $verbose=0;
 my $trace=0;
+my $strict;
 
 while(@ARGV) {
     if($ARGV[0] eq "-v") {
@@ -53,6 +54,11 @@ while(@ARGV) {
         $trace=1;
         shift @ARGV;
     }
+    elsif($ARGV[0] eq "-s") {
+        $strict= $ARGV[1];
+        shift @ARGV;
+        shift @ARGV;
+    }
     elsif($ARGV[0] eq "-l") {
         # only show what alloc that caused a memlimit failure
         $showlimit=1;
@@ -79,10 +85,11 @@ my $file = $ARGV[0] || '';
 
 if(! -f $file) {
     print "Usage: memanalyze.pl [options] <dump file>\n",
-    "Options:\n",
-    " -l  memlimit failure displayed\n",
-    " -v  Verbose\n",
-    " -t  Trace\n";
+        "Options:\n",
+        " -l  memlimit failure displayed\n",
+        " -v  Verbose\n",
+        " -s [path] Strict - warn on mallocs after memlimit is reached\n",
+        " -t  Trace\n";
     exit;
 }
 
@@ -124,18 +131,37 @@ my $addrinfos = 0;
 my $source;
 my $linenum;
 my $function;
+my $memwarn = 0;
 
 my $lnum = 0;
+
+my %overlook;
+
 while(<$fileh>) {
     chomp $_;
     my $line = $_;
     $lnum++;
-    if($line =~ /^LIMIT ([^ ]*):(\d*) (.*)/) {
+    if($line =~ /^FAIL (.*)/) {
+        # for informational purposes
+    }
+    elsif($line =~ /^RESTART ([^ ]*):(\d*)/) {
+        # allow a new LIMIT after this
+        $memwarn = 0;
+    }
+    elsif($line =~ /^OVERLOOK ([^ ]*):(\d*)/) {
+        $overlook{$1, $2} = 1;
+    }
+    elsif($line =~ /^LIMIT ([^ ]*):(\d*) (.*)/) {
         # new memory limit test prefix
         my $i = $3;
         my ($source, $linenum) = ($1, $2);
-        if($trace && ($i =~ /([^ ]*) reached memlimit/)) {
-            print "LIMIT: $1 returned error at $source:$linenum\n";
+        if($i =~ /([^ ]*) reached memlimit/) {
+            if($trace) {
+                print "LIMIT: $1 returned error at $source:$linenum\n";
+            }
+            if($strict && !$overlook{$source, $linenum}) {
+                $memwarn++;
+            }
         }
     }
     elsif($line =~ /^MEM ([^ ]*):(\d*) (.*)/) {

tests/runner.pm

@@ -549,7 +549,7 @@ sub torture {
             $fail=1;
         }
         else {
-            my @memdata=`$memanalyze "$LOGDIR/$MEMDUMP"`;
+            my @memdata=`$memanalyze -s "$srcdir" "$LOGDIR/$MEMDUMP"`;
             my $leak=0;
             for(@memdata) {
                 if($_ ne "") {