checksrc: avoid extra runs in CI, enable more check locally, fix fallouts

To avoid redundant work in CI and to avoid a single checksrc issue make
all autotools jobs fail. After this patch checksrc issues make fail
the checksrc job, the `dist / verify-out-of-tree-autotools-debug`,
`dist / maketgz-and-verify-in-tree`  jobs and the fuzzer job (if run).
Of these, the `dist` jobs replicate local builds, also testing the build
logic.

Also add a script to check the complete local repository, optionally
with the build tree to verify generated C files.

Also:
- automatically run checksrc in subdirectories having a `checksrc`
  target. (examples, OS400, tests http/client, unit and tunit)
- tests/libtest: make sure to run `checksrc` on generated `lib1521.c`.
  (requires in-tree autotools build.)
- tests: run `checksrc` on targets also for non-`DEBUGBUILD`
  builds. It ensures to check `lib1521.c` in CI via job
  `dist / maketgz-and-verify-in-tree`.
- src: drop redundant `$(builddir)` in autotools builds.
- scripts: add `checksrc-all.sh` script to check all C sources and
  the build directory as an option.
- use the above from CI, also make it verify all generated sources.
- silence `checksrc` issues in generated C sources.
- checksrc: add `-v` option to enable verbose mode.
- checksrc: make verbose mode show checked filename and fix to only
  return error on failure.
- make sure that generated C files pass `checksrc`.

Assisted-by: Daniel Stenberg

Closes #17376

src/CMakeLists.txt

@@ -33,6 +33,9 @@ if(ENABLE_CURL_MANUAL AND HAVE_MANUAL_TOOLS)
   add_custom_command(
     OUTPUT "tool_hugehelp.c"
     COMMAND ${CMAKE_COMMAND} -E echo "#include \"tool_setup.h\"" > "tool_hugehelp.c"
+    COMMAND ${CMAKE_COMMAND} -E echo "/* !checksrc! disable COPYRIGHT all */" >> "tool_hugehelp.c"
+    COMMAND ${CMAKE_COMMAND} -E echo "/* !checksrc! disable INCLUDEDUP all */" >> "tool_hugehelp.c"
+    COMMAND ${CMAKE_COMMAND} -E echo "/* !checksrc! disable LONGLINE all */" >> "tool_hugehelp.c"
     COMMAND ${CMAKE_COMMAND} -E echo "#ifndef HAVE_LIBZ" >> "tool_hugehelp.c"
     COMMAND "${PERL_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/mkhelp.pl"    < "${CURL_ASCIIPAGE}" >> "tool_hugehelp.c"
     COMMAND ${CMAKE_COMMAND} -E echo "#else" >> "tool_hugehelp.c"

src/Makefile.am

@@ -125,7 +125,7 @@ endif
 # Use absolute directory to disable VPATH
 ASCIIPAGE=$(top_builddir)/docs/cmdline-opts/curl.txt
 MKHELP=$(top_srcdir)/src/mkhelp.pl
-HUGE=$(builddir)/tool_hugehelp.c
+HUGE=tool_hugehelp.c
 
 HUGECMD = $(HUGEIT_$(V))
 HUGEIT_0 = @echo "  HUGE    " $@;
@@ -145,22 +145,29 @@ if HAVE_LIBZ
 # This generates the tool_hugehelp.c file in both uncompressed and
 # compressed formats.
 $(HUGE): $(ASCIIPAGE) $(MKHELP)
-	$(HUGECMD) (echo '#include "tool_setup.h"' > $(HUGE);   \
-	echo '#ifndef HAVE_LIBZ' >> $(HUGE);                    \
-	$(PERL) $(MKHELP) < $(ASCIIPAGE) >> $(HUGE);     \
-	echo '#else' >> $(HUGE);                                \
-	$(PERL) $(MKHELP) -c < $(ASCIIPAGE) >> $(HUGE);  \
+	$(HUGECMD)(                                                \
+	echo '/* !checksrc! disable COPYRIGHT all */' > $(HUGE);   \
+	echo '/* !checksrc! disable INCLUDEDUP all */' >> $(HUGE); \
+	echo '/* !checksrc! disable LONGLINE all */' >> $(HUGE);   \
+	echo '#include "tool_setup.h"' >> $(HUGE);                 \
+	echo '#ifndef HAVE_LIBZ' >> $(HUGE);                       \
+	$(PERL) $(MKHELP) < $(ASCIIPAGE) >> $(HUGE);               \
+	echo '#else' >> $(HUGE);                                   \
+	$(PERL) $(MKHELP) -c < $(ASCIIPAGE) >> $(HUGE);            \
 	echo '#endif /* HAVE_LIBZ */' >> $(HUGE) )
 else # HAVE_LIBZ
 # This generates the tool_hugehelp.c file uncompressed only
 $(HUGE): $(ASCIIPAGE) $(MKHELP)
-	$(HUGECMD)(echo '#include "tool_setup.h"' > $(HUGE);    \
+	$(HUGECMD)(                                                \
+	echo '/* !checksrc! disable COPYRIGHT all */' > $(HUGE);   \
+	echo '#include "tool_setup.h"' >> $(HUGE);                 \
 	$(PERL) $(MKHELP) < $(ASCIIPAGE) >> $(HUGE) )
 endif
 
 else # USE_MANUAL
 # built-in manual has been disabled, make a blank file
 $(HUGE):
+	echo '/* !checksrc! disable COPYRIGHT all */' > $(HUGE);   \
 	echo '#include "tool_hugehelp.h"' >> $(HUGE)
 endif
 
@@ -168,7 +175,7 @@ curl_cfiles_gen += $(HUGE)
 curl_hfiles_gen += tool_hugehelp.h
 CLEANFILES += $(HUGE)
 
-CA_EMBED_CSOURCE = $(builddir)/tool_ca_embed.c
+CA_EMBED_CSOURCE = tool_ca_embed.c
 curl_cfiles_gen += $(CA_EMBED_CSOURCE)
 CLEANFILES += $(CA_EMBED_CSOURCE)
 if CURL_CA_EMBED_SET
@@ -178,7 +185,8 @@ $(CA_EMBED_CSOURCE): $(MK_FILE_EMBED) $(CURL_CA_EMBED)
 	$(PERL) $(MK_FILE_EMBED) --var curl_ca_embed < $(CURL_CA_EMBED) > $(CA_EMBED_CSOURCE)
 else
 $(CA_EMBED_CSOURCE):
-	echo 'extern const void *curl_ca_embed; const void *curl_ca_embed;' > $(CA_EMBED_CSOURCE)
+	echo '/* !checksrc! disable COPYRIGHT all */' > $(CA_EMBED_CSOURCE)
+	echo 'extern const void *curl_ca_embed; const void *curl_ca_embed;' >> $(CA_EMBED_CSOURCE)
 endif
 
 CHECKSRC = $(CS_$(V))
@@ -190,10 +198,12 @@ CS_ = $(CS_0)
 checksrc:
 	$(CHECKSRC)(@PERL@ $(top_srcdir)/scripts/checksrc.pl -D$(srcdir) $(CURL_CFILES) $(CURL_HFILES))
 
+if NOT_CURL_CI
 if DEBUGBUILD
 # for debug builds, we scan the sources on all regular make invokes
 all-local: checksrc
 endif
+endif
 
 # disable the tests that are mostly causing false positives
 TIDYFLAGS := -checks=-clang-analyzer-security.insecureAPI.strcpy,-clang-analyzer-optin.performance.Padding,-clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling -quiet

src/mk-file-embed.pl

@@ -35,6 +35,7 @@ print <<HEAD
 /*
  * NEVER EVER edit this manually, fix the mk-file-embed.pl script instead!
  */
+/* !checksrc! disable COPYRIGHT all */
 #ifndef CURL_DECLARED_${varname_upper}
 #define CURL_DECLARED_${varname_upper}
 extern const unsigned char ${varname}[];