scorecard: flame graphs and documentation

Add `--flame` option to scorecard.py for generating flame graphs.
Add documentation in docs/internal/SCORECARD.md on how to use this.

Closes #17792

tests/http/testenv/curl.py

@@ -114,13 +114,17 @@ class DTraceProfile:
         self._pid = pid
         self._run_dir = run_dir
         self._proc = None
+        self._rc = 0
+        self._file = os.path.join(self._run_dir, 'curl.user_stacks')
 
     def start(self):
+        if os.path.exists(self._file):
+            os.remove(self._file)
         args = [
             'sudo', 'dtrace',
             '-x', 'ustackframes=100',
             '-n', f'profile-97 /pid == {self._pid}/ {{ @[ustack()] = count(); }} tick-60s {{ exit(0); }}',
-            '-o', f'{self._run_dir}/curl.user_stacks'
+            '-o', f'{self._file}'
         ]
         self._proc = subprocess.Popen(args, text=True, cwd=self._run_dir, shell=False)
         assert self._proc
@@ -128,6 +132,11 @@ class DTraceProfile:
     def finish(self):
         if self._proc:
             self._proc.terminate()
+            self._rc = self._proc.returncode
+
+    @property
+    def file(self):
+        return self._file
 
 
 class RunTcpDump:
@@ -490,7 +499,8 @@ class CurlClient:
                  silent: bool = False,
                  run_env: Optional[Dict[str, str]] = None,
                  server_addr: Optional[str] = None,
-                 with_dtrace: bool = False):
+                 with_dtrace: bool = False,
+                 with_flame: bool = False):
         self.env = env
         self._timeout = timeout if timeout else env.test_timeout
         self._curl = os.environ['CURL'] if 'CURL' in os.environ else env.curl
@@ -500,6 +510,9 @@ class CurlClient:
         self._headerfile = f'{self._run_dir}/curl.headers'
         self._log_path = f'{self._run_dir}/curl.log'
         self._with_dtrace = with_dtrace
+        self._with_flame = with_flame
+        if self._with_flame:
+            self._with_dtrace = True
         self._silent = silent
         self._run_env = run_env
         self._server_addr = server_addr if server_addr else '127.0.0.1'
@@ -792,10 +805,11 @@ class CurlClient:
         exception = None
         profile = None
         tcpdump = None
-        started_at = datetime.now()
+        dtrace = None
         if with_tcpdump:
             tcpdump = RunTcpDump(self.env, self._run_dir)
             tcpdump.start()
+        started_at = datetime.now()
         try:
             with open(self._stdoutfile, 'w') as cout, open(self._stderrfile, 'w') as cerr:
                 if with_profile:
@@ -824,8 +838,6 @@ class CurlClient:
                             ptimeout = 0.01
                     exitcode = p.returncode
                     profile.finish()
-                    if self._with_dtrace:
-                        dtrace.finish()
                     log.info(f'done: exit={exitcode}, profile={profile}')
                 else:
                     p = subprocess.run(args, stderr=cerr, stdout=cout,
@@ -841,13 +853,18 @@ class CurlClient:
                         f'(configured {self._timeout}s): {args}')
             exitcode = -1
             exception = 'TimeoutExpired'
+        ended_at = datetime.now()
         if tcpdump:
             tcpdump.finish()
+        if dtrace:
+            dtrace.finish()
+        if self._with_flame and dtrace:
+            self._generate_flame(dtrace, args)
         coutput = open(self._stdoutfile).readlines()
         cerrput = open(self._stderrfile).readlines()
         return ExecResult(args=args, exit_code=exitcode, exception=exception,
                           stdout=coutput, stderr=cerrput,
-                          duration=datetime.now() - started_at,
+                          duration=ended_at - started_at,
                           with_stats=with_stats,
                           profile=profile, tcpdump=tcpdump)
 
@@ -976,3 +993,52 @@ class CurlClient:
 
         fin_response(response)
         return r
+
+    def _generate_flame(self, dtrace: DTraceProfile, curl_args: List[str]):
+        log.info('generating flame graph from dtrace for this run')
+        if not os.path.exists(dtrace.file):
+            raise Exception(f'dtrace output file does not exist: {dtrace.file}')
+        if 'FLAMEGRAPH' not in os.environ:
+            raise Exception('Env variable FLAMEGRAPH not set')
+        fg_dir = os.environ['FLAMEGRAPH']
+        if not os.path.exists(fg_dir):
+            raise Exception(f'FlameGraph directory not found: {fg_dir}')
+
+        fg_collapse = os.path.join(fg_dir, 'stackcollapse.pl')
+        if not os.path.exists(fg_collapse):
+            raise Exception(f'FlameGraph script not found: {fg_collapse}')
+
+        fg_gen_flame = os.path.join(fg_dir, 'flamegraph.pl')
+        if not os.path.exists(fg_gen_flame):
+            raise Exception(f'FlameGraph script not found: {fg_gen_flame}')
+
+        file_collapsed = f'{dtrace.file}.collapsed'
+        file_svg = os.path.join(self._run_dir, 'curl.flamegraph.svg')
+        file_err = os.path.join(self._run_dir, 'curl.flamegraph.stderr')
+        log.info('waiting a sec for dtrace to finish flusheing its buffers')
+        time.sleep(1)
+        log.info(f'collapsing stacks into {file_collapsed}')
+        with open(file_collapsed, 'w') as cout, open(file_err, 'w') as cerr:
+            p = subprocess.run([
+                fg_collapse, dtrace.file
+            ], stdout=cout, stderr=cerr, cwd=self._run_dir, shell=False)
+            rc = p.returncode
+            if rc != 0:
+                raise Exception(f'{fg_collapse} returned error {rc}')
+        log.info(f'generating graph into {file_svg}')
+        cmdline = ' '.join(curl_args)
+        if len(cmdline) > 80:
+            title = f'{cmdline[:80]}...'
+            subtitle = f'...{cmdline[-80:]}'
+        else:
+            title = cmdline
+            subtitle = ''
+        with open(file_svg, 'w') as cout, open(file_err, 'w') as cerr:
+            p = subprocess.run([
+                fg_gen_flame, '--colors', 'green',
+                '--title', title, '--subtitle', subtitle,
+                file_collapsed
+            ], stdout=cout, stderr=cerr, cwd=self._run_dir, shell=False)
+            rc = p.returncode
+            if rc != 0:
+                raise Exception(f'{fg_gen_flame} returned error {rc}')