Python: Add various kinds of extractor telemetry

python/extractor/cli-integration-test/parser-telemetry/repo_dir/old_parser.py

@@ -0,0 +1 @@
+x = 1

python/extractor/cli-integration-test/parser-telemetry/repo_dir/tree_sitter_parser.py

@@ -0,0 +1,3 @@
+match 1:
+    case 1:
+        pass

python/extractor/cli-integration-test/parser-telemetry/test.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -Eeuo pipefail # see https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
+
+set -x
+
+CODEQL=${CODEQL:-codeql}
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$SCRIPTDIR"
+
+rm -rf db
+
+$CODEQL database create db --language python --source-root repo_dir/
+python3 test_parser_telemetry.py db
+
+rm -rf db

python/extractor/cli-integration-test/parser-telemetry/test_parser_telemetry.py

@@ -0,0 +1,25 @@
+import glob
+import json
+import os
+import sys
+
+
+database = sys.argv[1]
+diagnostics = []
+diagnostic_dir = os.path.join(database, "diagnostic", "extractors", "python")
+for path in glob.glob(os.path.join(diagnostic_dir, "*.jsonl")):
+    with open(path) as diagnostic_file:
+        diagnostics.extend(json.loads(line) for line in diagnostic_file)
+parser_statistics = [
+    diagnostic
+    for diagnostic in diagnostics
+    if diagnostic["source"]["id"] == "py/extractor/parser-statistics"
+]
+actual = (
+    sum(diagnostic["attributes"]["old_parser_file_count"] for diagnostic in parser_statistics),
+    sum(
+        diagnostic["attributes"]["tree_sitter_parser_file_count"]
+        for diagnostic in parser_statistics
+    ),
+)
+assert actual == (1, 1), actual

python/extractor/cli-integration-test/writing-diagnostics/diagnostics.expected

@@ -161,3 +161,24 @@
     "telemetry": true
   }
 }
+{
+  "attributes": {
+    "extractor_flags": "default",
+    "extractor_version": "7.1.10",
+    "python_analysis_version": "3.12",
+    "python_runtime_version": "3.12.3"
+  },
+  "markdownMessage": "Internal telemetry for the Python extractor.\n\nNo action needed.",
+  "severity": "note",
+  "source": {
+    "extractorName": "python",
+    "id": "py/extractor/summary",
+    "name": "Python extractor telemetry"
+  },
+  "timestamp": "2026-09-01T13:41:33.056818Z",
+  "visibility": {
+    "cliSummaryTable": false,
+    "statusPage": false,
+    "telemetry": true
+  }
+}

python/extractor/cli-integration-test/writing-diagnostics/test_diagnostics_output.py

@@ -1,7 +1,28 @@
 import os
 import sys
+import glob
+import json
 sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "..", "integration-tests"))
 import diagnostics_test_utils
 
 test_db = "db"
-diagnostics_test_utils.check_diagnostics(".", test_db, skip_attributes=True)
+diagnostics = []
+diagnostic_dir = os.path.join(test_db, "diagnostic", "extractors", "python")
+for path in glob.glob(os.path.join(diagnostic_dir, "*.jsonl")):
+    with open(path) as diagnostic_file:
+        diagnostics.extend(json.loads(line) for line in diagnostic_file)
+summary = [
+    diagnostic
+    for diagnostic in diagnostics
+    if diagnostic["source"]["id"] == "py/extractor/summary"
+]
+assert len(summary) == 1
+assert summary[0]["attributes"]["extractor_flags"] == "default"
+diagnostics_test_utils.check_diagnostics(
+    ".",
+    test_db,
+    skip_attributes=True,
+    replacements={
+        r'"py/extractor/parser-statistics"': '"cli/py/extractor/parser-statistics"'
+    },
+)

python/extractor/semmle/cmdline.py

@@ -1,4 +1,4 @@
-from optparse import OptionParser, OptionGroup, HelpFormatter
+from optparse import Option, OptionParser, OptionGroup, HelpFormatter
 import shlex
 import sys
 import os
@@ -8,9 +8,21 @@
 from semmle.util import VERSION
 
 
+DEFAULT_AUTOBUILDER_FLAGS = {"R", "c", "v", "verbosity", "z"}
+
+
+class RecordingOption(Option):
+    def process(self, opt, value, values, parser):
+        flag = (self._short_opts or self._long_opts)[0].lstrip("-")
+        if flag not in DEFAULT_AUTOBUILDER_FLAGS:
+            parser.extractor_flags.add(flag)
+        return Option.process(self, opt, value, values, parser)
+
+
 def make_parser():
     '''Parse command_line, returning options, arguments'''
-    parser = OptionParser(add_help_option=False, version='%s' % VERSION)
+    parser = OptionParser(option_class=RecordingOption, add_help_option=False, version='%s' % VERSION)
+    parser.extractor_flags = set()
 
     import_options = OptionGroup(parser, "Import following options",
         description="Note that -a -n -g and -t are included for backwards compatibility. They are ignored")
@@ -172,6 +184,7 @@ def parse(command_line):
                     setattr(options, attr, dval)
         args.extend(extra_args)
     del options.file
+    options.extractor_flags = sorted(parser.extractor_flags)
     if options.help:
         if options.verbose:
             for opt in parser._get_all_options():

python/extractor/semmle/extractors/module_printer.py

@@ -6,9 +6,9 @@ class ModulePrinter(object):
 
     name = "module printer"
 
-    def __init__(self, options, trap_folder, src_archive, renamer, logger):
+    def __init__(self, options, trap_folder, src_archive, renamer, logger, diagnostics_writer):
         self.logger = logger
-        self.py_extractor = PythonExtractor(options, trap_folder, src_archive, logger)
+        self.py_extractor = PythonExtractor(options, trap_folder, src_archive, logger, diagnostics_writer)
 
     def process(self, unit):
         imports = ()

python/extractor/semmle/extractors/py_extractor.py

@@ -16,6 +16,7 @@ def __init__(self, options, trap_folder, src_archive, logger: Logger, diagnostic
         self.module_extractor = extractor.Extractor.from_options(options, trap_folder, src_archive, logger, diagnostics_writer)
         self.finder = finder.Finder.from_options_and_env(options, logger)
         self.importer = imports.importer_from_options(options, self.finder, logger)
+        self.diagnostics_writer = diagnostics_writer
 
     def _get_module_and_imports(self, unit):
         if not isinstance(unit, util.FileExtractable):
@@ -24,7 +25,7 @@ def _get_module_and_imports(self, unit):
         module = self.finder.from_extractable(unit)
         if module is None:
             return None, ()
-        py_module = module.load(self.logger)
+        py_module = module.load(self.logger, self.diagnostics_writer)
         if py_module is None:
             return None, ()
         imports = set(mod.get_extractable() for mod in self.importer.get_imports(module, py_module))

python/extractor/semmle/logging.py

@@ -8,6 +8,9 @@
 import multiprocessing
 import enum
 import datetime
+import platform
+
+from semmle.util import VERSION, get_analysis_version
 
 
 #Use standard Semmle logging levels
@@ -355,6 +358,24 @@ def with_timestamp(self, timestamp):
         self.timestamp = timestamp
         return self
 
+def extractor_telemetry_message(extractor_flags):
+    return (DiagnosticMessage(Source("py/extractor/summary", "Python extractor telemetry"), Severity.NOTE)
+            .markdown("Internal telemetry for the Python extractor.\n\nNo action needed.")
+            .attribute("python_analysis_version", get_analysis_version())
+            .attribute("python_runtime_version", platform.python_version())
+            .attribute("extractor_version", VERSION)
+            .attribute("extractor_flags", " ".join(extractor_flags) or "default")
+            .telemetry()
+    )
+
+def parser_statistics_telemetry_message(old_parser_file_count, tree_sitter_parser_file_count):
+    return (DiagnosticMessage(Source("py/extractor/parser-statistics", "Python parser statistics"), Severity.NOTE)
+            .markdown("Internal parser telemetry for the Python extractor.\n\nNo action needed.")
+            .attribute("old_parser_file_count", old_parser_file_count)
+            .attribute("tree_sitter_parser_file_count", tree_sitter_parser_file_count)
+            .telemetry()
+    )
+
 def get_stack_trace_lines():
     """Creates a stack trace for inclusion into the `attributes` part of a diagnostic message.
     Limits the size of the stack trace to 5000 characters, so as to not make the SARIF file overly big.

python/extractor/semmle/python/finder.py

@@ -65,8 +65,8 @@ def all_sub_modules(self):
     def get_extractable(self):
         return FileExtractable(self.path)
 
-    def load(self, logger=None):
-        return PythonSourceModule(self.name, self.path, logger=logger)
+    def load(self, logger, diagnostics_writer):
+        return PythonSourceModule(self.name, self.path, logger=logger, diagnostics_writer=diagnostics_writer)
 
     def __str__(self):
         return "Python module at %s" % self.path

python/extractor/semmle/python/modules.py

@@ -18,7 +18,7 @@ class PythonSourceModule(object):
 
     kind = None
 
-    def __init__(self, name, path, logger, bytes_source = None):
+    def __init__(self, name, path, logger, diagnostics_writer, bytes_source = None):
         assert isinstance(path, str), path
         self.name = name # May be None
         self.path = path
@@ -34,6 +34,7 @@ def __init__(self, name, path, logger, bytes_source = None):
         self._line_types = None
         self._comments = None
         self._tokens = None
+        self.diagnostics_writer = diagnostics_writer
         self.logger = logger
         with timers["decode"]:
             self.encoding, self.bytes_source  = semmle.python.parser.tokenizer.encoding_from_source(bytes_source)
@@ -113,6 +114,7 @@ def old_py_ast(self):
                 self.logger.debug("Trying old parser on %s", self.path)
                 self._py_ast = semmle.python.parser.parse(self.tokens, self.logger)
                 self.logger.debug("Old parser successful on %s", self.path)
+                self.diagnostics_writer.record_old_parser()
         else:
             self.logger.debug("Found (during old_py_ast) parse tree for %s in cache", self.path)
         return self._py_ast
@@ -147,6 +149,7 @@ def py_ast(self):
                             self.logger.debug("Trying tsg-python on %s", self.path)
                             self._py_ast = semmle.python.parser.tsg_parser.parse(self.path, self.logger)
                             self.logger.debug("tsg-python successful on %s", self.path)
+                            self.diagnostics_writer.record_tree_sitter_parser()
                         else:
                             self.logger.debug("Found (during py_ast) parse tree for %s in cache", self.path)
                     return self._py_ast

python/extractor/semmle/python/parser/dump_ast.py

@@ -119,7 +119,8 @@ def reset_error_count(self):
         self.error_count = 0
 
 def old_parser(inputfile, logger):
-    mod = PythonSourceModule(None, inputfile, logger)
+    from semmle.worker import DiagnosticsWriter
+    mod = PythonSourceModule(None, inputfile, logger, DiagnosticsWriter(0))
     logger.close()
     return mod.old_py_ast
 

python/extractor/semmle/python/passes/flow.py

@@ -1916,7 +1916,8 @@ def write_ssa_phi(out, phi, arg):
         import semmle.python.parser.tsg_parser
         parsed_ast = semmle.python.parser.tsg_parser.parse(inputfile, FakeLogger())
     else:
-        module = modules.PythonSourceModule("__main__", inputfile, FakeLogger())
+        from semmle.worker import DiagnosticsWriter
+        module = modules.PythonSourceModule("__main__", inputfile, FakeLogger(), DiagnosticsWriter(0))
         parsed_ast = module.ast
     FlowPass(options.split, options.prune, options.unroll).extract(parsed_ast, writer)
     writer.close()

python/extractor/semmle/util.py

@@ -10,7 +10,7 @@
 
 #Semantic version of extractor.
 #Update this if any changes are made
-VERSION = "7.1.9"
+VERSION = "7.1.10"
 
 PY_EXTENSIONS = ".py", ".pyw"
 

python/extractor/semmle/worker.py

@@ -10,7 +10,8 @@
 from semmle.extractors import SuperExtractor, ModulePrinter, SkippedBuiltin
 from semmle.profiling import get_profiler
 from semmle.path_rename import renamer_from_options_and_env
-from semmle.logging import WARN, recursion_error_message, internal_error_message, Logger
+from semmle.logging import WARN, recursion_error_message, internal_error_message, extractor_telemetry_message, Logger
+from semmle.logging import parser_statistics_telemetry_message
 from semmle.util import FileExtractable, FolderExtractable
 
 class ExtractorFailure(Exception):
@@ -239,9 +240,35 @@ def _drain_queue(queue):
         #Emptied queue as best we can.
         pass
 
+def _write_extractor_telemetry(diagnostics_writer, logger: Logger, extractor_flags):
+    try:
+        diagnostics_writer.write(extractor_telemetry_message(extractor_flags))
+    except OSError as ex:
+        logger.warning("Failed to write extractor telemetry: %s", ex)
+
+def _write_parser_statistics_telemetry(diagnostics_writer, logger: Logger):
+    counts = diagnostics_writer.parser_statistics()
+    if counts == (0, 0):
+        return
+    try:
+        diagnostics_writer.write(parser_statistics_telemetry_message(*counts))
+    except OSError as ex:
+        logger.warning("Failed to write parser statistics telemetry: %s", ex)
+
 class DiagnosticsWriter(object):
     def __init__(self, proc_id):
         self.proc_id = proc_id
+        self.old_parser_file_count = 0
+        self.tree_sitter_parser_file_count = 0
+
+    def record_old_parser(self):
+        self.old_parser_file_count += 1
+
+    def record_tree_sitter_parser(self):
+        self.tree_sitter_parser_file_count += 1
+
+    def parser_statistics(self):
+        return self.old_parser_file_count, self.tree_sitter_parser_file_count
 
     def write(self, message):
         dir = os.environ.get("CODEQL_EXTRACTOR_PYTHON_DIAGNOSTIC_DIR")
@@ -276,9 +303,11 @@ def _extract_loop(proc_id, queue, trap_dir, archive, options, reply_queue, logge
         reply_queue.put(("INTERRUPT", None, None))
         sys.exit(2)
     logger.set_process_id(proc_id)
+    if write_global_data:
+        _write_extractor_telemetry(diagnostics_writer, logger, options.extractor_flags)
     try:
         if options.trace_only:
-            extractor = ModulePrinter(options, trap_dir, archive, renamer, logger)
+            extractor = ModulePrinter(options, trap_dir, archive, renamer, logger, diagnostics_writer)
         else:
             extractor = SuperExtractor(options, trap_dir, archive, renamer, logger, diagnostics_writer)
         profiler = get_profiler(options, id, logger)
@@ -291,6 +320,7 @@ def _extract_loop(proc_id, queue, trap_dir, archive, options, reply_queue, logge
                     if write_global_data:
                         extractor.write_global_data()
                     extractor.close()
+                    _write_parser_statistics_telemetry(diagnostics_writer, logger)
                     return
                 try:
                     start = time.time()
@@ -344,4 +374,5 @@ def _extract_loop(proc_id, queue, trap_dir, archive, options, reply_queue, logge
     except _Empty:
         #Cleared queue enough to avoid deadlock.
         pass
+    _write_parser_statistics_telemetry(diagnostics_writer, logger)
     sys.exit(2)

python/extractor/tests/test_cmdline.py

@@ -0,0 +1,34 @@
+from semmle import cmdline
+
+
+def test_records_flags_without_values():
+    options, args = cmdline.parse(
+        [
+            "--verbosity=3",
+            "-zall",
+            "-R",
+            "/src",
+            "-vv",
+            "--path",
+            "/lib",
+            "-p",
+            "/other-lib",
+            "module",
+        ]
+    )
+
+    assert options.extractor_flags == ["p"]
+    assert args == ["module"]
+
+
+def test_records_flags_from_option_file(tmp_path):
+    options_file = tmp_path / "extractor-options"
+    options_file.write_text("--colorize --max-import-depth 2")
+
+    options, _ = cmdline.parse(["-f", str(options_file)])
+
+    assert options.extractor_flags == [
+        "colorize",
+        "f",
+        "max-import-depth",
+    ]