Python: Add extractor flag telemetry

Records any non-default extractor flags (without their arguments) as a
normalised string. This will enable us to determine which flags are
actually used (and which ones we might therefore get rid of).

When there are no flags other than the ones the autobuilder injects, we
simply report the string `"default"`. That way, there's no need to
remember exactly which flags are enabled by default during extraction.

Author: tausbn

python/extractor/cli-integration-test/writing-diagnostics/diagnostics.expected

@@ -163,6 +163,7 @@
 }
 {
   "attributes": {
+    "extractor_flags": "default",
     "extractor_version": "7.1.10",
     "python_analysis_version": "3.12",
     "python_runtime_version": "3.12.3"

python/extractor/cli-integration-test/writing-diagnostics/test_diagnostics_output.py

@@ -11,6 +11,13 @@
 for path in glob.glob(os.path.join(diagnostic_dir, "*.jsonl")):
     with open(path) as diagnostic_file:
         diagnostics.extend(json.loads(line) for line in diagnostic_file)
+summary = [
+    diagnostic
+    for diagnostic in diagnostics
+    if diagnostic["source"]["id"] == "py/extractor/summary"
+]
+assert len(summary) == 1
+assert summary[0]["attributes"]["extractor_flags"] == "default"
 parser_statistics = [
     diagnostic
     for diagnostic in diagnostics

python/extractor/semmle/cmdline.py

@@ -1,4 +1,4 @@
-from optparse import OptionParser, OptionGroup, HelpFormatter
+from optparse import Option, OptionParser, OptionGroup, HelpFormatter
 import shlex
 import sys
 import os
@@ -8,9 +8,21 @@
 from semmle.util import VERSION
 
 
+DEFAULT_AUTOBUILDER_FLAGS = {"R", "c", "v", "verbosity", "z"}
+
+
+class RecordingOption(Option):
+    def process(self, opt, value, values, parser):
+        flag = (self._short_opts or self._long_opts)[0].lstrip("-")
+        if flag not in DEFAULT_AUTOBUILDER_FLAGS:
+            parser.extractor_flags.add(flag)
+        return Option.process(self, opt, value, values, parser)
+
+
 def make_parser():
     '''Parse command_line, returning options, arguments'''
-    parser = OptionParser(add_help_option=False, version='%s' % VERSION)
+    parser = OptionParser(option_class=RecordingOption, add_help_option=False, version='%s' % VERSION)
+    parser.extractor_flags = set()
 
     import_options = OptionGroup(parser, "Import following options",
         description="Note that -a -n -g and -t are included for backwards compatibility. They are ignored")
@@ -172,6 +184,7 @@ def parse(command_line):
                     setattr(options, attr, dval)
         args.extend(extra_args)
     del options.file
+    options.extractor_flags = sorted(parser.extractor_flags)
     if options.help:
         if options.verbose:
             for opt in parser._get_all_options():

python/extractor/semmle/logging.py

@@ -358,12 +358,13 @@ def with_timestamp(self, timestamp):
         self.timestamp = timestamp
         return self
 
-def extractor_telemetry_message():
+def extractor_telemetry_message(extractor_flags):
     return (DiagnosticMessage(Source("py/extractor/summary", "Python extractor telemetry"), Severity.NOTE)
             .markdown("Internal telemetry for the Python extractor.\n\nNo action needed.")
             .attribute("python_analysis_version", get_analysis_version())
             .attribute("python_runtime_version", platform.python_version())
             .attribute("extractor_version", VERSION)
+            .attribute("extractor_flags", " ".join(extractor_flags) or "default")
             .telemetry()
     )
 

python/extractor/semmle/worker.py

@@ -240,9 +240,9 @@ def _drain_queue(queue):
         #Emptied queue as best we can.
         pass
 
-def _write_extractor_telemetry(diagnostics_writer, logger: Logger):
+def _write_extractor_telemetry(diagnostics_writer, logger: Logger, extractor_flags):
     try:
-        diagnostics_writer.write(extractor_telemetry_message())
+        diagnostics_writer.write(extractor_telemetry_message(extractor_flags))
     except OSError as ex:
         logger.warning("Failed to write extractor telemetry: %s", ex)
 
@@ -304,7 +304,7 @@ def _extract_loop(proc_id, queue, trap_dir, archive, options, reply_queue, logge
         sys.exit(2)
     logger.set_process_id(proc_id)
     if write_global_data:
-        _write_extractor_telemetry(diagnostics_writer, logger)
+        _write_extractor_telemetry(diagnostics_writer, logger, options.extractor_flags)
     try:
         if options.trace_only:
             extractor = ModulePrinter(options, trap_dir, archive, renamer, logger, diagnostics_writer)

python/extractor/tests/test_cmdline.py

@@ -0,0 +1,34 @@
+from semmle import cmdline
+
+
+def test_records_flags_without_values():
+    options, args = cmdline.parse(
+        [
+            "--verbosity=3",
+            "-zall",
+            "-R",
+            "/src",
+            "-vv",
+            "--path",
+            "/lib",
+            "-p",
+            "/other-lib",
+            "module",
+        ]
+    )
+
+    assert options.extractor_flags == ["p"]
+    assert args == ["module"]
+
+
+def test_records_flags_from_option_file(tmp_path):
+    options_file = tmp_path / "extractor-options"
+    options_file.write_text("--colorize --max-import-depth 2")
+
+    options, _ = cmdline.parse(["-f", str(options_file)])
+
+    assert options.extractor_flags == [
+        "colorize",
+        "f",
+        "max-import-depth",
+    ]

python/extractor/tests/test_diagnostics.py

@@ -9,7 +9,7 @@
 def test_extractor_telemetry_message(mocker):
     mocker.patch("semmle.logging.get_analysis_version", return_value="3.13")
 
-    message = logging.extractor_telemetry_message().to_dict()
+    message = logging.extractor_telemetry_message(["colorize", "p"]).to_dict()
     message.pop("timestamp")
 
     assert message == {
@@ -29,6 +29,7 @@ def test_extractor_telemetry_message(mocker):
             "python_analysis_version": "3.13",
             "python_runtime_version": platform.python_version(),
             "extractor_version": util.VERSION,
+            "extractor_flags": "colorize p",
         },
     }
 
@@ -59,17 +60,24 @@ def test_parser_statistics_telemetry_message():
     }
 
 
+def test_extractor_telemetry_message_includes_empty_flags():
+    message = logging.extractor_telemetry_message([]).to_dict()
+
+    assert message["attributes"]["extractor_flags"] == "default"
+
+
 def test_write_extractor_telemetry(mocker):
     diagnostics_writer = mocker.Mock()
     logger = mocker.Mock()
 
-    worker._write_extractor_telemetry(diagnostics_writer, logger)
+    worker._write_extractor_telemetry(diagnostics_writer, logger, ["quiet"])
 
     diagnostics_writer.write.assert_called_once()
     assert diagnostics_writer.write.call_args.args[0].to_dict()["attributes"] == {
         "python_analysis_version": util.get_analysis_version(),
         "python_runtime_version": platform.python_version(),
         "extractor_version": util.VERSION,
+        "extractor_flags": "quiet",
     }
     logger.warning.assert_not_called()
 
@@ -79,7 +87,7 @@ def test_write_extractor_telemetry_handles_io_error(mocker):
     diagnostics_writer.write.side_effect = OSError("write failed")
     logger = mocker.Mock()
 
-    worker._write_extractor_telemetry(diagnostics_writer, logger)
+    worker._write_extractor_telemetry(diagnostics_writer, logger, [])
 
     logger.warning.assert_called_once_with(
         "Failed to write extractor telemetry: %s", diagnostics_writer.write.side_effect