[SDTEST-3873] Add extended telemetry

docs/error-codes.md

@@ -0,0 +1,61 @@
+# DDTest error codes
+
+Fatal `ddtest plan` and `ddtest run` errors include a stable error code in the
+form `[error_code] error message`. The same value is reported by the
+`error_code` tag on the `ddtest.cli.command` and `ddtest.cli.command_ms`
+telemetry metrics.
+
+Error codes identify the actionable failure point while the rest of the error
+message and its wrapped Go error retain the specific OS, platform, framework,
+or API cause. Existing codes must not be reused for a different condition.
+
+## Special telemetry values
+
+| Code | Meaning |
+| --- | --- |
+| `none` | The command completed successfully. |
+| `unknown` | An error from an external or injected implementation did not contain a DDTest error code. Production plan and run failure paths should not use this value. |
+
+## Planning errors
+
+| Code | Condition |
+| --- | --- |
+| `plan_git_unavailable` | Git was not installed or could not be found before planning. |
+| `plan_platform_detection_failed` | The configured platform could not be selected or did not pass its sanity check. |
+| `plan_platform_tags_creation_failed` | Runtime or operating-system tags could not be collected from the selected platform. |
+| `plan_runtime_tags_invalid` | The `runtime-tags` override could not be parsed. |
+| `plan_framework_detection_failed` | The configured test framework is not supported or could not be initialized for the selected platform. |
+| `plan_optimization_client_creation_failed` | The Test Optimization client could not be created. |
+| `plan_test_files_resolution_failed` | The test include/exclude patterns could not be resolved or the test-file scan failed. |
+| `plan_optimization_client_initialization_failed` | The Test Optimization client failed during initialization. |
+| `plan_full_test_discovery_failed` | Required full test discovery failed while strict discovery was enabled. |
+| `plan_fast_test_discovery_failed` | Fast test-file discovery failed and no full-discovery result was available. |
+| `plan_full_discovery_results_processing_failed` | Full-discovery results could not be matched to the configured test selection. |
+| `plan_fast_discovery_results_processing_failed` | Fast-discovery results could not be matched to the configured test selection. |
+| `plan_manifest_write_failed` | The Test Optimization manifest could not be written. |
+| `plan_cache_write_failed` | The Test Optimization plan cache could not be stored. |
+| `plan_test_files_write_failed` | The selected test-files artifact could not be written. |
+| `plan_skippable_percentage_write_failed` | The skippable-percentage artifact could not be written. |
+| `plan_parallel_runners_write_failed` | The parallel-runner-count artifact could not be written. |
+| `plan_test_splits_write_failed` | Test split artifacts could not be created or written. |
+
+## Run errors
+
+| Code | Condition |
+| --- | --- |
+| `run_git_unavailable` | Git was not installed or could not be found before running tests. |
+| `run_planning_failed` | The automatic planning phase returned an unclassified error. A classified planning failure retains its more precise `plan_*` code. |
+| `run_plan_status_check_failed` | DDTest could not check whether planning artifacts exist. |
+| `run_plan_load_failed` | The Test Optimization plan cache could not be loaded. |
+| `run_parallel_runners_read_failed` | The parallel-runner-count artifact could not be read. |
+| `run_parallel_runners_parse_failed` | The parallel-runner-count artifact did not contain a valid integer. |
+| `run_platform_detection_failed` | The configured platform could not be selected or did not pass its sanity check before running tests. |
+| `run_framework_detection_failed` | The configured test framework is not supported or could not be initialized before running tests. |
+| `run_sequential_test_files_read_failed` | The sequential test-files artifact could not be read. |
+| `run_sequential_tests_failed` | The test framework failed while running the sequential test batch. |
+| `run_parallel_splits_read_failed` | The test-splits directory could not be read. |
+| `run_parallel_test_files_read_failed` | A test split file could not be read. |
+| `run_parallel_tests_failed` | The test framework failed in a local parallel worker. |
+| `run_ci_node_test_files_missing` | The requested CI-node split file does not exist. |
+| `run_ci_node_test_files_read_failed` | The requested CI-node split file could not be read. |
+| `run_ci_node_tests_failed` | The test framework failed in a CI-node worker. |

docs/telemetry-metrics.md

@@ -0,0 +1,51 @@
+# DDTest telemetry metrics
+
+This document tracks DDTest-specific metrics that must be added to the
+[`civisibility` namespace in `common_metrics.json`](https://github.com/DataDog/dd-go/blob/prod/trace/apps/tracer-telemetry-intake/telemetry-metrics/static/common_metrics.json).
+The names below are the metric names emitted in telemetry payloads; the intake
+adds the `dd.instrumentation_telemetry_data.civisibility.` prefix. All metrics
+are common metrics and are not sent to customer organizations.
+
+## Pending allowlist additions
+
+| Metric | Type | Data type | Allowed tags | Description |
+| --- | --- | --- | --- | --- |
+| `ddtest.cli.command` | count | command | `command`, `exit_code`, `error_code`, `platform`, `framework`, `test_skipping_mode` | Number of completed top-level ddtest commands. `command` is `plan` or `run`; `exit_code` is `0` or `1`; `error_code` is a value from the [DDTest error code catalog](error-codes.md); the remaining tags contain the resolved CLI configuration. |
+| `ddtest.cli.command_ms` | distribution | milliseconds | `command`, `exit_code`, `error_code`, `platform`, `framework`, `test_skipping_mode` | Duration of a top-level ddtest command, tagged by command, exit code, error code, and resolved CLI configuration. |
+| `ddtest.itr_skippable_tests.is_empty` | count | responses | None | Number of successful skippable-tests fetches that returned zero skippable tests or suites. |
+| `ddtest.planning.decision` | count | plans | `platform`, `framework`, `test_skipping_mode`, `discovery_mode`, `tia_enabled`, `reason`, `target_status` | Number of completed plans. `reason` explains the constraint that selected the parallel runner split; `target_status` is `disabled`, `met`, or `missed`. |
+| `ddtest.planning.test_files` | distribution | test files | `platform`, `framework`, `test_skipping_mode`, `discovery_mode`, `tia_enabled`, `state` | Number of test files at each planning stage. `state` is `discovered`, `runnable`, or `fully_skipped`. |
+| `ddtest.planning.estimated_time_saved_pct` | distribution | percentage | `platform`, `framework`, `test_skipping_mode`, `discovery_mode`, `tia_enabled` | Estimated percentage of test runtime saved by skipping decisions. |
+| `ddtest.planning.test_file_durations` | distribution | test files | `platform`, `framework`, `test_skipping_mode`, `discovery_mode`, `tia_enabled`, `source` | Number of runnable test files weighted using `backend` durations or `default` estimates. |
+| `ddtest.planning.parallel_runners` | distribution | runners | `platform`, `framework`, `test_skipping_mode`, `discovery_mode`, `tia_enabled` | Number of parallel runners selected by the planner. |
+| `ddtest.planning.expected_full_runtime_ms` | distribution | milliseconds | `platform`, `framework`, `test_skipping_mode`, `discovery_mode`, `tia_enabled` | Estimated serial runtime of all discovered test files before skipping. |
+| `ddtest.planning.expected_runnable_runtime_ms` | distribution | milliseconds | `platform`, `framework`, `test_skipping_mode`, `discovery_mode`, `tia_enabled` | Estimated serial runtime after skipping decisions. |
+| `ddtest.planning.expected_wall_time_ms` | distribution | milliseconds | `platform`, `framework`, `test_skipping_mode`, `discovery_mode`, `tia_enabled` | Estimated wall time for the selected parallel runner split. |
+| `ddtest.planning.split_imbalance_pct` | distribution | percentage | `platform`, `framework`, `test_skipping_mode`, `discovery_mode`, `tia_enabled` | Difference between the most- and least-loaded runners as a percentage of expected wall time. |
+| `ddtest.planning.disabled_tests` | distribution | tests | `platform`, `framework`, `test_skipping_mode`, `discovery_mode`, `tia_enabled` | Number of Test Management-disabled tests applied during planning. |
+| `ddtest.planning.forced_run_suites` | distribution | suites | `platform`, `framework`, `test_skipping_mode`, `discovery_mode`, `tia_enabled` | Number of otherwise-skippable suites kept runnable by an unskippable marker. |
+| `ddtest.test_discovery.duration_ms` | distribution | milliseconds | `discovery_mode`, `success`, `platform`, `framework` | Duration of the discovery strategy selected by the planner. `discovery_mode` is `full` for test discovery or `fast` for test-file discovery; `success` reports whether that selected strategy completed successfully. |
+| `ddtest.test_discovery.tests` | distribution | tests | `discovery_mode`, `success`, `platform`, `framework` | Number of tests returned by selected full test discovery. Emitted only with `discovery_mode:full`. |
+| `ddtest.test_discovery.test_files` | distribution | test files | `discovery_mode`, `success`, `platform`, `framework` | Number of test files returned by selected fast test-file discovery. Emitted only with `discovery_mode:fast`. |
+| `test_suite_durations.request` | count | requests | `rq_compressed` | Number of requests sent to the test suite durations endpoint, regardless of success. |
+| `test_suite_durations.request_errors` | count | requests | `error_type`, `status_code` | Number of terminal test suite durations request errors. `status_code` is emitted only for 400, 401, 403, 404, 408, and 429 responses. |
+| `test_suite_durations.request_ms` | distribution | milliseconds | None | Time to receive a terminal response from the test suite durations endpoint. |
+| `test_suite_durations.response_bytes` | distribution | bytes | `rs_compressed` | Wire size of a response page from the test suite durations endpoint. |
+| `test_suite_durations.response_suites` | distribution | suites | None | Total number of test suites returned across all response pages. |
+| `test_suite_durations.is_empty` | count | responses | None | Number of successful test suite durations fetches that returned zero test suites. |
+
+For `ddtest.cli.command` and `ddtest.cli.command_ms`, `platform`, `framework`,
+and `test_skipping_mode` are reported as `unknown` until platform and framework
+detection succeeds. The detected values are then used for the rest of the
+command. Raw CLI configuration values are never used as telemetry tags.
+
+### Planning tag values
+
+- `reason`: `no_runnable_tests`, `single_runner_only`, `lowest_score`,
+  `target_met_lowest_score`, `target_met_changed_selection`, or
+  `target_unreachable_lowest_wall_time`.
+- `target_status`: `disabled`, `met`, or `missed`.
+- `state`: `discovered`, `runnable`, or `fully_skipped`.
+- `source`: `backend` or `default`.
+- `tia_enabled`: `true` or `false`, representing whether TIA skipping was
+  effective after applying backend settings and framework capabilities.

internal/cmd/cmd.go

@@ -5,10 +5,12 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
+	"time"
 
 	"github.com/DataDog/ddtest/internal/buildinfo"
 	"github.com/DataDog/ddtest/internal/constants"
 	"github.com/DataDog/ddtest/internal/environment"
+	"github.com/DataDog/ddtest/internal/errcode"
 	"github.com/DataDog/ddtest/internal/git"
 	"github.com/DataDog/ddtest/internal/planner"
 	"github.com/DataDog/ddtest/internal/runmetadata"
@@ -23,13 +25,11 @@ import (
 var defaultParallelism = settings.DefaultParallelism()
 
 var rootCmd = &cobra.Command{
-	Use:     "ddtest",
-	Short:   "A test runner from Datadog",
-	Long:    "Command line tool for running tests with Datadog Test Optimization.",
-	Version: buildinfo.CurrentVersion(),
-	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-		return git.CheckAvailable()
-	},
+	Use:               "ddtest",
+	Short:             "A test runner from Datadog",
+	Long:              "Command line tool for running tests with Datadog Test Optimization.",
+	Version:           buildinfo.CurrentVersion(),
+	PersistentPreRunE: runPersistentPreRun,
 }
 
 var (
@@ -128,9 +128,33 @@ func bindPersistentFlags(cmd *cobra.Command, bindings []persistentFlagBinding) e
 	return nil
 }
 
+func runPersistentPreRun(cmd *cobra.Command, _ []string) error {
+	if err := git.CheckAvailable(); err != nil {
+		commandType, errorCode, ok := gitAvailabilityTelemetryContext(cmd)
+		if !ok {
+			return err
+		}
+		return runWithTelemetry(context.Background(), commandType, func(telemetry.Client) error {
+			return errcode.WithCode(errorCode, err)
+		})
+	}
+	return nil
+}
+
+func gitAvailabilityTelemetryContext(cmd *cobra.Command) (telemetry.CLICommandType, errcode.Code, bool) {
+	switch cmd.Name() {
+	case string(telemetry.CLICommandPlan):
+		return telemetry.CLICommandPlan, errcode.PlanGitUnavailable, true
+	case string(telemetry.CLICommandRun):
+		return telemetry.CLICommandRun, errcode.RunGitUnavailable, true
+	default:
+		return "", errcode.Unknown, false
+	}
+}
+
 func runPlanCommand(cmd *cobra.Command, args []string) {
 	ctx := context.Background()
-	err := runWithTelemetry(ctx, func(telemetryClient telemetry.Client) error {
+	err := runWithTelemetry(ctx, telemetry.CLICommandPlan, func(telemetryClient telemetry.Client) error {
 		return planCommand(ctx, telemetryClient)
 	})
 	if err != nil {
@@ -142,7 +166,7 @@ func runPlanCommand(cmd *cobra.Command, args []string) {
 
 func runTestCommand(cmd *cobra.Command, args []string) {
 	ctx := context.Background()
-	err := runWithTelemetry(ctx, func(telemetryClient telemetry.Client) error {
+	err := runWithTelemetry(ctx, telemetry.CLICommandRun, func(telemetryClient telemetry.Client) error {
 		return newRunner(telemetryClient).Run(ctx)
 	})
 	if err != nil {
@@ -161,15 +185,25 @@ func createTelemetryClient() (telemetry.Client, error) {
 	})
 }
 
-func runWithTelemetry(ctx context.Context, operation func(telemetry.Client) error) error {
+func runWithTelemetry(ctx context.Context, commandType telemetry.CLICommandType, operation func(telemetry.Client) error) error {
+	startTime := time.Now()
 	telemetryClient, err := newTelemetryClient()
 	if err != nil {
 		slog.Debug("Failed to create telemetry client", "error", err)
 		telemetryClient = telemetry.NoopClient()
 	}
+	commandTelemetryClient := telemetry.NewCLICommandAttributeTracker(telemetryClient)
 
-	operationErr := operation(telemetryClient)
-	if err := telemetryClient.Flush(context.WithoutCancel(ctx)); err != nil {
+	operationErr := operation(commandTelemetryClient)
+	exitCode := 0
+	if operationErr != nil {
+		exitCode = 1
+	}
+	errorCode := errcode.CodeOf(operationErr)
+	attributes := commandTelemetryClient.Attributes()
+	telemetry.CLICommand(commandTelemetryClient, commandType, exitCode, errorCode, attributes)
+	telemetry.CLICommandMs(commandTelemetryClient, commandType, exitCode, errorCode, attributes, time.Since(startTime))
+	if err := commandTelemetryClient.Flush(context.WithoutCancel(ctx)); err != nil {
 		slog.Debug("Failed to flush telemetry metrics", "error", err)
 	}
 	return operationErr