diff --git a/docs/error-codes.md b/docs/error-codes.md new file mode 100644 index 0000000..5285d5d --- /dev/null +++ b/docs/error-codes.md @@ -0,0 +1,61 @@ +# DDTest error codes + +Fatal `ddtest plan` and `ddtest run` errors include a stable error code in the +form `[error_code] error message`. The same value is reported by the +`error_code` tag on the `ddtest.cli.command` and `ddtest.cli.command_ms` +telemetry metrics. + +Error codes identify the actionable failure point while the rest of the error +message and its wrapped Go error retain the specific OS, platform, framework, +or API cause. Existing codes must not be reused for a different condition. + +## Special telemetry values + +| Code | Meaning | +| --- | --- | +| `none` | The command completed successfully. | +| `unknown` | An error from an external or injected implementation did not contain a DDTest error code. Production plan and run failure paths should not use this value. | + +## Planning errors + +| Code | Condition | +| --- | --- | +| `plan_git_unavailable` | Git was not installed or could not be found before planning. | +| `plan_platform_detection_failed` | The configured platform could not be selected or did not pass its sanity check. | +| `plan_platform_tags_creation_failed` | Runtime or operating-system tags could not be collected from the selected platform. | +| `plan_runtime_tags_invalid` | The `runtime-tags` override could not be parsed. | +| `plan_framework_detection_failed` | The configured test framework is not supported or could not be initialized for the selected platform. | +| `plan_optimization_client_creation_failed` | The Test Optimization client could not be created. | +| `plan_test_files_resolution_failed` | The test include/exclude patterns could not be resolved or the test-file scan failed. | +| `plan_optimization_client_initialization_failed` | The Test Optimization client failed during initialization. | +| `plan_full_test_discovery_failed` | Required full test discovery failed while strict discovery was enabled. | +| `plan_fast_test_discovery_failed` | Fast test-file discovery failed and no full-discovery result was available. | +| `plan_full_discovery_results_processing_failed` | Full-discovery results could not be matched to the configured test selection. | +| `plan_fast_discovery_results_processing_failed` | Fast-discovery results could not be matched to the configured test selection. | +| `plan_manifest_write_failed` | The Test Optimization manifest could not be written. | +| `plan_cache_write_failed` | The Test Optimization plan cache could not be stored. | +| `plan_test_files_write_failed` | The selected test-files artifact could not be written. | +| `plan_skippable_percentage_write_failed` | The skippable-percentage artifact could not be written. | +| `plan_parallel_runners_write_failed` | The parallel-runner-count artifact could not be written. | +| `plan_test_splits_write_failed` | Test split artifacts could not be created or written. | + +## Run errors + +| Code | Condition | +| --- | --- | +| `run_git_unavailable` | Git was not installed or could not be found before running tests. | +| `run_planning_failed` | The automatic planning phase returned an unclassified error. A classified planning failure retains its more precise `plan_*` code. | +| `run_plan_status_check_failed` | DDTest could not check whether planning artifacts exist. | +| `run_plan_load_failed` | The Test Optimization plan cache could not be loaded. | +| `run_parallel_runners_read_failed` | The parallel-runner-count artifact could not be read. | +| `run_parallel_runners_parse_failed` | The parallel-runner-count artifact did not contain a valid integer. | +| `run_platform_detection_failed` | The configured platform could not be selected or did not pass its sanity check before running tests. | +| `run_framework_detection_failed` | The configured test framework is not supported or could not be initialized before running tests. | +| `run_sequential_test_files_read_failed` | The sequential test-files artifact could not be read. | +| `run_sequential_tests_failed` | The test framework failed while running the sequential test batch. | +| `run_parallel_splits_read_failed` | The test-splits directory could not be read. | +| `run_parallel_test_files_read_failed` | A test split file could not be read. | +| `run_parallel_tests_failed` | The test framework failed in a local parallel worker. | +| `run_ci_node_test_files_missing` | The requested CI-node split file does not exist. | +| `run_ci_node_test_files_read_failed` | The requested CI-node split file could not be read. | +| `run_ci_node_tests_failed` | The test framework failed in a CI-node worker. | diff --git a/docs/telemetry-metrics.md b/docs/telemetry-metrics.md new file mode 100644 index 0000000..47fb499 --- /dev/null +++ b/docs/telemetry-metrics.md @@ -0,0 +1,51 @@ +# DDTest telemetry metrics + +This document tracks DDTest-specific metrics that must be added to the +[`civisibility` namespace in `common_metrics.json`](https://github.com/DataDog/dd-go/blob/prod/trace/apps/tracer-telemetry-intake/telemetry-metrics/static/common_metrics.json). +The names below are the metric names emitted in telemetry payloads; the intake +adds the `dd.instrumentation_telemetry_data.civisibility.` prefix. All metrics +are common metrics and are not sent to customer organizations. + +## Pending allowlist additions + +| Metric | Type | Data type | Allowed tags | Description | +| --- | --- | --- | --- | --- | +| `ddtest.cli.command` | count | command | `command`, `exit_code`, `error_code`, `platform`, `framework`, `test_skipping_mode` | Number of completed top-level ddtest commands. `command` is `plan` or `run`; `exit_code` is `0` or `1`; `error_code` is a value from the [DDTest error code catalog](error-codes.md); the remaining tags contain the resolved CLI configuration. | +| `ddtest.cli.command_ms` | distribution | milliseconds | `command`, `exit_code`, `error_code`, `platform`, `framework`, `test_skipping_mode` | Duration of a top-level ddtest command, tagged by command, exit code, error code, and resolved CLI configuration. | +| `ddtest.itr_skippable_tests.is_empty` | count | responses | None | Number of successful skippable-tests fetches that returned zero skippable tests or suites. | +| `ddtest.planning.decision` | count | plans | `platform`, `framework`, `test_skipping_mode`, `discovery_mode`, `tia_enabled`, `reason`, `target_status` | Number of completed plans. `reason` explains the constraint that selected the parallel runner split; `target_status` is `disabled`, `met`, or `missed`. | +| `ddtest.planning.test_files` | distribution | test files | `platform`, `framework`, `test_skipping_mode`, `discovery_mode`, `tia_enabled`, `state` | Number of test files at each planning stage. `state` is `discovered`, `runnable`, or `fully_skipped`. | +| `ddtest.planning.estimated_time_saved_pct` | distribution | percentage | `platform`, `framework`, `test_skipping_mode`, `discovery_mode`, `tia_enabled` | Estimated percentage of test runtime saved by skipping decisions. | +| `ddtest.planning.test_file_durations` | distribution | test files | `platform`, `framework`, `test_skipping_mode`, `discovery_mode`, `tia_enabled`, `source` | Number of runnable test files weighted using `backend` durations or `default` estimates. | +| `ddtest.planning.parallel_runners` | distribution | runners | `platform`, `framework`, `test_skipping_mode`, `discovery_mode`, `tia_enabled` | Number of parallel runners selected by the planner. | +| `ddtest.planning.expected_full_runtime_ms` | distribution | milliseconds | `platform`, `framework`, `test_skipping_mode`, `discovery_mode`, `tia_enabled` | Estimated serial runtime of all discovered test files before skipping. | +| `ddtest.planning.expected_runnable_runtime_ms` | distribution | milliseconds | `platform`, `framework`, `test_skipping_mode`, `discovery_mode`, `tia_enabled` | Estimated serial runtime after skipping decisions. | +| `ddtest.planning.expected_wall_time_ms` | distribution | milliseconds | `platform`, `framework`, `test_skipping_mode`, `discovery_mode`, `tia_enabled` | Estimated wall time for the selected parallel runner split. | +| `ddtest.planning.split_imbalance_pct` | distribution | percentage | `platform`, `framework`, `test_skipping_mode`, `discovery_mode`, `tia_enabled` | Difference between the most- and least-loaded runners as a percentage of expected wall time. | +| `ddtest.planning.disabled_tests` | distribution | tests | `platform`, `framework`, `test_skipping_mode`, `discovery_mode`, `tia_enabled` | Number of Test Management-disabled tests applied during planning. | +| `ddtest.planning.forced_run_suites` | distribution | suites | `platform`, `framework`, `test_skipping_mode`, `discovery_mode`, `tia_enabled` | Number of otherwise-skippable suites kept runnable by an unskippable marker. | +| `ddtest.test_discovery.duration_ms` | distribution | milliseconds | `discovery_mode`, `success`, `platform`, `framework` | Duration of the discovery strategy selected by the planner. `discovery_mode` is `full` for test discovery or `fast` for test-file discovery; `success` reports whether that selected strategy completed successfully. | +| `ddtest.test_discovery.tests` | distribution | tests | `discovery_mode`, `success`, `platform`, `framework` | Number of tests returned by selected full test discovery. Emitted only with `discovery_mode:full`. | +| `ddtest.test_discovery.test_files` | distribution | test files | `discovery_mode`, `success`, `platform`, `framework` | Number of test files returned by selected fast test-file discovery. Emitted only with `discovery_mode:fast`. | +| `test_suite_durations.request` | count | requests | `rq_compressed` | Number of requests sent to the test suite durations endpoint, regardless of success. | +| `test_suite_durations.request_errors` | count | requests | `error_type`, `status_code` | Number of terminal test suite durations request errors. `status_code` is emitted only for 400, 401, 403, 404, 408, and 429 responses. | +| `test_suite_durations.request_ms` | distribution | milliseconds | None | Time to receive a terminal response from the test suite durations endpoint. | +| `test_suite_durations.response_bytes` | distribution | bytes | `rs_compressed` | Wire size of a response page from the test suite durations endpoint. | +| `test_suite_durations.response_suites` | distribution | suites | None | Total number of test suites returned across all response pages. | +| `test_suite_durations.is_empty` | count | responses | None | Number of successful test suite durations fetches that returned zero test suites. | + +For `ddtest.cli.command` and `ddtest.cli.command_ms`, `platform`, `framework`, +and `test_skipping_mode` are reported as `unknown` until platform and framework +detection succeeds. The detected values are then used for the rest of the +command. Raw CLI configuration values are never used as telemetry tags. + +### Planning tag values + +- `reason`: `no_runnable_tests`, `single_runner_only`, `lowest_score`, + `target_met_lowest_score`, `target_met_changed_selection`, or + `target_unreachable_lowest_wall_time`. +- `target_status`: `disabled`, `met`, or `missed`. +- `state`: `discovered`, `runnable`, or `fully_skipped`. +- `source`: `backend` or `default`. +- `tia_enabled`: `true` or `false`, representing whether TIA skipping was + effective after applying backend settings and framework capabilities. diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index 017f705..4e29169 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -5,10 +5,12 @@ import ( "fmt" "log/slog" "os" + "time" "github.com/DataDog/ddtest/internal/buildinfo" "github.com/DataDog/ddtest/internal/constants" "github.com/DataDog/ddtest/internal/environment" + "github.com/DataDog/ddtest/internal/errcode" "github.com/DataDog/ddtest/internal/git" "github.com/DataDog/ddtest/internal/planner" "github.com/DataDog/ddtest/internal/runmetadata" @@ -23,13 +25,11 @@ import ( var defaultParallelism = settings.DefaultParallelism() var rootCmd = &cobra.Command{ - Use: "ddtest", - Short: "A test runner from Datadog", - Long: "Command line tool for running tests with Datadog Test Optimization.", - Version: buildinfo.CurrentVersion(), - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - return git.CheckAvailable() - }, + Use: "ddtest", + Short: "A test runner from Datadog", + Long: "Command line tool for running tests with Datadog Test Optimization.", + Version: buildinfo.CurrentVersion(), + PersistentPreRunE: runPersistentPreRun, } var ( @@ -128,9 +128,33 @@ func bindPersistentFlags(cmd *cobra.Command, bindings []persistentFlagBinding) e return nil } +func runPersistentPreRun(cmd *cobra.Command, _ []string) error { + if err := git.CheckAvailable(); err != nil { + commandType, errorCode, ok := gitAvailabilityTelemetryContext(cmd) + if !ok { + return err + } + return runWithTelemetry(context.Background(), commandType, func(telemetry.Client) error { + return errcode.WithCode(errorCode, err) + }) + } + return nil +} + +func gitAvailabilityTelemetryContext(cmd *cobra.Command) (telemetry.CLICommandType, errcode.Code, bool) { + switch cmd.Name() { + case string(telemetry.CLICommandPlan): + return telemetry.CLICommandPlan, errcode.PlanGitUnavailable, true + case string(telemetry.CLICommandRun): + return telemetry.CLICommandRun, errcode.RunGitUnavailable, true + default: + return "", errcode.Unknown, false + } +} + func runPlanCommand(cmd *cobra.Command, args []string) { ctx := context.Background() - err := runWithTelemetry(ctx, func(telemetryClient telemetry.Client) error { + err := runWithTelemetry(ctx, telemetry.CLICommandPlan, func(telemetryClient telemetry.Client) error { return planCommand(ctx, telemetryClient) }) if err != nil { @@ -142,7 +166,7 @@ func runPlanCommand(cmd *cobra.Command, args []string) { func runTestCommand(cmd *cobra.Command, args []string) { ctx := context.Background() - err := runWithTelemetry(ctx, func(telemetryClient telemetry.Client) error { + err := runWithTelemetry(ctx, telemetry.CLICommandRun, func(telemetryClient telemetry.Client) error { return newRunner(telemetryClient).Run(ctx) }) if err != nil { @@ -161,15 +185,25 @@ func createTelemetryClient() (telemetry.Client, error) { }) } -func runWithTelemetry(ctx context.Context, operation func(telemetry.Client) error) error { +func runWithTelemetry(ctx context.Context, commandType telemetry.CLICommandType, operation func(telemetry.Client) error) error { + startTime := time.Now() telemetryClient, err := newTelemetryClient() if err != nil { slog.Debug("Failed to create telemetry client", "error", err) telemetryClient = telemetry.NoopClient() } + commandTelemetryClient := telemetry.NewCLICommandAttributeTracker(telemetryClient) - operationErr := operation(telemetryClient) - if err := telemetryClient.Flush(context.WithoutCancel(ctx)); err != nil { + operationErr := operation(commandTelemetryClient) + exitCode := 0 + if operationErr != nil { + exitCode = 1 + } + errorCode := errcode.CodeOf(operationErr) + attributes := commandTelemetryClient.Attributes() + telemetry.CLICommand(commandTelemetryClient, commandType, exitCode, errorCode, attributes) + telemetry.CLICommandMs(commandTelemetryClient, commandType, exitCode, errorCode, attributes, time.Since(startTime)) + if err := commandTelemetryClient.Flush(context.WithoutCancel(ctx)); err != nil { slog.Debug("Failed to flush telemetry metrics", "error", err) } return operationErr diff --git a/internal/cmd/cmd_test.go b/internal/cmd/cmd_test.go index f5d5723..52894d0 100644 --- a/internal/cmd/cmd_test.go +++ b/internal/cmd/cmd_test.go @@ -5,9 +5,11 @@ import ( "context" "errors" "os" + "slices" "strings" "testing" + "github.com/DataDog/ddtest/internal/errcode" "github.com/DataDog/ddtest/internal/git" runnerpkg "github.com/DataDog/ddtest/internal/runner" "github.com/DataDog/ddtest/internal/settings" @@ -176,22 +178,50 @@ func TestCommandHierarchy(t *testing.T) { } } -func TestRootPersistentPreRunChecksGitAvailability(t *testing.T) { +func TestRootPersistentPreRunReportsGitAvailabilityFailures(t *testing.T) { originalLookPathFunc := git.LookPathFunc + originalNewTelemetryClient := newTelemetryClient git.LookPathFunc = func(file string) (string, error) { return "", errors.New("missing git") } t.Cleanup(func() { git.LookPathFunc = originalLookPathFunc + newTelemetryClient = originalNewTelemetryClient }) - err := rootCmd.PersistentPreRunE(rootCmd, nil) - if err == nil || !strings.Contains(err.Error(), "git executable not found") { - t.Fatalf("PersistentPreRunE() error = %v, want git availability error", err) + tests := []struct { + name string + command *cobra.Command + commandType string + errorCode errcode.Code + }{ + {name: "plan", command: planCmd, commandType: "plan", errorCode: errcode.PlanGitUnavailable}, + {name: "run", command: runCmd, commandType: "run", errorCode: errcode.RunGitUnavailable}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + telemetryClient := &fakeTelemetryClient{} + newTelemetryClient = func() (telemetry.Client, error) { return telemetryClient, nil } + + err := rootCmd.PersistentPreRunE(test.command, nil) + if err == nil || !strings.Contains(err.Error(), "git executable not found") { + t.Fatalf("PersistentPreRunE() error = %v, want git availability error", err) + } + if got := errcode.CodeOf(err); got != test.errorCode { + t.Fatalf("PersistentPreRunE() error code = %q, want %q", got, test.errorCode) + } + if telemetryClient.flushCalls != 1 { + t.Fatalf("telemetry flush calls = %d, want 1", telemetryClient.flushCalls) + } + tags := cliMetricTags(test.commandType, "1", test.errorCode, unknownCLICommandAttributes()) + telemetryClient.assertValue(t, "count", "ddtest.cli.command", tags, 1) + telemetryClient.assertSamples(t, "distribution", "ddtest.cli.command_ms", tags, 1) + }) } } func TestRunPlanCommand(t *testing.T) { + attributes := detectedCLICommandAttributes() originalPlanCommand := planCommand originalNewTelemetryClient := newTelemetryClient originalExitProcess := exitProcess @@ -206,9 +236,7 @@ func TestRunPlanCommand(t *testing.T) { calls := 0 planCommand = func(ctx context.Context, got telemetry.Client) error { calls++ - if got != telemetryClient { - t.Fatal("plan command did not receive the command telemetry client") - } + telemetry.RecordCLICommandAttributes(got, attributes) return nil } exitProcess = func(code int) { @@ -223,6 +251,12 @@ func TestRunPlanCommand(t *testing.T) { if telemetryClient.flushCalls != 1 { t.Fatalf("telemetry flush calls = %d, want 1", telemetryClient.flushCalls) } + if telemetryClient.metricsAtFlush < 2 { + t.Fatalf("telemetry metrics at flush = %d, want at least 2", telemetryClient.metricsAtFlush) + } + tags := cliMetricTags("plan", "0", errcode.None, attributes) + telemetryClient.assertValue(t, "count", "ddtest.cli.command", tags, 1) + telemetryClient.assertSamples(t, "distribution", "ddtest.cli.command_ms", tags, 1) } func TestRunPlanCommandExitsOnError(t *testing.T) { @@ -237,7 +271,7 @@ func TestRunPlanCommandExitsOnError(t *testing.T) { telemetryClient := &fakeTelemetryClient{} newTelemetryClient = func() (telemetry.Client, error) { return telemetryClient, nil } - planErr := errors.New("planner failed") + planErr := errcode.New(errcode.PlanPlatformDetectionFailed, "planner failed") planCommand = func(ctx context.Context, got telemetry.Client) error { return planErr } @@ -254,9 +288,16 @@ func TestRunPlanCommandExitsOnError(t *testing.T) { if telemetryClient.flushCalls != 1 { t.Fatalf("telemetry flush calls = %d, want 1", telemetryClient.flushCalls) } + if telemetryClient.metricsAtFlush < 2 { + t.Fatalf("telemetry metrics at flush = %d, want at least 2", telemetryClient.metricsAtFlush) + } + tags := cliMetricTags("plan", "1", errcode.PlanPlatformDetectionFailed, unknownCLICommandAttributes()) + telemetryClient.assertValue(t, "count", "ddtest.cli.command", tags, 1) + telemetryClient.assertSamples(t, "distribution", "ddtest.cli.command_ms", tags, 1) } func TestRunTestCommand(t *testing.T) { + attributes := detectedCLICommandAttributes() originalNewRunner := newRunner originalNewTelemetryClient := newTelemetryClient originalExitProcess := exitProcess @@ -270,9 +311,7 @@ func TestRunTestCommand(t *testing.T) { newTelemetryClient = func() (telemetry.Client, error) { return telemetryClient, nil } fake := &fakeCommandRunner{} newRunner = func(got telemetry.Client) runnerpkg.Runner { - if got != telemetryClient { - t.Fatal("runner did not receive the command telemetry client") - } + telemetry.RecordCLICommandAttributes(got, attributes) return fake } exitProcess = func(code int) { @@ -287,9 +326,16 @@ func TestRunTestCommand(t *testing.T) { if telemetryClient.flushCalls != 1 { t.Fatalf("telemetry flush calls = %d, want 1", telemetryClient.flushCalls) } + if telemetryClient.metricsAtFlush < 2 { + t.Fatalf("telemetry metrics at flush = %d, want at least 2", telemetryClient.metricsAtFlush) + } + tags := cliMetricTags("run", "0", errcode.None, attributes) + telemetryClient.assertValue(t, "count", "ddtest.cli.command", tags, 1) + telemetryClient.assertSamples(t, "distribution", "ddtest.cli.command_ms", tags, 1) } func TestRunTestCommandExitsOnError(t *testing.T) { + attributes := detectedCLICommandAttributes() originalNewRunner := newRunner originalNewTelemetryClient := newTelemetryClient originalExitProcess := exitProcess @@ -301,8 +347,9 @@ func TestRunTestCommandExitsOnError(t *testing.T) { telemetryClient := &fakeTelemetryClient{} newTelemetryClient = func() (telemetry.Client, error) { return telemetryClient, nil } - fake := &fakeCommandRunner{err: errors.New("runner failed")} + fake := &fakeCommandRunner{err: errcode.New(errcode.RunParallelTestsFailed, "runner failed")} newRunner = func(got telemetry.Client) runnerpkg.Runner { + telemetry.RecordCLICommandAttributes(got, attributes) return fake } var exitCodes []int @@ -318,6 +365,12 @@ func TestRunTestCommandExitsOnError(t *testing.T) { if telemetryClient.flushCalls != 1 { t.Fatalf("telemetry flush calls = %d, want 1", telemetryClient.flushCalls) } + if telemetryClient.metricsAtFlush < 2 { + t.Fatalf("telemetry metrics at flush = %d, want at least 2", telemetryClient.metricsAtFlush) + } + tags := cliMetricTags("run", "1", errcode.RunParallelTestsFailed, attributes) + telemetryClient.assertValue(t, "count", "ddtest.cli.command", tags, 1) + telemetryClient.assertSamples(t, "distribution", "ddtest.cli.command_ms", tags, 1) } func TestRunWithTelemetryFallsBackWhenCreationFails(t *testing.T) { @@ -328,7 +381,7 @@ func TestRunWithTelemetryFallsBackWhenCreationFails(t *testing.T) { } operationErr := errors.New("operation failed") - got := runWithTelemetry(context.Background(), func(client telemetry.Client) error { + got := runWithTelemetry(context.Background(), telemetry.CLICommandPlan, func(client telemetry.Client) error { if client == nil { t.Fatal("operation received nil telemetry client") } @@ -347,7 +400,7 @@ func TestRunWithTelemetryDoesNotReplaceCommandErrorWithFlushError(t *testing.T) newTelemetryClient = func() (telemetry.Client, error) { return telemetryClient, nil } operationErr := errors.New("operation failed") - got := runWithTelemetry(context.Background(), func(telemetry.Client) error { + got := runWithTelemetry(context.Background(), telemetry.CLICommandRun, func(telemetry.Client) error { return operationErr }) if !errors.Is(got, operationErr) { @@ -629,29 +682,103 @@ type fakeCommandRunner struct { err error } +func detectedCLICommandAttributes() telemetry.CLICommandAttributes { + return telemetry.CLICommandAttributes{ + Platform: "javascript", + Framework: "jest", + TestSkippingMode: "suite", + } +} + +func unknownCLICommandAttributes() telemetry.CLICommandAttributes { + return telemetry.CLICommandAttributes{ + Platform: "unknown", + Framework: "unknown", + TestSkippingMode: "unknown", + } +} + +func cliMetricTags(command, exitCode string, errorCode errcode.Code, attributes telemetry.CLICommandAttributes) []string { + return []string{ + "command:" + command, + "exit_code:" + exitCode, + "error_code:" + string(errorCode), + "platform:" + attributes.Platform, + "framework:" + attributes.Framework, + "test_skipping_mode:" + attributes.TestSkippingMode, + } +} + func (f *fakeCommandRunner) Run(ctx context.Context) error { f.calls++ return f.err } type fakeTelemetryClient struct { - flushCalls int - flushErr error + flushCalls int + metricsAtFlush int + flushErr error + metrics []fakeRecordedMetric } -func (f *fakeTelemetryClient) Count(string, []string) telemetry.Metric { - return fakeTelemetryMetric{} +type fakeRecordedMetric struct { + kind string + name string + tags []string + value float64 } -func (f *fakeTelemetryClient) Distribution(string, []string) telemetry.Metric { - return fakeTelemetryMetric{} +func (f *fakeTelemetryClient) Count(name string, tags []string) telemetry.Metric { + return &fakeTelemetryMetric{client: f, kind: "count", name: name, tags: slices.Clone(tags)} +} + +func (f *fakeTelemetryClient) Distribution(name string, tags []string) telemetry.Metric { + return &fakeTelemetryMetric{client: f, kind: "distribution", name: name, tags: slices.Clone(tags)} } func (f *fakeTelemetryClient) Flush(context.Context) error { f.flushCalls++ + f.metricsAtFlush = len(f.metrics) return f.flushErr } -type fakeTelemetryMetric struct{} +func (f *fakeTelemetryClient) values(kind, name string, tags []string) []float64 { + var values []float64 + for _, metric := range f.metrics { + if metric.kind == kind && metric.name == name && slices.Equal(metric.tags, tags) { + values = append(values, metric.value) + } + } + return values +} -func (fakeTelemetryMetric) Submit(float64) {} +func (f *fakeTelemetryClient) assertValue(t *testing.T, kind, name string, tags []string, want float64) { + t.Helper() + values := f.values(kind, name, tags) + if len(values) != 1 || values[0] != want { + t.Errorf("%s %s %v values = %v, want [%v]", kind, name, tags, values, want) + } +} + +func (f *fakeTelemetryClient) assertSamples(t *testing.T, kind, name string, tags []string, want int) { + t.Helper() + if values := f.values(kind, name, tags); len(values) != want { + t.Errorf("%s %s %v sample count = %d, want %d; values=%v", kind, name, tags, len(values), want, values) + } +} + +type fakeTelemetryMetric struct { + client *fakeTelemetryClient + kind string + name string + tags []string +} + +func (m *fakeTelemetryMetric) Submit(value float64) { + m.client.metrics = append(m.client.metrics, fakeRecordedMetric{ + kind: m.kind, + name: m.name, + tags: m.tags, + value: value, + }) +} diff --git a/internal/errcode/error.go b/internal/errcode/error.go new file mode 100644 index 0000000..9b703e1 --- /dev/null +++ b/internal/errcode/error.go @@ -0,0 +1,103 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +// Package errcode provides stable, machine-readable codes for fatal ddtest +// command errors. +package errcode + +import ( + "errors" + "fmt" +) + +// Code identifies an actionable fatal error condition. +type Code string + +const ( + None Code = "none" + Unknown Code = "unknown" + + PlanGitUnavailable Code = "plan_git_unavailable" + PlanPlatformDetectionFailed Code = "plan_platform_detection_failed" + PlanPlatformTagsCreationFailed Code = "plan_platform_tags_creation_failed" + PlanRuntimeTagsInvalid Code = "plan_runtime_tags_invalid" + PlanFrameworkDetectionFailed Code = "plan_framework_detection_failed" + PlanOptimizationClientCreationFailed Code = "plan_optimization_client_creation_failed" + PlanTestFilesResolutionFailed Code = "plan_test_files_resolution_failed" + PlanOptimizationClientInitializationFailed Code = "plan_optimization_client_initialization_failed" + PlanFullTestDiscoveryFailed Code = "plan_full_test_discovery_failed" + PlanFastTestDiscoveryFailed Code = "plan_fast_test_discovery_failed" + PlanFullDiscoveryResultsProcessingFailed Code = "plan_full_discovery_results_processing_failed" + PlanFastDiscoveryResultsProcessingFailed Code = "plan_fast_discovery_results_processing_failed" + PlanManifestWriteFailed Code = "plan_manifest_write_failed" + PlanCacheWriteFailed Code = "plan_cache_write_failed" + PlanTestFilesWriteFailed Code = "plan_test_files_write_failed" + PlanSkippablePercentageWriteFailed Code = "plan_skippable_percentage_write_failed" + PlanParallelRunnersWriteFailed Code = "plan_parallel_runners_write_failed" + PlanTestSplitsWriteFailed Code = "plan_test_splits_write_failed" + RunGitUnavailable Code = "run_git_unavailable" + RunPlanningFailed Code = "run_planning_failed" + RunPlanStatusCheckFailed Code = "run_plan_status_check_failed" + RunPlanLoadFailed Code = "run_plan_load_failed" + RunParallelRunnersReadFailed Code = "run_parallel_runners_read_failed" + RunParallelRunnersParseFailed Code = "run_parallel_runners_parse_failed" + RunPlatformDetectionFailed Code = "run_platform_detection_failed" + RunFrameworkDetectionFailed Code = "run_framework_detection_failed" + RunSequentialTestFilesReadFailed Code = "run_sequential_test_files_read_failed" + RunSequentialTestsFailed Code = "run_sequential_tests_failed" + RunParallelSplitsReadFailed Code = "run_parallel_splits_read_failed" + RunParallelTestFilesReadFailed Code = "run_parallel_test_files_read_failed" + RunParallelTestsFailed Code = "run_parallel_tests_failed" + RunCINodeTestFilesMissing Code = "run_ci_node_test_files_missing" + RunCINodeTestFilesReadFailed Code = "run_ci_node_test_files_read_failed" + RunCINodeTestsFailed Code = "run_ci_node_tests_failed" +) + +// Error associates a stable code with an underlying error while preserving +// standard Go error-chain behavior. +type Error struct { + Code Code + cause error +} + +func (e *Error) Error() string { + return fmt.Sprintf("[%s] %s", e.Code, e.cause) +} + +func (e *Error) Unwrap() error { + return e.cause +} + +// WithCode associates code with err. An existing ddtest error code deeper in +// the chain wins, so callers can add context without hiding a more precise +// code from the original failure point. +func WithCode(code Code, err error) error { + if err == nil { + return nil + } + if CodeOf(err) != Unknown { + return err + } + return &Error{Code: code, cause: err} +} + +// New creates a coded error without a separate underlying cause. +func New(code Code, message string) error { + return &Error{Code: code, cause: errors.New(message)} +} + +// CodeOf returns the first ddtest error code in err's chain. Successful +// operations use None and unclassified external errors use Unknown. +func CodeOf(err error) Code { + if err == nil { + return None + } + + var codedErr *Error + if errors.As(err, &codedErr) && codedErr != nil && codedErr.Code != "" { + return codedErr.Code + } + return Unknown +} diff --git a/internal/errcode/error_test.go b/internal/errcode/error_test.go new file mode 100644 index 0000000..f462345 --- /dev/null +++ b/internal/errcode/error_test.go @@ -0,0 +1,100 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package errcode + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestErrorPreservesCodeAndCause(t *testing.T) { + cause := errors.New("disk full") + err := WithCode(PlanCacheWriteFailed, cause) + + if got := CodeOf(err); got != PlanCacheWriteFailed { + t.Fatalf("CodeOf() = %q, want %q", got, PlanCacheWriteFailed) + } + if !errors.Is(err, cause) { + t.Fatal("coded error does not preserve its cause") + } + if !strings.Contains(err.Error(), "[plan_cache_write_failed] disk full") { + t.Fatalf("Error() = %q, want code and cause", err) + } +} + +func TestWithCodePreservesMoreSpecificNestedCode(t *testing.T) { + cause := errors.New("discovery process failed") + planErr := WithCode(PlanFullTestDiscoveryFailed, cause) + runErr := WithCode(RunPlanningFailed, fmt.Errorf("failed to run planning phase: %w", planErr)) + + if got := CodeOf(runErr); got != PlanFullTestDiscoveryFailed { + t.Fatalf("CodeOf() = %q, want nested code %q", got, PlanFullTestDiscoveryFailed) + } + if !errors.Is(runErr, cause) { + t.Fatal("nested coded error does not preserve its cause") + } +} + +func TestCodeOfSpecialValues(t *testing.T) { + if got := CodeOf(nil); got != None { + t.Fatalf("CodeOf(nil) = %q, want %q", got, None) + } + if got := CodeOf(errors.New("external failure")); got != Unknown { + t.Fatalf("CodeOf(unclassified) = %q, want %q", got, Unknown) + } +} + +func TestFatalCodesAreUnique(t *testing.T) { + codes := []Code{ + PlanGitUnavailable, + PlanPlatformDetectionFailed, + PlanPlatformTagsCreationFailed, + PlanRuntimeTagsInvalid, + PlanFrameworkDetectionFailed, + PlanOptimizationClientCreationFailed, + PlanTestFilesResolutionFailed, + PlanOptimizationClientInitializationFailed, + PlanFullTestDiscoveryFailed, + PlanFastTestDiscoveryFailed, + PlanFullDiscoveryResultsProcessingFailed, + PlanFastDiscoveryResultsProcessingFailed, + PlanManifestWriteFailed, + PlanCacheWriteFailed, + PlanTestFilesWriteFailed, + PlanSkippablePercentageWriteFailed, + PlanParallelRunnersWriteFailed, + PlanTestSplitsWriteFailed, + RunGitUnavailable, + RunPlanningFailed, + RunPlanStatusCheckFailed, + RunPlanLoadFailed, + RunParallelRunnersReadFailed, + RunParallelRunnersParseFailed, + RunPlatformDetectionFailed, + RunFrameworkDetectionFailed, + RunSequentialTestFilesReadFailed, + RunSequentialTestsFailed, + RunParallelSplitsReadFailed, + RunParallelTestFilesReadFailed, + RunParallelTestsFailed, + RunCINodeTestFilesMissing, + RunCINodeTestFilesReadFailed, + RunCINodeTestsFailed, + } + + seen := make(map[Code]struct{}, len(codes)) + for _, code := range codes { + if code == None || code == Unknown || code == "" { + t.Fatalf("fatal error code uses reserved value %q", code) + } + if _, exists := seen[code]; exists { + t.Fatalf("duplicate fatal error code %q", code) + } + seen[code] = struct{}{} + } +} diff --git a/internal/planner/planner.go b/internal/planner/planner.go index 2323cb4..3d12489 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -15,6 +15,7 @@ import ( "github.com/DataDog/ddtest/internal/constants" "github.com/DataDog/ddtest/internal/discovery" "github.com/DataDog/ddtest/internal/environment" + "github.com/DataDog/ddtest/internal/errcode" "github.com/DataDog/ddtest/internal/framework" "github.com/DataDog/ddtest/internal/platform" "github.com/DataDog/ddtest/internal/runmetadata" @@ -88,6 +89,7 @@ type TestPlanner struct { ciProviderDetector environment.CIProviderDetector telemetryClient telemetry.Client reportWriter io.Writer + tiaSkippingEnabled bool } const ( @@ -207,20 +209,20 @@ func (tp *TestPlanner) Plan(ctx context.Context) error { } if err := writePlanFile(constants.ManifestPath, []byte(constants.ManifestVersion+"\n")); err != nil { - return fmt.Errorf("failed to write test optimization manifest: %w", err) + return errcode.WithCode(errcode.PlanManifestWriteFailed, fmt.Errorf("failed to write test optimization manifest: %w", err)) } if err := tp.storeTestOptimizationPlanCache(); err != nil { - return fmt.Errorf("failed to store test optimization plan cache: %w", err) + return errcode.WithCode(errcode.PlanCacheWriteFailed, fmt.Errorf("failed to store test optimization plan cache: %w", err)) } if err := writeTestFilesArtifact(tp.testFileWeights); err != nil { - return err + return errcode.WithCode(errcode.PlanTestFilesWriteFailed, err) } percentageContent := fmt.Sprintf("%.2f", tp.skippablePercentage) if err := writePlanFile(constants.SkippablePercentageOutputPath, []byte(percentageContent)); err != nil { - return fmt.Errorf("failed to write skippable percentage: %w", err) + return errcode.WithCode(errcode.PlanSkippablePercentageWriteFailed, fmt.Errorf("failed to write skippable percentage: %w", err)) } parallelRunnerSelection := calculateParallelRunnerSplitSelection( @@ -234,7 +236,7 @@ func (tp *TestPlanner) Plan(ctx context.Context) error { parallelRunners := parallelRunnerSplit.parallelRunners runnersContent := fmt.Sprintf("%d", parallelRunners) if err := writePlanFile(constants.ParallelRunnersOutputPath, []byte(runnersContent)); err != nil { - return fmt.Errorf("failed to write parallel runners: %w", err) + return errcode.WithCode(errcode.PlanParallelRunnersWriteFailed, fmt.Errorf("failed to write parallel runners: %w", err)) } if ciProvider, err := tp.ciProviderDetector.DetectCIProvider(); err == nil { @@ -249,13 +251,14 @@ func (tp *TestPlanner) Plan(ctx context.Context) error { } if err := tp.CreateTestSplits(tp.testFileWeights, parallelRunners, constants.TestFilesOutputPath); err != nil { - return fmt.Errorf("failed to create test splits: %w", err) + return errcode.WithCode(errcode.PlanTestSplitsWriteFailed, fmt.Errorf("failed to create test splits: %w", err)) } if settings.GetReportEnabled() { printPlanReport(tp.reportWriter, tp, parallelRunnerSelection) } + tp.recordPlanningTelemetry(parallelRunnerSelection) tp.planLoaded = true return nil } @@ -263,19 +266,19 @@ func (tp *TestPlanner) Plan(ctx context.Context) error { func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { detectedPlatform, err := tp.platformDetector.DetectPlatform() if err != nil { - return fmt.Errorf("failed to detect platform: %w", err) + return errcode.WithCode(errcode.PlanPlatformDetectionFailed, fmt.Errorf("failed to detect platform: %w", err)) } // Get platform-detected tags first tags, err := detectedPlatform.CreateTagsMap() if err != nil { - return fmt.Errorf("failed to create platform tags: %w", err) + return errcode.WithCode(errcode.PlanPlatformTagsCreationFailed, fmt.Errorf("failed to create platform tags: %w", err)) } // Check if runtime tags override is provided and merge onto detected tags overrideTags, err := settings.GetRuntimeTagsMap() if err != nil { - return fmt.Errorf("failed to parse runtime tags override: %w", err) + return errcode.WithCode(errcode.PlanRuntimeTagsInvalid, fmt.Errorf("failed to parse runtime tags override: %w", err)) } if overrideTags != nil { @@ -289,10 +292,15 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { // Detect framework once to avoid duplicate work testFramework, err := detectedPlatform.DetectFramework() if err != nil { - return fmt.Errorf("failed to detect framework: %w", err) + return errcode.WithCode(errcode.PlanFrameworkDetectionFailed, fmt.Errorf("failed to detect framework: %w", err)) } slog.Info("Framework detected", "framework", testFramework.Name()) testSkippingLevel := detectedPlatform.TestSkippingLevel() + telemetry.RecordCLICommandAttributes(tp.telemetryClient, telemetry.CLICommandAttributes{ + Platform: detectedPlatform.Name(), + Framework: testFramework.Name(), + TestSkippingMode: testSkippingLevel.String(), + }) isSuiteLevelSkipping := testSkippingLevel == settings.TestSkippingLevelSuite isTestLevelSkipping := testSkippingLevel == settings.TestSkippingLevelTest fullTestDiscoverySupported := testFramework.SupportsFullTestDiscovery() @@ -303,14 +311,14 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { tp.planMetadata = NewPlanMetadata(tags, detectedPlatform.Name(), testFramework.Name(), testSkippingLevel) if tp.optimizationClient == nil { if tp.newOptimizationClient == nil { - return fmt.Errorf("failed to create optimization client: missing client factory") + return errcode.New(errcode.PlanOptimizationClientCreationFailed, "failed to create optimization client: missing client factory") } tp.optimizationClient = tp.newOptimizationClient(testSkippingLevel) } resolvedTestFiles, err := discovery.ResolveTestFiles(testFramework.TestPattern(), settings.GetTestsExcludePattern()) if err != nil { - return err + return errcode.WithCode(errcode.PlanTestFilesResolutionFailed, err) } var skipMatcher skippableMatcher @@ -325,6 +333,9 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { var selectedDiscoveryMode discoveryMode var selectedDiscoveryDuration time.Duration var cacheResult discoveryCacheResult + recordDiscoveryTelemetry := func(mode telemetry.TestDiscoveryMode, success bool, duration time.Duration, discovered int) { + telemetry.TestDiscovery(tp.telemetryClient, mode, success, detectedPlatform.Name(), testFramework.Name(), duration, discovered) + } tp.resetDiscoveryResults() tp.testSuiteDurations = make(map[string]map[string]api.TestSuiteDurationInfo) @@ -345,7 +356,7 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { defer tp.optimizationClient.StoreCacheAndExit() if err := tp.optimizationClient.Initialize(tags); err != nil { - return fmt.Errorf("failed to initialize optimization client: %w", err) + return errcode.WithCode(errcode.PlanOptimizationClientInitializationFailed, fmt.Errorf("failed to initialize optimization client: %w", err)) } repositorySettings := tp.optimizationClient.GetSettings() @@ -406,6 +417,8 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { } res, discoveryErr := discoverLocalTests(discoveryCtx, testFramework, resolvedTestFiles) + discoveredTests = res + fullDiscoveryDuration = time.Since(fullDiscoveryStartTime) if discoveryErr != nil { if discoveryCtx.Err() == nil { fullDiscoveryErr = discoveryErr @@ -413,9 +426,7 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { return nil // Don't fail the entire process, we have fast discovery as fallback. } discoveryCache.store() - discoveredTests = res fullDiscoverySucceeded = true - fullDiscoveryDuration = time.Since(fullDiscoveryStartTime) return nil }) @@ -426,13 +437,13 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { slog.Info("Discovering test files (fast)...", "framework", testFramework.Name()) var res []string res, discErr := testFramework.DiscoverTestFiles(ctx, resolvedTestFiles) + discoveredTestFiles = res + fastDiscoveryDuration = time.Since(startTime) if discErr != nil { fastDiscoveryErr = discErr slog.Warn("Fast test discovery failed", "error", discErr) return nil // Don't fail the entire process if full discovery succeeded } - discoveredTestFiles = res - fastDiscoveryDuration = time.Since(startTime) slog.Info("Discovered test files (fast)", "duration", fastDiscoveryDuration, "count", len(discoveredTestFiles)) return nil @@ -446,8 +457,9 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { // backend data cancels it, use it even when TIA has no skips: full discovery // is more precise than fast file discovery. if fullDiscoverySucceeded { + recordDiscoveryTelemetry(telemetry.TestDiscoveryModeFull, true, fullDiscoveryDuration, len(discoveredTests)) if err := tp.recordFullDiscoveryResults(discoveredTests, resolvedTestFiles, skipMatcher); err != nil { - return err + return errcode.WithCode(errcode.PlanFullDiscoveryResultsProcessingFailed, err) } selectedDiscoveryMode = discoveryModeFull selectedDiscoveryDuration = fullDiscoveryDuration @@ -460,13 +472,16 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { "fastDiscoveredTestFilesCount", len(discoveredTestFiles)) } else { if strictDiscovery && fullDiscoveryErr != nil { - return fmt.Errorf("full test discovery failed: %w", fullDiscoveryErr) + recordDiscoveryTelemetry(telemetry.TestDiscoveryModeFull, false, fullDiscoveryDuration, len(discoveredTests)) + return errcode.WithCode(errcode.PlanFullTestDiscoveryFailed, fmt.Errorf("full test discovery failed: %w", fullDiscoveryErr)) } if fastDiscoveryErr != nil { - return fmt.Errorf("test discovery failed: %w", fastDiscoveryErr) + recordDiscoveryTelemetry(telemetry.TestDiscoveryModeFast, false, fastDiscoveryDuration, len(discoveredTestFiles)) + return errcode.WithCode(errcode.PlanFastTestDiscoveryFailed, fmt.Errorf("test discovery failed: %w", fastDiscoveryErr)) } + recordDiscoveryTelemetry(telemetry.TestDiscoveryModeFast, true, fastDiscoveryDuration, len(discoveredTestFiles)) if err := tp.recordFastDiscoveryFallbackFiles(discoveredTestFiles); err != nil { - return err + return errcode.WithCode(errcode.PlanFastDiscoveryResultsProcessingFailed, err) } selectedDiscoveryMode = discoveryModeFast selectedDiscoveryDuration = fastDiscoveryDuration @@ -486,12 +501,91 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { tp.testFileWeights = tp.calculateFileWeights() tp.recordDiscoveryReport(selectedDiscoveryMode, cacheResult, selectedDiscoveryDuration) + tp.tiaSkippingEnabled = tiaSkippingEnabled slog.Info("Test files prepared", "testFilesCount", len(tp.testFiles)) return nil } +func (tp *TestPlanner) recordPlanningTelemetry(selection splitSelection) { + backendDurationTestFiles := 0 + defaultDurationTestFiles := 0 + for _, source := range tp.testFileDurationSources { + if source == testFileDurationSourceKnown { + backendDurationTestFiles++ + } else { + defaultDurationTestFiles++ + } + } + + fullySkippedTestFiles := len(tp.testFiles) - len(tp.testFileWeights) + if fullySkippedTestFiles < 0 { + fullySkippedTestFiles = 0 + } + + telemetry.Planning(tp.telemetryClient, telemetry.PlanningMetrics{ + Attributes: telemetry.PlanningAttributes{ + Platform: tp.planMetadata.Platform, + Framework: tp.planMetadata.Framework, + TestSkippingMode: tp.planMetadata.TestSkippingLevel, + DiscoveryMode: telemetry.TestDiscoveryMode(tp.reportStats.discoveryMode), + TIAEnabled: tp.tiaSkippingEnabled, + }, + DecisionReason: planningDecisionReason(selection, len(tp.testFileWeights)), + TargetStatus: planningTargetStatus(selection), + DiscoveredTestFiles: len(tp.testFiles), + RunnableTestFiles: len(tp.testFileWeights), + FullySkippedTestFiles: fullySkippedTestFiles, + BackendDurationTestFiles: backendDurationTestFiles, + DefaultDurationTestFiles: defaultDurationTestFiles, + EstimatedTimeSavedPercent: tp.skippablePercentage, + ParallelRunners: selection.selected.parallelRunners, + ExpectedFullRuntime: tp.expectedFullDuration(), + ExpectedRunnableRuntime: selection.selected.totalRuntimeDuration(), + ExpectedWallTime: selection.selected.wallTimeDuration(), + SplitImbalancePercent: splitImbalancePercent(selection.selected), + DisabledTests: tp.reportStats.disabledTestsApplied, + UnskippableMarkerSuites: tp.reportStats.unskippableMarkerSuitesForced, + }) +} + +func planningDecisionReason(selection splitSelection, runnableTestFiles int) telemetry.PlanningDecisionReason { + if runnableTestFiles == 0 { + return telemetry.PlanningDecisionNoRunnableTests + } + if len(selection.candidates) == 1 && selection.selected.parallelRunners == 1 { + return telemetry.PlanningDecisionSingleRunnerOnly + } + if selection.targetTime <= 0 { + return telemetry.PlanningDecisionLowestScore + } + if selection.meetsTargetTime(selection.selected) { + if sameSplitScore(selection.selected, selection.bestWithoutTarget) { + return telemetry.PlanningDecisionTargetMetLowestScore + } + return telemetry.PlanningDecisionTargetMetChangedSelection + } + return telemetry.PlanningDecisionTargetUnreachableLowestWall +} + +func planningTargetStatus(selection splitSelection) telemetry.PlanningTargetStatus { + if selection.targetTime <= 0 { + return telemetry.PlanningTargetDisabled + } + if selection.meetsTargetTime(selection.selected) { + return telemetry.PlanningTargetMet + } + return telemetry.PlanningTargetMissed +} + +func splitImbalancePercent(split splitScore) float64 { + if split.wallTime <= 0 { + return 0 + } + return float64(split.imbalance) / float64(split.wallTime) * 100 +} + func discoverLocalTests(ctx context.Context, testFramework framework.Framework, testFiles discovery.TestFileSet) ([]testoptimization.Test, error) { startTime := time.Now() slog.Info("Discovering local tests...", "framework", testFramework.Name()) diff --git a/internal/planner/planner_test.go b/internal/planner/planner_test.go index 967cff7..8a2df1e 100644 --- a/internal/planner/planner_test.go +++ b/internal/planner/planner_test.go @@ -21,6 +21,7 @@ import ( "github.com/DataDog/ddtest/internal/constants" "github.com/DataDog/ddtest/internal/discovery" "github.com/DataDog/ddtest/internal/environment" + "github.com/DataDog/ddtest/internal/errcode" "github.com/DataDog/ddtest/internal/framework" "github.com/DataDog/ddtest/internal/platform" "github.com/DataDog/ddtest/internal/settings" @@ -74,6 +75,13 @@ func (c *plannerTelemetryClient) value(name string, tags ...string) float64 { return c.metrics[name+"|"+strings.Join(tags, ",")] } +func (c *plannerTelemetryClient) has(name string, tags ...string) bool { + c.mu.Lock() + defer c.mu.Unlock() + _, ok := c.metrics[name+"|"+strings.Join(tags, ",")] + return ok +} + func (m *MockPlatformDetector) DetectPlatform() (platform.Platform, error) { return m.Platform, m.Err } @@ -917,10 +925,21 @@ func TestTestPlanner_Plan_JestSuiteSkippingFetchesSkippablesWithoutFullDiscovery mockOptimizationClient, newDefaultMockCIProviderDetector(), ) + telemetryClient := newPlannerTelemetryClient() + commandAttributeTracker := telemetry.NewCLICommandAttributeTracker(telemetryClient) + runner.telemetryClient = commandAttributeTracker if err := runner.Plan(context.Background()); err != nil { t.Fatalf("Plan() should not return error, got: %v", err) } + wantCommandAttributes := telemetry.CLICommandAttributes{ + Platform: "javascript", + Framework: "jest", + TestSkippingMode: "suite", + } + if got := commandAttributeTracker.Attributes(); got != wantCommandAttributes { + t.Fatalf("CLI command attributes = %#v, want %#v", got, wantCommandAttributes) + } if !mockOptimizationClient.GetSkippablesCalled { t.Fatal("expected planner to fetch suite skippables when full test discovery is unsupported") @@ -954,6 +973,153 @@ func TestTestPlanner_Plan_JestSuiteSkippingFetchesSkippablesWithoutFullDiscovery assertFileContent(t, constants.TestFilesOutputPath, expectedTestFiles) assertFileContent(t, constants.SkippablePercentageOutputPath, "50.00") assertFileContent(t, filepath.Join(constants.TestsSplitDir, "runner-0"), expectedTestFiles) + discoveryTags := []string{"discovery_mode:fast", "success:true", "platform:javascript", "framework:jest"} + if !telemetryClient.has("ddtest.test_discovery.duration_ms", discoveryTags...) { + t.Fatal("expected fast discovery duration telemetry") + } + if got := telemetryClient.value("ddtest.test_discovery.test_files", discoveryTags...); got != 2 { + t.Fatalf("fast discovery test files = %v, want 2", got) + } + planningTags := []string{ + "platform:javascript", + "framework:jest", + "test_skipping_mode:suite", + "discovery_mode:fast", + "tia_enabled:true", + } + if !telemetryClient.has("ddtest.planning.decision", append(slices.Clone(planningTags), + "reason:single_runner_only", "target_status:disabled")...) { + t.Fatal("expected planning decision telemetry") + } + if got := telemetryClient.value("ddtest.planning.test_files", append(slices.Clone(planningTags), "state:discovered")...); got != 2 { + t.Fatalf("planning discovered test files = %v, want 2", got) + } + if got := telemetryClient.value("ddtest.planning.test_files", append(slices.Clone(planningTags), "state:runnable")...); got != 1 { + t.Fatalf("planning runnable test files = %v, want 1", got) + } + if got := telemetryClient.value("ddtest.planning.test_files", append(slices.Clone(planningTags), "state:fully_skipped")...); got != 1 { + t.Fatalf("planning fully skipped test files = %v, want 1", got) + } + if got := telemetryClient.value("ddtest.planning.test_file_durations", append(slices.Clone(planningTags), "source:backend")...); got != 1 { + t.Fatalf("planning backend duration test files = %v, want 1", got) + } + if got := telemetryClient.value("ddtest.planning.test_file_durations", append(slices.Clone(planningTags), "source:default")...); got != 0 { + t.Fatalf("planning default duration test files = %v, want 0", got) + } + if got := telemetryClient.value("ddtest.planning.estimated_time_saved_pct", planningTags...); got != 50 { + t.Fatalf("planning estimated time saved = %v, want 50", got) + } + if got := telemetryClient.value("ddtest.planning.parallel_runners", planningTags...); got != 1 { + t.Fatalf("planning parallel runners = %v, want 1", got) + } + if got := telemetryClient.value("ddtest.planning.expected_full_runtime_ms", planningTags...); got != 2000 { + t.Fatalf("planning expected full runtime = %v, want 2000", got) + } + if got := telemetryClient.value("ddtest.planning.expected_runnable_runtime_ms", planningTags...); got != 1000 { + t.Fatalf("planning expected runnable runtime = %v, want 1000", got) + } + if got := telemetryClient.value("ddtest.planning.expected_wall_time_ms", planningTags...); got != 1000 { + t.Fatalf("planning expected wall time = %v, want 1000", got) + } + if got := telemetryClient.value("ddtest.planning.split_imbalance_pct", planningTags...); got != 0 { + t.Fatalf("planning split imbalance = %v, want 0", got) + } + if got := telemetryClient.value("ddtest.planning.disabled_tests", planningTags...); got != 0 { + t.Fatalf("planning disabled tests = %v, want 0", got) + } + if got := telemetryClient.value("ddtest.planning.forced_run_suites", planningTags...); got != 0 { + t.Fatalf("planning forced run suites = %v, want 0", got) + } +} + +func TestPlanningDecisionReason(t *testing.T) { + tests := []struct { + name string + selection splitSelection + runnableTestFiles int + wantReason telemetry.PlanningDecisionReason + wantTargetStatus telemetry.PlanningTargetStatus + }{ + { + name: "no runnable tests", + selection: splitSelection{selected: splitScore{parallelRunners: 2}}, + runnableTestFiles: 0, + wantReason: telemetry.PlanningDecisionNoRunnableTests, + wantTargetStatus: telemetry.PlanningTargetDisabled, + }, + { + name: "single runner only", + selection: splitSelection{ + selected: splitScore{parallelRunners: 1, wallTime: 1000}, + candidates: []splitScore{{parallelRunners: 1, wallTime: 1000}}, + }, + runnableTestFiles: 1, + wantReason: telemetry.PlanningDecisionSingleRunnerOnly, + wantTargetStatus: telemetry.PlanningTargetDisabled, + }, + { + name: "lowest score without target", + selection: splitSelection{ + selected: splitScore{parallelRunners: 2, wallTime: 1000}, + candidates: []splitScore{{parallelRunners: 1, wallTime: 2000}, {parallelRunners: 2, wallTime: 1000}}, + }, + runnableTestFiles: 2, + wantReason: telemetry.PlanningDecisionLowestScore, + wantTargetStatus: telemetry.PlanningTargetDisabled, + }, + { + name: "target met by lowest score", + selection: splitSelection{ + selected: splitScore{parallelRunners: 2, wallTime: 1000}, + bestWithoutTarget: splitScore{parallelRunners: 2, wallTime: 1000}, + targetTime: 1500 * time.Millisecond, + }, + runnableTestFiles: 2, + wantReason: telemetry.PlanningDecisionTargetMetLowestScore, + wantTargetStatus: telemetry.PlanningTargetMet, + }, + { + name: "target changes selection", + selection: splitSelection{ + selected: splitScore{parallelRunners: 3, wallTime: 900}, + bestWithoutTarget: splitScore{parallelRunners: 2, wallTime: 1100}, + targetTime: time.Second, + }, + runnableTestFiles: 3, + wantReason: telemetry.PlanningDecisionTargetMetChangedSelection, + wantTargetStatus: telemetry.PlanningTargetMet, + }, + { + name: "target unreachable", + selection: splitSelection{ + selected: splitScore{parallelRunners: 3, wallTime: 1100}, + targetTime: time.Second, + }, + runnableTestFiles: 3, + wantReason: telemetry.PlanningDecisionTargetUnreachableLowestWall, + wantTargetStatus: telemetry.PlanningTargetMissed, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := planningDecisionReason(test.selection, test.runnableTestFiles); got != test.wantReason { + t.Errorf("planningDecisionReason() = %q, want %q", got, test.wantReason) + } + if got := planningTargetStatus(test.selection); got != test.wantTargetStatus { + t.Errorf("planningTargetStatus() = %q, want %q", got, test.wantTargetStatus) + } + }) + } +} + +func TestSplitImbalancePercent(t *testing.T) { + if got := splitImbalancePercent(splitScore{wallTime: 1000, imbalance: 250}); got != 25 { + t.Fatalf("splitImbalancePercent() = %v, want 25", got) + } + if got := splitImbalancePercent(splitScore{}); got != 0 { + t.Fatalf("splitImbalancePercent() for empty split = %v, want 0", got) + } } func TestTestPlanner_PreparePlanningData_RubySuiteModeSkipsFullDiscoveryAndSkipsFile(t *testing.T) { @@ -1126,6 +1292,13 @@ func TestTestPlanner_PreparePlanningData_RubySuiteModeForceFullDiscovery(t *test if got := telemetryClient.value("itr_forced_run", "event_type:suite"); got != 0 { t.Errorf("itr_forced_run suite count = %v, want 0", got) } + discoveryTags := []string{"discovery_mode:full", "success:true", "platform:ruby", "framework:rspec"} + if !telemetryClient.has("ddtest.test_discovery.duration_ms", discoveryTags...) { + t.Fatal("expected full discovery duration telemetry") + } + if got := telemetryClient.value("ddtest.test_discovery.tests", discoveryTags...); got != 5 { + t.Fatalf("full discovery tests = %v, want 5", got) + } } func TestTestPlanner_PreparePlanningData_ForceFullDiscoveryKeepsRunningWithNoTIASkippables(t *testing.T) { @@ -3122,6 +3295,8 @@ func TestTestPlanner_PreparePlanningData_StrictDiscoveryFailsWhenFullDiscoveryFa &MockTestOptimizationClient{}, newDefaultMockCIProviderDetector(), ) + telemetryClient := newPlannerTelemetryClient() + runner.telemetryClient = telemetryClient err := runner.PreparePlanningData(ctx) if err == nil { @@ -3133,6 +3308,12 @@ func TestTestPlanner_PreparePlanningData_StrictDiscoveryFailsWhenFullDiscoveryFa if !strings.Contains(err.Error(), "duplicate shared_context name") { t.Fatalf("PreparePlanningData() error = %v, want original discovery error", err) } + assertPlannerErrorCode(t, err, errcode.PlanFullTestDiscoveryFailed) + discoveryTags := []string{"discovery_mode:full", "success:false", "platform:ruby", "framework:rspec"} + if !telemetryClient.has("ddtest.test_discovery.duration_ms", discoveryTags...) || + !telemetryClient.has("ddtest.test_discovery.tests", discoveryTags...) { + t.Fatal("expected failed full discovery telemetry") + } } func TestTestPlanner_PreparePlanningData_StrictDiscoveryDoesNotFailWhenFullDiscoveryIsCancelled(t *testing.T) { @@ -3932,6 +4113,7 @@ func TestTestPlanner_PreparePlanningData_PlatformDetectionError(t *testing.T) { if !strings.Contains(err.Error(), expectedMsg) { t.Errorf("PreparePlanningData() error should contain '%s', got: %v", expectedMsg, err) } + assertPlannerErrorCode(t, err, errcode.PlanPlatformDetectionFailed) } func TestTestPlanner_PreparePlanningData_TagsCreationError(t *testing.T) { @@ -3959,6 +4141,7 @@ func TestTestPlanner_PreparePlanningData_TagsCreationError(t *testing.T) { if !strings.Contains(err.Error(), expectedMsg) { t.Errorf("PreparePlanningData() error should contain '%s', got: %v", expectedMsg, err) } + assertPlannerErrorCode(t, err, errcode.PlanPlatformTagsCreationFailed) } func TestTestPlanner_PreparePlanningData_OptimizationClientInitError(t *testing.T) { @@ -3995,6 +4178,7 @@ func TestTestPlanner_PreparePlanningData_OptimizationClientInitError(t *testing. if !strings.Contains(err.Error(), expectedMsg) { t.Errorf("PreparePlanningData() error should contain '%s', got: %v", expectedMsg, err) } + assertPlannerErrorCode(t, err, errcode.PlanOptimizationClientInitializationFailed) } func TestTestPlanner_PreparePlanningData_FrameworkDetectionError(t *testing.T) { @@ -4023,18 +4207,21 @@ func TestTestPlanner_PreparePlanningData_FrameworkDetectionError(t *testing.T) { if !strings.Contains(err.Error(), expectedMsg) { t.Errorf("PreparePlanningData() error should contain '%s', got: %v", expectedMsg, err) } + assertPlannerErrorCode(t, err, errcode.PlanFrameworkDetectionFailed) } func TestTestPlanner_PreparePlanningData_TestDiscoveryError(t *testing.T) { ctx := context.Background() mockFramework := &MockFramework{ - Err: errors.New("test discovery failed"), + FrameworkName: "rspec", + Err: errors.New("test discovery failed"), } mockPlatform := &MockPlatform{ - Tags: map[string]string{"platform": "ruby"}, - Framework: mockFramework, + PlatformName: "ruby", + Tags: map[string]string{"platform": "ruby"}, + Framework: mockFramework, } mockPlatformDetector := &MockPlatformDetector{ @@ -4044,6 +4231,8 @@ func TestTestPlanner_PreparePlanningData_TestDiscoveryError(t *testing.T) { mockOptimizationClient := &MockTestOptimizationClient{} runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, newDefaultMockCIProviderDetector()) + telemetryClient := newPlannerTelemetryClient() + runner.telemetryClient = telemetryClient err := runner.PreparePlanningData(ctx) @@ -4055,6 +4244,12 @@ func TestTestPlanner_PreparePlanningData_TestDiscoveryError(t *testing.T) { if !strings.Contains(err.Error(), expectedMsg) { t.Errorf("PreparePlanningData() error should contain '%s', got: %v", expectedMsg, err) } + assertPlannerErrorCode(t, err, errcode.PlanFastTestDiscoveryFailed) + discoveryTags := []string{"discovery_mode:fast", "success:false", "platform:ruby", "framework:rspec"} + if !telemetryClient.has("ddtest.test_discovery.duration_ms", discoveryTags...) || + !telemetryClient.has("ddtest.test_discovery.test_files", discoveryTags...) { + t.Fatal("expected failed fast discovery telemetry") + } } func TestTestPlanner_PreparePlanningData_EmptyTests(t *testing.T) { @@ -4257,6 +4452,7 @@ func TestTestPlanner_PreparePlanningData_RuntimeTagsOverrideInvalidJSON(t *testi if !strings.Contains(err.Error(), expectedMsg) { t.Errorf("PreparePlanningData() error should contain '%s', got: %v", expectedMsg, err) } + assertPlannerErrorCode(t, err, errcode.PlanRuntimeTagsInvalid) // Optimization client should not be initialized when there's a parse error if mockOptimizationClient.InitializeCalled { @@ -4671,3 +4867,10 @@ func TestPreparePlanningData_ITRSubdir_SkipMatching_WithSuitePathsMatchingCwd(t t.Errorf("Expected weighted test files to contain only order_spec.rb, got %v", weightedFiles) } } + +func assertPlannerErrorCode(t *testing.T, err error, want errcode.Code) { + t.Helper() + if got := errcode.CodeOf(err); got != want { + t.Fatalf("error code = %q, want %q; error: %v", got, want, err) + } +} diff --git a/internal/runner/ci_node_executor.go b/internal/runner/ci_node_executor.go index 3fe6e5c..adb566e 100644 --- a/internal/runner/ci_node_executor.go +++ b/internal/runner/ci_node_executor.go @@ -7,6 +7,7 @@ import ( "path/filepath" "github.com/DataDog/ddtest/internal/constants" + "github.com/DataDog/ddtest/internal/errcode" "golang.org/x/sync/errgroup" ) @@ -47,10 +48,10 @@ func loadCINodeTestFiles(ciNode int) ([]string, error) { runnerFilePath := filepath.Join(constants.TestsSplitDir, fmt.Sprintf("runner-%d", ciNode)) testFiles, err := loadTestBatch(runnerFilePath) if os.IsNotExist(err) { - return nil, fmt.Errorf("runner file for ci-node %d does not exist: %s", ciNode, runnerFilePath) + return nil, errcode.New(errcode.RunCINodeTestFilesMissing, fmt.Sprintf("runner file for ci-node %d does not exist: %s", ciNode, runnerFilePath)) } if err != nil { - return nil, fmt.Errorf("failed to read test files for ci-node %d from %s: %w", ciNode, runnerFilePath, err) + return nil, errcode.WithCode(errcode.RunCINodeTestFilesReadFailed, fmt.Errorf("failed to read test files for ci-node %d from %s: %w", ciNode, runnerFilePath, err)) } return testFiles, nil } @@ -61,7 +62,10 @@ func (e testExecutor) runCINodeSingleWorker(ciNode int, testFiles []string) erro slog.Info("No tests to run", "nodeIndex", ciNode, "workerIndex", 0) return nil } - return e.runBatch(testFiles, ciNode, 0) + if err := e.runBatch(testFiles, ciNode, 0); err != nil { + return errcode.WithCode(errcode.RunCINodeTestsFailed, fmt.Errorf("failed to run tests for ci-node %d: %w", ciNode, err)) + } + return nil } func (e testExecutor) runCINodeWorkers(ciNode int, ciNodeWorkers int, testFiles []string) error { @@ -95,7 +99,7 @@ func (e testExecutor) runCINodeWorkerGroups(ciNode int, groups [][]string) error } if err := g.Wait(); err != nil { - return fmt.Errorf("failed to run tests for ci-node %d: %w", ciNode, err) + return errcode.WithCode(errcode.RunCINodeTestsFailed, fmt.Errorf("failed to run tests for ci-node %d: %w", ciNode, err)) } return nil } diff --git a/internal/runner/parallel_executor.go b/internal/runner/parallel_executor.go index 8439cc4..119697b 100644 --- a/internal/runner/parallel_executor.go +++ b/internal/runner/parallel_executor.go @@ -7,6 +7,7 @@ import ( "path/filepath" "github.com/DataDog/ddtest/internal/constants" + "github.com/DataDog/ddtest/internal/errcode" "golang.org/x/sync/errgroup" ) @@ -21,7 +22,7 @@ func (e testExecutor) runParallel() runExecutionResult { entries, err := os.ReadDir(constants.TestsSplitDir) if err != nil { - return report.failure(fmt.Errorf("failed to read tests split directory %s: %w", constants.TestsSplitDir, err)) + return report.failure(errcode.WithCode(errcode.RunParallelSplitsReadFailed, fmt.Errorf("failed to read tests split directory %s: %w", constants.TestsSplitDir, err))) } var g errgroup.Group @@ -35,7 +36,7 @@ func (e testExecutor) runParallel() runExecutionResult { splitFilePath := filepath.Join(constants.TestsSplitDir, entry.Name()) testFiles, err := loadTestBatch(splitFilePath) if err != nil { - return report.failure(fmt.Errorf("failed to read test files from %s: %w", splitFilePath, err)) + return report.failure(errcode.WithCode(errcode.RunParallelTestFilesReadFailed, fmt.Errorf("failed to read test files from %s: %w", splitFilePath, err))) } report.TestFilesRun += len(testFiles) if len(testFiles) == 0 { @@ -48,7 +49,7 @@ func (e testExecutor) runParallel() runExecutionResult { } if err := g.Wait(); err != nil { - return report.failure(fmt.Errorf("failed to run parallel tests: %w", err)) + return report.failure(errcode.WithCode(errcode.RunParallelTestsFailed, fmt.Errorf("failed to run parallel tests: %w", err))) } return report.success() } diff --git a/internal/runner/runner.go b/internal/runner/runner.go index d57dfaa..6558870 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -11,6 +11,7 @@ import ( "github.com/DataDog/ddtest/internal/constants" ciUtils "github.com/DataDog/ddtest/internal/environment" + "github.com/DataDog/ddtest/internal/errcode" "github.com/DataDog/ddtest/internal/planner" "github.com/DataDog/ddtest/internal/platform" "github.com/DataDog/ddtest/internal/runmetadata" @@ -75,16 +76,16 @@ func (tr *TestRunner) Run(ctx context.Context) error { // Run Setup if the file doesn't exist if err := tr.planner.Plan(ctx); err != nil { - return fmt.Errorf("failed to run planning phase: %w", err) + return errcode.WithCode(errcode.RunPlanningFailed, fmt.Errorf("failed to run planning phase: %w", err)) } } else if err != nil { - return fmt.Errorf("failed to check parallel runners count at %s: %w", constants.ParallelRunnersOutputPath, err) + return errcode.WithCode(errcode.RunPlanStatusCheckFailed, fmt.Errorf("failed to check parallel runners count at %s: %w", constants.ParallelRunnersOutputPath, err)) } planMetadata, err := tr.planner.LoadPlan() if err != nil { slog.Error("Test optimization plan is not available", "error", err) - return fmt.Errorf("test optimization plan is not available: %w", err) + return errcode.WithCode(errcode.RunPlanLoadFailed, fmt.Errorf("test optimization plan is not available: %w", err)) } parallelRunners, err := readParallelRunnersCount() @@ -100,15 +101,20 @@ func (tr *TestRunner) Run(ctx context.Context) error { // Detect platform and framework detectedPlatform, err := tr.platformDetector.DetectPlatform() if err != nil { - return fmt.Errorf("failed to detect platform: %w", err) + return errcode.WithCode(errcode.RunPlatformDetectionFailed, fmt.Errorf("failed to detect platform: %w", err)) } slog.Info("Platform detected", "platform", detectedPlatform.Name()) framework, err := detectedPlatform.DetectFramework() if err != nil { - return fmt.Errorf("failed to detect framework: %w", err) + return errcode.WithCode(errcode.RunFrameworkDetectionFailed, fmt.Errorf("failed to detect framework: %w", err)) } slog.Info("Framework detected", "framework", framework.Name()) + telemetry.RecordCLICommandAttributes(tr.telemetryClient, telemetry.CLICommandAttributes{ + Platform: detectedPlatform.Name(), + Framework: framework.Name(), + TestSkippingMode: detectedPlatform.TestSkippingLevel().String(), + }) runInfo := runmetadata.New(ciUtils.GetCITags()) if planMetadata.IsZero() { planMetadata = planner.NewPlanMetadata(nil, detectedPlatform.Name(), framework.Name(), detectedPlatform.TestSkippingLevel()) @@ -141,13 +147,13 @@ func (tr *TestRunner) Run(ctx context.Context) error { func readParallelRunnersCount() (int, error) { runnersData, err := os.ReadFile(constants.ParallelRunnersOutputPath) if err != nil { - return 0, fmt.Errorf("failed to read parallel runners count from %s: %w", constants.ParallelRunnersOutputPath, err) + return 0, errcode.WithCode(errcode.RunParallelRunnersReadFailed, fmt.Errorf("failed to read parallel runners count from %s: %w", constants.ParallelRunnersOutputPath, err)) } runnersString := strings.TrimSpace(string(runnersData)) parallelRunners := 0 if _, err := fmt.Sscanf(runnersString, "%d", ¶llelRunners); err != nil { - return 0, fmt.Errorf("failed to parse parallel runners count from %s: %w", runnersString, err) + return 0, errcode.WithCode(errcode.RunParallelRunnersParseFailed, fmt.Errorf("failed to parse parallel runners count from %s: %w", runnersString, err)) } return parallelRunners, nil diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index 880bb25..299a4b9 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/DataDog/ddtest/internal/constants" + "github.com/DataDog/ddtest/internal/errcode" "github.com/DataDog/ddtest/internal/planner" "github.com/DataDog/ddtest/internal/settings" "github.com/DataDog/ddtest/internal/telemetry" @@ -114,10 +115,20 @@ func TestTestRunner_Run_UsesExistingArtifactsWithoutPlanning(t *testing.T) { platform := &MockPlatform{PlatformName: "ruby", Framework: framework} testPlanner := &fakePlanner{} runner := NewWithDependencies(&MockPlatformDetector{Platform: platform}, testPlanner) + commandAttributeTracker := telemetry.NewCLICommandAttributeTracker(telemetry.NoopClient()) + runner.telemetryClient = commandAttributeTracker if err := runner.Run(context.Background()); err != nil { t.Fatalf("Run() returned error: %v", err) } + wantCommandAttributes := telemetry.CLICommandAttributes{ + Platform: "ruby", + Framework: "rspec", + TestSkippingMode: "test", + } + if got := commandAttributeTracker.Attributes(); got != wantCommandAttributes { + t.Fatalf("CLI command attributes = %#v, want %#v", got, wantCommandAttributes) + } if testPlanner.planCalls != 0 { t.Fatalf("expected planner Plan() not to be called, got %d calls", testPlanner.planCalls) @@ -151,6 +162,7 @@ func TestTestRunner_Run_ReturnsErrorWhenPlanUnavailable(t *testing.T) { if !errors.Is(err, loadErr) { t.Fatalf("expected Run() to return LoadPlan() error, got %v", err) } + assertRunnerErrorCode(t, err, errcode.RunPlanLoadFailed) if testPlanner.loadCalls != 1 { t.Fatalf("expected LoadPlan() to be called once, got %d", testPlanner.loadCalls) } @@ -179,6 +191,7 @@ func TestTestRunner_Run_ReturnsPlanErrorWhenArtifactsAreMissing(t *testing.T) { if !errors.Is(err, planErr) { t.Fatalf("expected Run() to return planning error, got %v", err) } + assertRunnerErrorCode(t, err, errcode.RunPlanningFailed) if testPlanner.planCalls != 1 { t.Fatalf("expected Plan() to be called once, got %d", testPlanner.planCalls) } @@ -201,6 +214,7 @@ func TestTestRunner_Run_ReturnsStatErrorForBrokenRunnerArtifactsPath(t *testing. if err == nil || !strings.Contains(err.Error(), "failed to check parallel runners count") { t.Fatalf("Run() error = %v, want stat failure", err) } + assertRunnerErrorCode(t, err, errcode.RunPlanStatusCheckFailed) if testPlanner.planCalls != 0 { t.Fatalf("expected Plan() not to be called for stat failure, got %d", testPlanner.planCalls) } @@ -218,6 +232,7 @@ func TestTestRunner_Run_ReturnsErrorForInvalidParallelRunnerCount(t *testing.T) if err == nil || !strings.Contains(err.Error(), "failed to parse parallel runners count") { t.Fatalf("Run() error = %v, want parse failure", err) } + assertRunnerErrorCode(t, err, errcode.RunParallelRunnersParseFailed) } func TestTestRunner_Run_ReturnsPlatformDetectionError(t *testing.T) { @@ -233,6 +248,7 @@ func TestTestRunner_Run_ReturnsPlatformDetectionError(t *testing.T) { if !errors.Is(err, detectErr) { t.Fatalf("expected Run() to return platform detection error, got %v", err) } + assertRunnerErrorCode(t, err, errcode.RunPlatformDetectionFailed) } func TestTestRunner_Run_ReturnsFrameworkDetectionError(t *testing.T) { @@ -244,11 +260,22 @@ func TestTestRunner_Run_ReturnsFrameworkDetectionError(t *testing.T) { platform := &MockPlatform{PlatformName: "ruby", FrameworkErr: frameworkErr} testPlanner := &fakePlanner{} runner := NewWithDependencies(&MockPlatformDetector{Platform: platform}, testPlanner) + commandAttributeTracker := telemetry.NewCLICommandAttributeTracker(telemetry.NoopClient()) + runner.telemetryClient = commandAttributeTracker err := runner.Run(context.Background()) if !errors.Is(err, frameworkErr) { t.Fatalf("expected Run() to return framework detection error, got %v", err) } + assertRunnerErrorCode(t, err, errcode.RunFrameworkDetectionFailed) + wantCommandAttributes := telemetry.CLICommandAttributes{ + Platform: "unknown", + Framework: "unknown", + TestSkippingMode: "unknown", + } + if got := commandAttributeTracker.Attributes(); got != wantCommandAttributes { + t.Fatalf("CLI command attributes = %#v, want %#v", got, wantCommandAttributes) + } } func TestTestRunner_Run_WritesReportWhenEnabled(t *testing.T) { @@ -294,6 +321,7 @@ func TestRunSequentialMissingTestFile(t *testing.T) { if result.err == nil || !strings.Contains(result.err.Error(), "failed to read test files") { t.Fatalf("runSequential() error = %v, want missing test file error", result.err) } + assertRunnerErrorCode(t, result.err, errcode.RunSequentialTestFilesReadFailed) } func TestRunSequentialWithEmptyTestFile(t *testing.T) { @@ -326,6 +354,7 @@ func TestRunSequentialReturnsWorkerError(t *testing.T) { if result.err == nil || !strings.Contains(result.err.Error(), "failed to run tests") { t.Fatalf("runSequential() error = %v, want worker error", result.err) } + assertRunnerErrorCode(t, result.err, errcode.RunSequentialTestsFailed) } func TestRunParallelMissingSplitDirectory(t *testing.T) { @@ -337,6 +366,7 @@ func TestRunParallelMissingSplitDirectory(t *testing.T) { if result.err == nil || !strings.Contains(result.err.Error(), "failed to read tests split directory") { t.Fatalf("runParallel() error = %v, want missing split directory error", result.err) } + assertRunnerErrorCode(t, result.err, errcode.RunParallelSplitsReadFailed) } func TestRunParallelSkipsDirectoriesAndEmptyBatches(t *testing.T) { @@ -372,6 +402,7 @@ func TestRunParallelReturnsWorkerError(t *testing.T) { if result.err == nil || !strings.Contains(result.err.Error(), "failed to run parallel tests") { t.Fatalf("runParallel() error = %v, want worker error", result.err) } + assertRunnerErrorCode(t, result.err, errcode.RunParallelTestsFailed) } func TestNewCINodeExecutionReportDefaultsWorkers(t *testing.T) { @@ -392,6 +423,7 @@ func TestLoadCINodeTestFilesMissingFile(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "runner file for ci-node 7 does not exist") { t.Fatalf("loadCINodeTestFiles() error = %v, want missing runner file", err) } + assertRunnerErrorCode(t, err, errcode.RunCINodeTestFilesMissing) } func TestRunCINodeSingleWorkerWithEmptyBatch(t *testing.T) { @@ -540,3 +572,10 @@ func writeRunnerTestFile(t *testing.T, path string, content string) { t.Fatalf("failed to write %s: %v", path, err) } } + +func assertRunnerErrorCode(t *testing.T, err error, want errcode.Code) { + t.Helper() + if got := errcode.CodeOf(err); got != want { + t.Fatalf("error code = %q, want %q; error: %v", got, want, err) + } +} diff --git a/internal/runner/sequential_executor.go b/internal/runner/sequential_executor.go index 7cd95fc..5d83094 100644 --- a/internal/runner/sequential_executor.go +++ b/internal/runner/sequential_executor.go @@ -5,6 +5,7 @@ import ( "log/slog" "github.com/DataDog/ddtest/internal/constants" + "github.com/DataDog/ddtest/internal/errcode" ) const runModeSequential = "sequential" @@ -19,7 +20,7 @@ func (e testExecutor) runSequential() runExecutionResult { testFiles, err := loadTestBatch(constants.TestFilesOutputPath) if err != nil { - return report.failure(fmt.Errorf("failed to read test files from %s: %w", constants.TestFilesOutputPath, err)) + return report.failure(errcode.WithCode(errcode.RunSequentialTestFilesReadFailed, fmt.Errorf("failed to read test files from %s: %w", constants.TestFilesOutputPath, err))) } report.TestFilesRun = len(testFiles) @@ -29,7 +30,7 @@ func (e testExecutor) runSequential() runExecutionResult { } if err := e.runBatch(testFiles, 0, 0); err != nil { - return report.failure(fmt.Errorf("failed to run tests: %w", err)) + return report.failure(errcode.WithCode(errcode.RunSequentialTestsFailed, fmt.Errorf("failed to run tests: %w", err))) } return report.success() } diff --git a/internal/telemetry/civisibility.go b/internal/telemetry/civisibility.go index 0a5d5a9..bc79965 100644 --- a/internal/telemetry/civisibility.go +++ b/internal/telemetry/civisibility.go @@ -9,8 +9,10 @@ import ( "errors" "os/exec" "strconv" + "sync" "time" + "github.com/DataDog/ddtest/internal/errcode" "github.com/DataDog/ddtest/internal/git" ) @@ -22,6 +24,129 @@ const ( EventTypeSuite EventType = "suite" ) +// CLICommandType identifies a top-level ddtest command. +type CLICommandType string + +const ( + CLICommandPlan CLICommandType = "plan" + CLICommandRun CLICommandType = "run" +) + +// TestDiscoveryMode identifies the discovery strategy selected by the planner. +type TestDiscoveryMode string + +const ( + TestDiscoveryModeFull TestDiscoveryMode = "full" + TestDiscoveryModeFast TestDiscoveryMode = "fast" +) + +// PlanningDecisionReason identifies the constraint that determined the +// selected parallel runner split. +type PlanningDecisionReason string + +const ( + PlanningDecisionNoRunnableTests PlanningDecisionReason = "no_runnable_tests" + PlanningDecisionSingleRunnerOnly PlanningDecisionReason = "single_runner_only" + PlanningDecisionLowestScore PlanningDecisionReason = "lowest_score" + PlanningDecisionTargetMetLowestScore PlanningDecisionReason = "target_met_lowest_score" + PlanningDecisionTargetMetChangedSelection PlanningDecisionReason = "target_met_changed_selection" + PlanningDecisionTargetUnreachableLowestWall PlanningDecisionReason = "target_unreachable_lowest_wall_time" +) + +// PlanningTargetStatus identifies whether a configured target time affected +// the planning decision. +type PlanningTargetStatus string + +const ( + PlanningTargetDisabled PlanningTargetStatus = "disabled" + PlanningTargetMet PlanningTargetStatus = "met" + PlanningTargetMissed PlanningTargetStatus = "missed" +) + +// PlanningAttributes contains the bounded dimensions common to planning +// metrics. +type PlanningAttributes struct { + Platform string + Framework string + TestSkippingMode string + DiscoveryMode TestDiscoveryMode + TIAEnabled bool +} + +// PlanningMetrics contains the outcome of one completed planning operation. +type PlanningMetrics struct { + Attributes PlanningAttributes + DecisionReason PlanningDecisionReason + TargetStatus PlanningTargetStatus + DiscoveredTestFiles int + RunnableTestFiles int + FullySkippedTestFiles int + BackendDurationTestFiles int + DefaultDurationTestFiles int + EstimatedTimeSavedPercent float64 + ParallelRunners int + ExpectedFullRuntime time.Duration + ExpectedRunnableRuntime time.Duration + ExpectedWallTime time.Duration + SplitImbalancePercent float64 + DisabledTests int + UnskippableMarkerSuites int +} + +// CLICommandAttributes describes the resolved configuration attached to CLI +// command telemetry. +type CLICommandAttributes struct { + Platform string + Framework string + TestSkippingMode string +} + +const unknownCLICommandAttributeValue = "unknown" + +// CLICommandAttributeTracker forwards metrics to Client and retains the +// platform configuration detected during the command. Attributes remain +// unknown until platform and framework detection both succeed. +type CLICommandAttributeTracker struct { + Client + mu sync.RWMutex + attributes CLICommandAttributes +} + +func NewCLICommandAttributeTracker(client Client) *CLICommandAttributeTracker { + return &CLICommandAttributeTracker{ + Client: client, + attributes: CLICommandAttributes{ + Platform: unknownCLICommandAttributeValue, + Framework: unknownCLICommandAttributeValue, + TestSkippingMode: unknownCLICommandAttributeValue, + }, + } +} + +func (t *CLICommandAttributeTracker) Attributes() CLICommandAttributes { + t.mu.RLock() + defer t.mu.RUnlock() + return t.attributes +} + +func (t *CLICommandAttributeTracker) record(attributes CLICommandAttributes) { + t.mu.Lock() + defer t.mu.Unlock() + t.attributes = attributes +} + +type cliCommandAttributeRecorder interface { + record(CLICommandAttributes) +} + +// RecordCLICommandAttributes records values obtained from successful platform +// and framework detection when client is tracking top-level command telemetry. +func RecordCLICommandAttributes(client Client, attributes CLICommandAttributes) { + if recorder, ok := client.(cliCommandAttributeRecorder); ok { + recorder.record(attributes) + } +} + // SettingsResponse describes the settings flags represented in telemetry. type SettingsResponse struct { CodeCoverageEnabled bool @@ -122,6 +247,81 @@ func GitCommandMs(client Client, commandType git.CommandType, duration time.Dura distribution(client, "git.command_ms", []string{"command:" + string(commandType)}, milliseconds(duration)) } +func CLICommand(client Client, commandType CLICommandType, exitCode int, errorCode errcode.Code, attributes CLICommandAttributes) { + count(client, "ddtest.cli.command", cliCommandTags(commandType, exitCode, errorCode, attributes), 1) +} + +func CLICommandMs(client Client, commandType CLICommandType, exitCode int, errorCode errcode.Code, attributes CLICommandAttributes, duration time.Duration) { + distribution(client, "ddtest.cli.command_ms", cliCommandTags(commandType, exitCode, errorCode, attributes), milliseconds(duration)) +} + +func TestDiscovery(client Client, mode TestDiscoveryMode, success bool, platform, framework string, duration time.Duration, discovered int) { + tags := []string{ + "discovery_mode:" + string(mode), + "success:" + strconv.FormatBool(success), + "platform:" + platform, + "framework:" + framework, + } + distribution(client, "ddtest.test_discovery.duration_ms", tags, milliseconds(duration)) + + switch mode { + case TestDiscoveryModeFull: + distribution(client, "ddtest.test_discovery.tests", tags, float64(discovered)) + case TestDiscoveryModeFast: + distribution(client, "ddtest.test_discovery.test_files", tags, float64(discovered)) + } +} + +// Planning records the decisions and estimates from one completed plan. +func Planning(client Client, metrics PlanningMetrics) { + commonTags := planningTags(metrics.Attributes) + count(client, "ddtest.planning.decision", appendPlanningTags(commonTags, + "reason:"+string(metrics.DecisionReason), + "target_status:"+string(metrics.TargetStatus), + ), 1) + + distribution(client, "ddtest.planning.test_files", appendPlanningTags(commonTags, "state:discovered"), float64(metrics.DiscoveredTestFiles)) + distribution(client, "ddtest.planning.test_files", appendPlanningTags(commonTags, "state:runnable"), float64(metrics.RunnableTestFiles)) + distribution(client, "ddtest.planning.test_files", appendPlanningTags(commonTags, "state:fully_skipped"), float64(metrics.FullySkippedTestFiles)) + distribution(client, "ddtest.planning.estimated_time_saved_pct", commonTags, metrics.EstimatedTimeSavedPercent) + distribution(client, "ddtest.planning.test_file_durations", appendPlanningTags(commonTags, "source:backend"), float64(metrics.BackendDurationTestFiles)) + distribution(client, "ddtest.planning.test_file_durations", appendPlanningTags(commonTags, "source:default"), float64(metrics.DefaultDurationTestFiles)) + distribution(client, "ddtest.planning.parallel_runners", commonTags, float64(metrics.ParallelRunners)) + distribution(client, "ddtest.planning.expected_full_runtime_ms", commonTags, milliseconds(metrics.ExpectedFullRuntime)) + distribution(client, "ddtest.planning.expected_runnable_runtime_ms", commonTags, milliseconds(metrics.ExpectedRunnableRuntime)) + distribution(client, "ddtest.planning.expected_wall_time_ms", commonTags, milliseconds(metrics.ExpectedWallTime)) + distribution(client, "ddtest.planning.split_imbalance_pct", commonTags, metrics.SplitImbalancePercent) + distribution(client, "ddtest.planning.disabled_tests", commonTags, float64(metrics.DisabledTests)) + distribution(client, "ddtest.planning.forced_run_suites", commonTags, float64(metrics.UnskippableMarkerSuites)) +} + +func planningTags(attributes PlanningAttributes) []string { + return []string{ + "platform:" + attributes.Platform, + "framework:" + attributes.Framework, + "test_skipping_mode:" + attributes.TestSkippingMode, + "discovery_mode:" + string(attributes.DiscoveryMode), + "tia_enabled:" + strconv.FormatBool(attributes.TIAEnabled), + } +} + +func appendPlanningTags(tags []string, additional ...string) []string { + result := make([]string, 0, len(tags)+len(additional)) + result = append(result, tags...) + return append(result, additional...) +} + +func cliCommandTags(commandType CLICommandType, exitCode int, errorCode errcode.Code, attributes CLICommandAttributes) []string { + return []string{ + "command:" + string(commandType), + "exit_code:" + strconv.Itoa(exitCode), + "error_code:" + string(errorCode), + "platform:" + attributes.Platform, + "framework:" + attributes.Framework, + "test_skipping_mode:" + attributes.TestSkippingMode, + } +} + func gitCommandErrorTags(err error) []string { var exitErr *exec.ExitError if !errors.As(err, &exitErr) { @@ -215,6 +415,10 @@ func ITRSkippableTestsResponseSuites(client Client, value int) { count(client, "itr_skippable_tests.response_suites", nil, float64(value)) } +func ITRSkippableTestsIsEmpty(client Client) { + count(client, "ddtest.itr_skippable_tests.is_empty", nil, 1) +} + func ITRSkippableTestsRequestMs(client Client, duration time.Duration) { distribution(client, "itr_skippable_tests.request_ms", nil, milliseconds(duration)) } @@ -266,3 +470,27 @@ func TestManagementTestsResponseBytes(client Client, responseCompressed bool, va func TestManagementTestsResponseTests(client Client, value int) { distribution(client, "test_management_tests.response_tests", nil, float64(value)) } + +func TestSuiteDurationsRequest(client Client, requestCompressed bool) { + count(client, "test_suite_durations.request", requestCompressedTags(requestCompressed), 1) +} + +func TestSuiteDurationsRequestErrors(client Client, statusCode int) { + count(client, "test_suite_durations.request_errors", requestErrorTags(statusCode), 1) +} + +func TestSuiteDurationsRequestMs(client Client, duration time.Duration) { + distribution(client, "test_suite_durations.request_ms", nil, milliseconds(duration)) +} + +func TestSuiteDurationsResponseBytes(client Client, responseCompressed bool, value int) { + distribution(client, "test_suite_durations.response_bytes", responseCompressedTags(responseCompressed), float64(value)) +} + +func TestSuiteDurationsResponseSuites(client Client, value int) { + distribution(client, "test_suite_durations.response_suites", nil, float64(value)) +} + +func TestSuiteDurationsIsEmpty(client Client) { + count(client, "test_suite_durations.is_empty", nil, 1) +} diff --git a/internal/telemetry/civisibility_test.go b/internal/telemetry/civisibility_test.go index d6767a1..ce1c232 100644 --- a/internal/telemetry/civisibility_test.go +++ b/internal/telemetry/civisibility_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + "github.com/DataDog/ddtest/internal/errcode" "github.com/DataDog/ddtest/internal/git" ) @@ -98,6 +99,7 @@ func TestCIVisibilityRequestMetrics(t *testing.T) { ITRSkippableTestsResponseBytes(client, true, 43) ITRSkippableTestsResponseTests(client, 3) ITRSkippableTestsResponseSuites(client, 2) + ITRSkippableTestsIsEmpty(client) KnownTestsRequest(client, false) KnownTestsRequestErrors(client, 0) KnownTestsRequestMs(client, duration) @@ -108,6 +110,12 @@ func TestCIVisibilityRequestMetrics(t *testing.T) { TestManagementTestsRequestMs(client, duration) TestManagementTestsResponseBytes(client, true, 45) TestManagementTestsResponseTests(client, 5) + TestSuiteDurationsRequest(client, true) + TestSuiteDurationsRequestErrors(client, 429) + TestSuiteDurationsRequestMs(client, duration) + TestSuiteDurationsResponseBytes(client, true, 46) + TestSuiteDurationsResponseSuites(client, 6) + TestSuiteDurationsIsEmpty(client) want := []recordedMetric{ {kind: "count", name: "git_requests.search_commits", tags: []string{"rq_compressed:true"}, value: 1}, @@ -127,6 +135,7 @@ func TestCIVisibilityRequestMetrics(t *testing.T) { {kind: "distribution", name: "itr_skippable_tests.response_bytes", tags: []string{"rs_compressed:true"}, value: 43}, {kind: "count", name: "itr_skippable_tests.response_tests", value: 3}, {kind: "count", name: "itr_skippable_tests.response_suites", value: 2}, + {kind: "count", name: "ddtest.itr_skippable_tests.is_empty", value: 1}, {kind: "count", name: "known_tests.request", value: 1}, {kind: "count", name: "known_tests.request_errors", tags: []string{"error_type:network"}, value: 1}, {kind: "distribution", name: "known_tests.request_ms", value: 1500}, @@ -137,6 +146,12 @@ func TestCIVisibilityRequestMetrics(t *testing.T) { {kind: "distribution", name: "test_management_tests.request_ms", value: 1500}, {kind: "distribution", name: "test_management_tests.response_bytes", tags: []string{"rs_compressed:true"}, value: 45}, {kind: "distribution", name: "test_management_tests.response_tests", value: 5}, + {kind: "count", name: "test_suite_durations.request", tags: []string{"rq_compressed:true"}, value: 1}, + {kind: "count", name: "test_suite_durations.request_errors", tags: []string{"error_type:status_code_4xx_response", "status_code:429"}, value: 1}, + {kind: "distribution", name: "test_suite_durations.request_ms", value: 1500}, + {kind: "distribution", name: "test_suite_durations.response_bytes", tags: []string{"rs_compressed:true"}, value: 46}, + {kind: "distribution", name: "test_suite_durations.response_suites", value: 6}, + {kind: "count", name: "test_suite_durations.is_empty", value: 1}, } assertRecordedMetrics(t, client.metrics, want) } @@ -182,6 +197,141 @@ func TestGitCommandMetrics(t *testing.T) { assertRecordedMetrics(t, client.metrics, want) } +func TestCLICommandMetrics(t *testing.T) { + client := &recordingClient{} + attributes := CLICommandAttributes{ + Platform: "ruby", + Framework: "rspec", + TestSkippingMode: "suite", + } + CLICommand(client, CLICommandPlan, 0, errcode.None, attributes) + CLICommandMs(client, CLICommandPlan, 0, errcode.None, attributes, 1500*time.Millisecond) + CLICommand(client, CLICommandRun, 1, errcode.RunParallelTestsFailed, attributes) + CLICommandMs(client, CLICommandRun, 1, errcode.RunParallelTestsFailed, attributes, 2*time.Second) + tags := func(command, exitCode string, errorCode errcode.Code) []string { + return []string{ + "command:" + command, + "exit_code:" + exitCode, + "error_code:" + string(errorCode), + "platform:ruby", + "framework:rspec", + "test_skipping_mode:suite", + } + } + + want := []recordedMetric{ + {kind: "count", name: "ddtest.cli.command", tags: tags("plan", "0", errcode.None), value: 1}, + {kind: "distribution", name: "ddtest.cli.command_ms", tags: tags("plan", "0", errcode.None), value: 1500}, + {kind: "count", name: "ddtest.cli.command", tags: tags("run", "1", errcode.RunParallelTestsFailed), value: 1}, + {kind: "distribution", name: "ddtest.cli.command_ms", tags: tags("run", "1", errcode.RunParallelTestsFailed), value: 2000}, + } + assertRecordedMetrics(t, client.metrics, want) +} + +func TestCLICommandAttributeTracker(t *testing.T) { + client := &recordingClient{} + tracker := NewCLICommandAttributeTracker(client) + + unknown := CLICommandAttributes{ + Platform: "unknown", + Framework: "unknown", + TestSkippingMode: "unknown", + } + if got := tracker.Attributes(); got != unknown { + t.Fatalf("initial attributes = %#v, want %#v", got, unknown) + } + + detected := CLICommandAttributes{ + Platform: "javascript", + Framework: "jest", + TestSkippingMode: "suite", + } + RecordCLICommandAttributes(tracker, detected) + if got := tracker.Attributes(); got != detected { + t.Fatalf("detected attributes = %#v, want %#v", got, detected) + } + + tracker.Count("forwarded", nil).Submit(1) + assertRecordedMetrics(t, client.metrics, []recordedMetric{{kind: "count", name: "forwarded", value: 1}}) +} + +func TestTestDiscoveryMetrics(t *testing.T) { + client := &recordingClient{} + TestDiscovery(client, TestDiscoveryModeFull, true, "ruby", "rspec", 1500*time.Millisecond, 42) + TestDiscovery(client, TestDiscoveryModeFast, false, "javascript", "jest", 2*time.Second, 3) + + fullTags := []string{"discovery_mode:full", "success:true", "platform:ruby", "framework:rspec"} + fastTags := []string{"discovery_mode:fast", "success:false", "platform:javascript", "framework:jest"} + want := []recordedMetric{ + {kind: "distribution", name: "ddtest.test_discovery.duration_ms", tags: fullTags, value: 1500}, + {kind: "distribution", name: "ddtest.test_discovery.tests", tags: fullTags, value: 42}, + {kind: "distribution", name: "ddtest.test_discovery.duration_ms", tags: fastTags, value: 2000}, + {kind: "distribution", name: "ddtest.test_discovery.test_files", tags: fastTags, value: 3}, + } + assertRecordedMetrics(t, client.metrics, want) +} + +func TestPlanningMetrics(t *testing.T) { + client := &recordingClient{} + Planning(client, PlanningMetrics{ + Attributes: PlanningAttributes{ + Platform: "ruby", + Framework: "rspec", + TestSkippingMode: "test", + DiscoveryMode: TestDiscoveryModeFull, + TIAEnabled: true, + }, + DecisionReason: PlanningDecisionTargetMetChangedSelection, + TargetStatus: PlanningTargetMet, + DiscoveredTestFiles: 10, + RunnableTestFiles: 7, + FullySkippedTestFiles: 3, + BackendDurationTestFiles: 5, + DefaultDurationTestFiles: 2, + EstimatedTimeSavedPercent: 30, + ParallelRunners: 3, + ExpectedFullRuntime: 10 * time.Second, + ExpectedRunnableRuntime: 7 * time.Second, + ExpectedWallTime: 2500 * time.Millisecond, + SplitImbalancePercent: 20, + DisabledTests: 4, + UnskippableMarkerSuites: 1, + }) + + commonTags := []string{ + "platform:ruby", + "framework:rspec", + "test_skipping_mode:test", + "discovery_mode:full", + "tia_enabled:true", + } + withTag := func(tag string) []string { + return append(slices.Clone(commonTags), tag) + } + want := []recordedMetric{ + { + kind: "count", + name: "ddtest.planning.decision", + tags: append(withTag("reason:target_met_changed_selection"), "target_status:met"), + value: 1, + }, + {kind: "distribution", name: "ddtest.planning.test_files", tags: withTag("state:discovered"), value: 10}, + {kind: "distribution", name: "ddtest.planning.test_files", tags: withTag("state:runnable"), value: 7}, + {kind: "distribution", name: "ddtest.planning.test_files", tags: withTag("state:fully_skipped"), value: 3}, + {kind: "distribution", name: "ddtest.planning.estimated_time_saved_pct", tags: commonTags, value: 30}, + {kind: "distribution", name: "ddtest.planning.test_file_durations", tags: withTag("source:backend"), value: 5}, + {kind: "distribution", name: "ddtest.planning.test_file_durations", tags: withTag("source:default"), value: 2}, + {kind: "distribution", name: "ddtest.planning.parallel_runners", tags: commonTags, value: 3}, + {kind: "distribution", name: "ddtest.planning.expected_full_runtime_ms", tags: commonTags, value: 10000}, + {kind: "distribution", name: "ddtest.planning.expected_runnable_runtime_ms", tags: commonTags, value: 7000}, + {kind: "distribution", name: "ddtest.planning.expected_wall_time_ms", tags: commonTags, value: 2500}, + {kind: "distribution", name: "ddtest.planning.split_imbalance_pct", tags: commonTags, value: 20}, + {kind: "distribution", name: "ddtest.planning.disabled_tests", tags: commonTags, value: 4}, + {kind: "distribution", name: "ddtest.planning.forced_run_suites", tags: commonTags, value: 1}, + } + assertRecordedMetrics(t, client.metrics, want) +} + func TestGitCommandExitCode(t *testing.T) { for _, test := range []struct { exitCode int diff --git a/internal/testoptimization/api/durations_api.go b/internal/testoptimization/api/durations_api.go index eb434d6..528fdc0 100644 --- a/internal/testoptimization/api/durations_api.go +++ b/internal/testoptimization/api/durations_api.go @@ -4,6 +4,8 @@ import ( "fmt" "log/slog" "time" + + "github.com/DataDog/ddtest/internal/telemetry" ) const ( @@ -150,9 +152,10 @@ func (c *transport) fetchTestSuiteDurations(repositoryURL, service string) (map[ } duration := time.Since(startTime) - totalSuites := 0 - for _, suites := range allSuites { - totalSuites += len(suites) + totalSuites := countTestSuiteDurations(allSuites) + telemetry.TestSuiteDurationsResponseSuites(c.telemetryClient, totalSuites) + if totalSuites == 0 { + telemetry.TestSuiteDurationsIsEmpty(c.telemetryClient) } slog.Debug("Finished fetching test suite durations", "modules", len(allSuites), "suites", totalSuites, "duration", duration) @@ -184,10 +187,18 @@ func (c *transport) fetchTestSuiteDurationsPage(repositoryURL, service, cursor s } request := c.getPostRequestConfig(durationsURLPath, body) + telemetry.TestSuiteDurationsRequest(c.telemetryClient, request.Compressed) + requestStartTime := time.Now() response, err := c.handler.SendRequest(*request) + telemetry.TestSuiteDurationsRequestMs(c.telemetryClient, time.Since(requestStartTime)) if err != nil { + telemetry.TestSuiteDurationsRequestErrors(c.telemetryClient, responseStatusCode(response)) return nil, fmt.Errorf("sending test suite durations request: %s", err) } + if response.StatusCode < 200 || response.StatusCode >= 300 { + telemetry.TestSuiteDurationsRequestErrors(c.telemetryClient, response.StatusCode) + } + telemetry.TestSuiteDurationsResponseBytes(c.telemetryClient, response.Compressed, response.BodySize) slog.Debug("test_suite_durations", "responseBody", string(response.Body)) diff --git a/internal/testoptimization/api/http.go b/internal/testoptimization/api/http.go index c0b1fc2..338804d 100644 --- a/internal/testoptimization/api/http.go +++ b/internal/testoptimization/api/http.go @@ -38,6 +38,9 @@ const ( FormatMessagePack = "msgpack" ) +// Longer or malformed reset delays fall back to the existing capped backoff. +const maxRateLimitResetSeconds = 30 + // FormFile represents a file to be uploaded in a multipart form request. type FormFile struct { FieldName string // The name of the form field @@ -253,21 +256,9 @@ func (rh *RequestHandler) internalSendRequest(config *RequestConfig, attempt int slog.Debug("ciVisibilityHttpClient: response status code", "statusCode", resp.StatusCode) rateLimitReset := resp.Header.Get(HeaderRateLimitReset) - if rateLimitReset != "" { - if resetTime, err := strconv.ParseInt(rateLimitReset, 10, 64); err == nil { - var waitDuration time.Duration - if resetTime > time.Now().Unix() { - // Assume it's a Unix timestamp - waitDuration = time.Until(time.Unix(resetTime, 0)) - } else { - // Assume it's a duration in seconds - waitDuration = time.Duration(resetTime) * time.Second - } - if waitDuration > 0 { - time.Sleep(waitDuration) - } - return false, retryResponse, nil - } + if waitDuration, ok := parseRateLimitReset(rateLimitReset); ok { + time.Sleep(waitDuration) + return false, retryResponse, nil } // Fallback to exponential backoff if header is missing or invalid @@ -317,6 +308,17 @@ func (rh *RequestHandler) internalSendRequest(config *RequestConfig, attempt int return true, &Response{Body: responseBody, BodySize: responseBodySize, Format: responseFormat, StatusCode: statusCode, CanUnmarshal: canUnmarshal, Compressed: compressedResponse}, nil } +func parseRateLimitReset(value string) (time.Duration, bool) { + // Datadog defines X-RateLimit-Reset as the number of seconds until reset + // (not a Unix timestamp); reject invalid or excessive delays so a response + // cannot block the CLI indefinitely. + resetSeconds, err := strconv.ParseInt(value, 10, 64) + if err != nil || resetSeconds <= 0 || resetSeconds > maxRateLimitResetSeconds { + return 0, false + } + return time.Duration(resetSeconds) * time.Second, true +} + // Helper functions for data serialization, compression, and handling multipart form data // serializeData serializes the data based on the format. diff --git a/internal/testoptimization/api/skippable.go b/internal/testoptimization/api/skippable.go index cd18f72..8efced9 100644 --- a/internal/testoptimization/api/skippable.go +++ b/internal/testoptimization/api/skippable.go @@ -145,6 +145,9 @@ func (c *transport) GetSkippableTests() (correlationID string, skippables Skippa } else { telemetry.ITRSkippableTestsResponseTests(c.telemetryClient, len(responseObject.Data)) } + if len(responseObject.Data) == 0 { + telemetry.ITRSkippableTestsIsEmpty(c.telemetryClient) + } skippables = NewSkippables() warnedMissingTestBundle := false diff --git a/internal/testoptimization/api/telemetry_test.go b/internal/testoptimization/api/telemetry_test.go index 8f5ddee..788163f 100644 --- a/internal/testoptimization/api/telemetry_test.go +++ b/internal/testoptimization/api/telemetry_test.go @@ -7,6 +7,7 @@ package api import ( "context" + "errors" "io" "net/http" "net/http/httptest" @@ -98,8 +99,13 @@ func TestTransportRecordsBackendTelemetry(t *testing.T) { } skippableBody := `{"meta":{"correlation_id":"cid"},"data":[{"type":"test","attributes":{"suite":"suite-a","name":"test-a","configurations":{"test.bundle":"module-a"}}},{"type":"test","attributes":{"suite":"suite-b","name":"test-b","configurations":{"test.bundle":"module-b"}}}]}` testManagementBody := `{"data":{"attributes":{"modules":{"module-a":{"suites":{"suite-a":{"tests":{"test-a":{"properties":{}},"test-b":{"properties":{}}}}}}}}}}` + durationsBodies := []string{ + `{"data":{"attributes":{"test_suites":{"module-a":{"suite-a":{"source_file":"a_test.go","duration":{"p50":"100","p90":"200"}}}},"page_info":{"cursor":"page-2","has_next":true}}}}`, + `{"data":{"attributes":{"test_suites":{"module-a":{"suite-b":{"source_file":"b_test.go","duration":{"p50":"300","p90":"400"}}},"module-b":{"suite-c":{"source_file":"c_test.go","duration":{"p50":"500","p90":"600"}}}},"page_info":{"has_next":false}}}}`, + } searchCommitsBody := `{"data":[{"id":"remote-commit","type":"commit"}]}` knownRequests := 0 + durationsRequests := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set(HeaderContentType, constants.ContentTypeJSON) var body string @@ -113,6 +119,9 @@ func TestTransportRecordsBackendTelemetry(t *testing.T) { body = skippableBody case testManagementTestsURLPath: body = testManagementBody + case durationsURLPath: + body = durationsBodies[durationsRequests] + durationsRequests++ case searchCommitsURLPath: body = searchCommitsBody case sendPackFilesURLPath: @@ -140,6 +149,9 @@ func TestTransportRecordsBackendTelemetry(t *testing.T) { if _, err := client.GetTestManagementTests(); err != nil { t.Fatalf("GetTestManagementTests() error = %v", err) } + if durations := client.GetTestSuiteDurations(); len(durations.TestSuites) != 2 { + t.Fatalf("GetTestSuiteDurations() modules = %d, want 2", len(durations.TestSuites)) + } if _, err := client.GetCommits([]string{"local-commit"}); err != nil { t.Fatalf("GetCommits() error = %v", err) } @@ -172,10 +184,16 @@ func TestTransportRecordsBackendTelemetry(t *testing.T) { recorder.assertValue(t, "distribution", "itr_skippable_tests.response_bytes", nil, float64(len(skippableBody))) recorder.assertValue(t, "count", "itr_skippable_tests.response_tests", nil, 2) recorder.assertSamples(t, "count", "itr_skippable_tests.response_suites", nil, 0) + recorder.assertSamples(t, "count", "ddtest.itr_skippable_tests.is_empty", nil, 0) recorder.assertValue(t, "count", "test_management_tests.request", nil, 1) recorder.assertSamples(t, "distribution", "test_management_tests.request_ms", nil, 1) recorder.assertValue(t, "distribution", "test_management_tests.response_bytes", nil, float64(len(testManagementBody))) recorder.assertValue(t, "distribution", "test_management_tests.response_tests", nil, 2) + recorder.assertSamples(t, "count", "test_suite_durations.request", nil, 2) + recorder.assertSamples(t, "distribution", "test_suite_durations.request_ms", nil, 2) + recorder.assertSamples(t, "distribution", "test_suite_durations.response_bytes", nil, 2) + recorder.assertValue(t, "distribution", "test_suite_durations.response_suites", nil, 3) + recorder.assertSamples(t, "count", "test_suite_durations.is_empty", nil, 0) recorder.assertValue(t, "count", "git_requests.search_commits", nil, 1) recorder.assertSamples(t, "distribution", "git_requests.search_commits_ms", nil, 1) recorder.assertSamples(t, "count", "git_requests.objects_pack", nil, 2) @@ -201,6 +219,28 @@ func TestTransportRecordsSuiteOnlySkippableResponse(t *testing.T) { recorder.assertValue(t, "count", "itr_skippable_tests.response_suites", nil, 1) recorder.assertSamples(t, "count", "itr_skippable_tests.response_tests", nil, 0) + recorder.assertSamples(t, "count", "ddtest.itr_skippable_tests.is_empty", nil, 0) +} + +func TestTransportRecordsEmptySkippableResponse(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set(HeaderContentType, constants.ContentTypeJSON) + _, _ = io.WriteString(w, `{"meta":{"correlation_id":"cid"},"data":[]}`) + })) + defer server.Close() + + recorder := &apiRecordingClient{} + client := newRawResponseTestClient(server) + client.telemetryClient = recorder + + if _, skippables, err := client.GetSkippableTests(); err != nil { + t.Fatalf("GetSkippableTests() error = %v", err) + } else if len(skippables.Tests) != 0 || len(skippables.Suites) != 0 { + t.Fatalf("GetSkippableTests() = %#v, want no skippables", skippables) + } + + recorder.assertValue(t, "count", "itr_skippable_tests.response_tests", nil, 0) + recorder.assertValue(t, "count", "ddtest.itr_skippable_tests.is_empty", nil, 1) } func TestTransportRecordsCompressedResponseWireBytes(t *testing.T) { @@ -208,6 +248,7 @@ func TestTransportRecordsCompressedResponseWireBytes(t *testing.T) { knownTestsURLPath: `{"data":{"attributes":{"tests":{"module-a":{"suite-a":["test-a"]}}}}}`, skippableURLPath: `{"meta":{"correlation_id":"cid"},"data":[{"type":"test","attributes":{"suite":"suite-a","name":"test-a","configurations":{"test.bundle":"module-a"}}}]}`, testManagementTestsURLPath: `{"data":{"attributes":{"modules":{"module-a":{"suites":{"suite-a":{"tests":{"test-a":{"properties":{}}}}}}}}}}`, + durationsURLPath: `{"data":{"attributes":{"test_suites":{"module-a":{"suite-a":{"source_file":"a_test.go","duration":{"p50":"100","p90":"200"}}}}}}}`, } compressedBodies := make(map[string][]byte, len(responseBodies)) for path, body := range responseBodies { @@ -243,11 +284,15 @@ func TestTransportRecordsCompressedResponseWireBytes(t *testing.T) { if _, err := client.GetTestManagementTests(); err != nil { t.Fatalf("GetTestManagementTests() error = %v", err) } + if durations := client.GetTestSuiteDurations(); len(durations.TestSuites) != 1 { + t.Fatalf("GetTestSuiteDurations() modules = %d, want 1", len(durations.TestSuites)) + } tags := []string{"rs_compressed:true"} recorder.assertValue(t, "distribution", "known_tests.response_bytes", tags, float64(len(compressedBodies[knownTestsURLPath]))) recorder.assertValue(t, "distribution", "itr_skippable_tests.response_bytes", tags, float64(len(compressedBodies[skippableURLPath]))) recorder.assertValue(t, "distribution", "test_management_tests.response_bytes", tags, float64(len(compressedBodies[testManagementTestsURLPath]))) + recorder.assertValue(t, "distribution", "test_suite_durations.response_bytes", tags, float64(len(compressedBodies[durationsURLPath]))) } func TestTransportRecordsTerminalRetryStatusAndFailedSearchCommitsLatency(t *testing.T) { @@ -269,6 +314,51 @@ func TestTransportRecordsTerminalRetryStatusAndFailedSearchCommitsLatency(t *tes recorder.assertSamples(t, "distribution", "git_requests.search_commits_ms", nil, 1) } +func TestTransportRecordsTestSuiteDurationsNetworkError(t *testing.T) { + recorder := &apiRecordingClient{} + client := &transport{ + agentless: true, + baseURL: "https://example.test", + serviceName: "my-service", + repositoryURL: "github.com/DataDog/foo", + headers: map[string]string{}, + handler: NewRequestHandlerWithClient(&http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return nil, errors.New("network unavailable") + })}), + telemetryClient: recorder, + } + + if durations := client.GetTestSuiteDurations(); len(durations.TestSuites) != 0 { + t.Fatalf("GetTestSuiteDurations() suites = %d, want 0", len(durations.TestSuites)) + } + + recorder.assertValue(t, "count", "test_suite_durations.request", nil, 1) + recorder.assertSamples(t, "distribution", "test_suite_durations.request_ms", nil, 1) + recorder.assertValue(t, "count", "test_suite_durations.request_errors", []string{"error_type:network"}, 1) + recorder.assertSamples(t, "distribution", "test_suite_durations.response_bytes", nil, 0) + recorder.assertSamples(t, "distribution", "test_suite_durations.response_suites", nil, 0) + recorder.assertSamples(t, "count", "test_suite_durations.is_empty", nil, 0) +} + +func TestTransportRecordsEmptyTestSuiteDurationsResponse(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set(HeaderContentType, constants.ContentTypeJSON) + _, _ = io.WriteString(w, `{"data":{"attributes":{"test_suites":{}}}}`) + })) + defer server.Close() + + recorder := &apiRecordingClient{} + client := newRawResponseTestClient(server) + client.telemetryClient = recorder + + if durations := client.GetTestSuiteDurations(); len(durations.TestSuites) != 0 { + t.Fatalf("GetTestSuiteDurations() suites = %d, want 0", len(durations.TestSuites)) + } + + recorder.assertValue(t, "distribution", "test_suite_durations.response_suites", nil, 0) + recorder.assertValue(t, "count", "test_suite_durations.is_empty", nil, 1) +} + func TestTransportRecordsBackendStatusErrors(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "bad request", http.StatusBadRequest) @@ -282,6 +372,7 @@ func TestTransportRecordsBackendStatusErrors(t *testing.T) { _, _ = client.GetKnownTests() _, _, _ = client.GetSkippableTests() _, _ = client.GetTestManagementTests() + _ = client.GetTestSuiteDurations() _, _ = client.GetCommits([]string{"local-commit"}) packFile := filepath.Join(t.TempDir(), "objects.pack") if err := os.WriteFile(packFile, []byte("pack"), 0o600); err != nil { @@ -295,11 +386,13 @@ func TestTransportRecordsBackendStatusErrors(t *testing.T) { "known_tests.request_errors", "itr_skippable_tests.request_errors", "test_management_tests.request_errors", + "test_suite_durations.request_errors", "git_requests.search_commits_errors", "git_requests.objects_pack_errors", } { recorder.assertValue(t, "count", name, tags, 1) } + recorder.assertSamples(t, "count", "ddtest.itr_skippable_tests.is_empty", nil, 0) } func TestNewTransportWithTelemetryRetainsClient(t *testing.T) { diff --git a/internal/testoptimization/api/transport_test.go b/internal/testoptimization/api/transport_test.go index 230d43d..2b99d66 100644 --- a/internal/testoptimization/api/transport_test.go +++ b/internal/testoptimization/api/transport_test.go @@ -306,6 +306,41 @@ func TestHTTPSerializationHelpersErrorBranches(t *testing.T) { } } +func TestParseRateLimitReset(t *testing.T) { + tests := []struct { + name string + value string + want time.Duration + wantValid bool + }{ + {name: "seconds", value: "2", want: 2 * time.Second, wantValid: true}, + { + name: "maximum accepted delay", + value: "30", + want: 30 * time.Second, + wantValid: true, + }, + {name: "excessive delay", value: "31"}, + {name: "Unix timestamp", value: "2000000005"}, + {name: "zero", value: "0"}, + {name: "negative", value: "-1"}, + {name: "integer overflow", value: "9223372036854775808"}, + {name: "not an integer", value: "soon"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, valid := parseRateLimitReset(tt.value) + if valid != tt.wantValid { + t.Fatalf("parseRateLimitReset(%q) valid = %t, want %t", tt.value, valid, tt.wantValid) + } + if got != tt.want { + t.Fatalf("parseRateLimitReset(%q) = %s, want %s", tt.value, got, tt.want) + } + }) + } +} + func TestSerializeDataReaderAndBytes(t *testing.T) { bytesData, err := serializeData([]byte("bytes"), constants.FormatJSON) if err != nil || string(bytesData) != "bytes" {