feat(dashboard-agent): give the golden runs their own eval sample rate

The CI lane reads its own variable and defaults to every turn, so neither lane can change
the other's rate.

Author: kathiekiwi

internal-packages/dashboard-agent/package.json

@@ -28,7 +28,7 @@
     "typecheck": "tsc --noEmit",
     "test": "vitest run",
     "test:watch": "vitest",
-    "test:evals": "vitest run --config vitest.eval.config.ts",
+    "test:evals": "DASHBOARD_AGENT_EVAL_CONTEXT=ci vitest run --config vitest.eval.config.ts",
     "dev": "trigger dev",
     "deploy": "trigger deploy"
   }

internal-packages/dashboard-agent/src/dashboard-agent.test.ts

@@ -19,9 +19,11 @@ import {
   dashboardAgentModelKey,
   dashboardAgentStoreKey,
   dashboardAgentToolsKey,
+  DEFAULT_CI_EVAL_SAMPLE_RATE,
   DEFAULT_EVAL_SAMPLE_RATE,
   evalSampleRate,
   extractToolActivity,
+  isCiEvalContext,
   MAX_EVAL_TOOL_OUTPUT_CHARS,
   sanitizeReplayedToolInputs,
   truncateEvalToolOutput,
@@ -515,6 +517,63 @@ describe("per-turn eval sampling", () => {
   });
 });
 
+describe("the CI sample rate", () => {
+  const original = {
+    context: process.env.DASHBOARD_AGENT_EVAL_CONTEXT,
+    ci: process.env.DASHBOARD_AGENT_EVAL_SAMPLE_RATE_CI,
+    production: process.env.DASHBOARD_AGENT_EVAL_SAMPLE_RATE,
+  };
+
+  function restore(name: string, value: string | undefined) {
+    if (value === undefined) delete process.env[name];
+    else process.env[name] = value;
+  }
+
+  afterEach(() => {
+    restore("DASHBOARD_AGENT_EVAL_CONTEXT", original.context);
+    restore("DASHBOARD_AGENT_EVAL_SAMPLE_RATE_CI", original.ci);
+    restore("DASHBOARD_AGENT_EVAL_SAMPLE_RATE", original.production);
+  });
+
+  it("judges every turn in the CI lane", () => {
+    process.env.DASHBOARD_AGENT_EVAL_CONTEXT = "ci";
+    delete process.env.DASHBOARD_AGENT_EVAL_SAMPLE_RATE_CI;
+    expect(isCiEvalContext()).toBe(true);
+    expect(evalSampleRate()).toBe(DEFAULT_CI_EVAL_SAMPLE_RATE);
+    expect(DEFAULT_CI_EVAL_SAMPLE_RATE).toBe(1);
+  });
+
+  it("does not let the CI rate reach production", () => {
+    delete process.env.DASHBOARD_AGENT_EVAL_CONTEXT;
+    delete process.env.DASHBOARD_AGENT_EVAL_SAMPLE_RATE;
+    process.env.DASHBOARD_AGENT_EVAL_SAMPLE_RATE_CI = "1";
+    expect(isCiEvalContext()).toBe(false);
+    expect(evalSampleRate()).toBe(DEFAULT_EVAL_SAMPLE_RATE);
+  });
+
+  it("does not let the production rate de-sample CI", () => {
+    process.env.DASHBOARD_AGENT_EVAL_CONTEXT = "ci";
+    process.env.DASHBOARD_AGENT_EVAL_SAMPLE_RATE = "0";
+    delete process.env.DASHBOARD_AGENT_EVAL_SAMPLE_RATE_CI;
+    expect(evalSampleRate()).toBe(1);
+  });
+
+  it("honours a parseable CI rate, and falls back to full on a bad one", () => {
+    process.env.DASHBOARD_AGENT_EVAL_CONTEXT = "ci";
+    process.env.DASHBOARD_AGENT_EVAL_SAMPLE_RATE_CI = "0.25";
+    expect(evalSampleRate()).toBe(0.25);
+    process.env.DASHBOARD_AGENT_EVAL_SAMPLE_RATE_CI = "nope";
+    expect(evalSampleRate()).toBe(DEFAULT_CI_EVAL_SAMPLE_RATE);
+  });
+
+  it("only the exact context value selects the CI lane", () => {
+    for (const raw of ["CI", "true", "1", "golden", ""]) {
+      process.env.DASHBOARD_AGENT_EVAL_CONTEXT = raw;
+      expect(isCiEvalContext()).toBe(false);
+    }
+  });
+});
+
 describe("the per-org opt-out", () => {
   let harness: MockChatAgentHarness | undefined;
   const originalRate = process.env.DASHBOARD_AGENT_EVAL_SAMPLE_RATE;

internal-packages/dashboard-agent/src/dashboard-agent.ts

@@ -8,7 +8,12 @@ import {
   type ToolSet,
   type UIMessage,
 } from "ai";
-import { orgAllowsTurnEvals, redactEvalToolValue, turnReadSource } from "./eval-policy";
+import {
+  orgAllowsTurnEvals,
+  redactEvalToolValue,
+  shouldEvalTurn,
+  turnReadSource,
+} from "./eval-policy";
 import type { EvalTurnPayload, evalTurn } from "./eval-turn";
 import {
   buildTurnTools,
@@ -41,6 +46,18 @@ export {
   sanitizeReplayedToolInputs,
   type DashboardAgentStore,
 } from "./agent-runtime";
+// The eval's data-handling policy lives in `eval-policy.ts`; re-exported so every
+// existing import path still resolves.
+export {
+  DEFAULT_CI_EVAL_SAMPLE_RATE,
+  DEFAULT_EVAL_SAMPLE_RATE,
+  evalSampleRate,
+  isCiEvalContext,
+  orgAllowsTurnEvals,
+  redactEvalToolValue,
+  shouldEvalTurn,
+  turnReadSource,
+} from "./eval-policy";
 export {
   dashboardAgentActionSchema,
   wakeStartsInvestigation,
@@ -132,29 +149,6 @@ function getEvalTrigger(): DashboardAgentEvalTrigger {
   );
 }
 
-/**
- * Fraction of turns to eval, from DASHBOARD_AGENT_EVAL_SAMPLE_RATE. Read per turn
- * so the rate can change without a redeploy.
- *
- * The judge is a full model call per turn, and nothing reads `chat_turn_evals`
- * yet, so the default samples a tenth — including when the value is unparseable,
- * where the old fallback of 1 quietly turned a typo into full-rate billing.
- */
-export const DEFAULT_EVAL_SAMPLE_RATE = 0.1;
-
-export function evalSampleRate(): number {
-  const raw = process.env.DASHBOARD_AGENT_EVAL_SAMPLE_RATE;
-  if (raw === undefined || raw.trim() === "") return DEFAULT_EVAL_SAMPLE_RATE;
-  const parsed = Number(raw);
-  if (!Number.isFinite(parsed) || parsed < 0 || parsed > 1) return DEFAULT_EVAL_SAMPLE_RATE;
-  return parsed;
-}
-
-// `Math.random()` is in [0, 1), so rate 0 never samples and rate 1 always does.
-function shouldEvalTurn(): boolean {
-  return Math.random() < evalSampleRate();
-}
-
 function extractText(message: UIMessage): string {
   return (message.parts ?? [])
     .flatMap((part) => (part.type === "text" ? [part.text] : []))