fix(evals): remove synthetic system prompt hints, install skills natively per harness (AI-1034)

README.md

@@ -143,10 +143,10 @@ Skills come from [`supabase/agent-skills`](https://github.com/supabase/agent-ski
 
 To use a skill in an experiment, reference its directory name in the experiment's `skills` array.
 
-Both runtimes load skills lazily ([progressive disclosure](https://ai-sdk.dev/cookbook/guides/agent-skills)): only each skill's name+description is in the system prompt, and the agent pulls a skill's full instructions on demand. They differ only in how the body is fetched, because the tools-mode agent has no filesystem:
+Skills are always loaded lazily ([progressive disclosure](https://ai-sdk.dev/cookbook/guides/agent-skills)) — a skill's full instructions are pulled on demand, never preloaded. How that happens depends on the harness:
 
-- **Local-stack (sandbox) mode:** skills are installed into the workspace with [Vercel's `skills` CLI](https://github.com/vercel-labs/skills) (baked into the sandbox image, sourced from the local `skills/` directory — never the network) under `.claude/skills/`. When a task matches, the agent reads `.claude/skills/<name>/SKILL.md` (and any files it references) with its file tools.
-- **Tools mode:** no filesystem, so a `load_skill` tool returns a skill's full instructions when the agent calls it with the skill's name.
+- **CLI harnesses (Claude Code, Codex, OpenCode)** use their own built-in skills mechanism. Skills are installed into the sandbox workspace with [Vercel's `skills` CLI](https://github.com/vercel-labs/skills) (baked into the sandbox image, sourced from the local `skills/` directory — never the network), for each harness's own project scope: `.claude/skills/` for Claude Code, `.agents/skills/` for Codex and OpenCode. Each CLI then discovers, advertises and loads the skills itself. The framework injects nothing — an agent's real-world skill-following behaviour is part of what an eval measures.
+- **The in-process `ai-sdk` harness** has no such mechanism, so the framework supplies one. In local-stack mode it lists each skill's name+description in the system prompt and the agent reads `.claude/skills/<name>/SKILL.md` with its file tools. In tools mode there is no filesystem at all, so a `load_skill` tool returns a skill's full instructions when the agent calls it with the skill's name.
 
 ## Framework Checks
 

apps/framework/harness/run-eval.ts

@@ -29,6 +29,7 @@ import {
 } from '../lib/cli-args.js';
 import { bootPlatformBackend } from './platform-backend.js';
 import { viteBuild, vitestRun } from './project-runner.js';
+import { buildSystemPrompt } from './system-prompt.js';
 import {
   buildDocsResult,
   buildSkillResult,
@@ -237,9 +238,9 @@ function buildLoadSkillTool(skills: readonly ToolsSkill[]): ToolSet {
 }
 
 /**
- * Local-stack skill sources: resolve each skill name to its host directory so
- * the sandbox can install it with Vercel's `skills` CLI; the agent then
- * discovers each skill by reading its SKILL.md with its file tools. The
+ * Sandbox skill sources: resolve each skill name to its host directory so the
+ * sandbox can install it with Vercel's `skills` CLI, which places it in every
+ * CLI harness's native project scope for that harness to discover. The
  * `skills/` entries are symlinks into the agent-skills submodule; realpath them
  * so `docker cp` copies real files, not dangling links. Missing skills are
  * skipped with a warning.
@@ -298,31 +299,6 @@ function readSessionSeedArgs(ev: EvalManifest) {
   };
 }
 
-function basePromptFor(mode: EvalMode): string {
-  if (mode === 'local-stack') {
-    return (
-      'You are an agent solving a Supabase eval task in a Linux workspace. ' +
-      'Use the provided tools to inspect and modify the workspace and run commands. ' +
-      'When you are done, end your turn with a short summary of what you did.'
-    );
-  }
-  return (
-    'You are an agent solving a Supabase eval task. ' +
-    'Use the provided tools to inspect and modify the project. ' +
-    'When you are done, end your turn with a short summary of what you did ' +
-    '(or for audit tasks, your findings).'
-  );
-}
-
-function buildSystemPrompt(
-  mode: EvalMode,
-  addendum?: string,
-  skillContext?: string
-): string {
-  const blocks = [basePromptFor(mode), addendum, skillContext].filter(Boolean);
-  return blocks.join('\n\n');
-}
-
 /**
  * Adapt a `{ close() }` resource to `AsyncDisposable` so it can be bound with
  * `await using` — cleanup then runs on scope exit (normal fall-through, `continue`,
@@ -352,6 +328,13 @@ async function runOne(
     transcript: TranscriptPart[];
     agentReport: string;
     stoppedReason: string;
+    /**
+     * The exact system prompt handed to the agent (`''` when it got none). CLI
+     * harnesses receive theirs as a file in the sandbox scratch dir, outside the
+     * exported workspace, so recording it here is the only way to verify from a
+     * run artifact what the agent was actually told.
+     */
+    systemPrompt: string;
   }
 > {
   const prompt = parseEvalMarkdown(
@@ -383,6 +366,7 @@ async function runOne(
   let lastTranscript: TranscriptPart[] = [];
   let lastAgentReport = '';
   let lastStoppedReason = 'not_started';
+  let lastSystemPrompt = '';
 
   for (let attempt = 1; attempt <= RUNS; attempt += 1) {
     if (ev.mode === 'local-stack') {
@@ -415,6 +399,7 @@ async function runOne(
         : undefined;
       await using session = disposable(
         await exp.localStack.startSession({
+          agent: exp.agent.id,
           cliVersion: ev.metadata.cliVersion,
           localDir: ev.localDir,
           includeServices: ev.metadata.services,
@@ -435,8 +420,13 @@ async function runOne(
         })
       );
 
+      const systemPrompt = buildSystemPrompt(
+        exp.agent.id,
+        'local-stack',
+        session.promptAddendum
+      );
       const run = await exp.agent.run({
-        systemPrompt: buildSystemPrompt('local-stack', session.promptAddendum),
+        systemPrompt,
         userPrompt: prompt,
         tools: session.tools,
         sandbox: session.sandbox,
@@ -447,6 +437,7 @@ async function runOne(
       lastTranscript = run.transcript;
       lastAgentReport = run.agentReport;
       lastStoppedReason = run.stoppedReason;
+      lastSystemPrompt = systemPrompt;
 
       // Export the agent's workspace to the host so scorers can run host
       // tooling (vite/vitest from the repo root) against the produced files
@@ -488,6 +479,7 @@ async function runOne(
           transcript: run.transcript,
           agentReport: run.agentReport,
           stoppedReason: run.stoppedReason,
+          systemPrompt,
         };
       }
       logRetryAttempt(expName, ev, attempt, last);
@@ -500,7 +492,12 @@ async function runOne(
     // platform-lite via host.docker.internal (so platform-lite binds 0.0.0.0).
     // An in-process agent runs host-side with no sandbox.
     await using cliSandbox = agentRunsInSandbox
-      ? disposable(await createBareSandbox({ skills: skillSources }))
+      ? disposable(
+          await createBareSandbox({
+            agent: exp.agent.id,
+            skills: skillSources,
+          })
+        )
       : undefined;
     await using session = disposable(
       await exp.runtime.startSession({
@@ -509,14 +506,16 @@ async function runOne(
       })
     );
 
-    // CLI agents read their installed skills from disk (the bare sandbox folds
-    // the discovery listing into its promptAddendum). In-process agents have
-    // no filesystem, so their skills are advertised in the prompt and pulled
-    // on demand via the load_skill tool.
+    // CLI agents discover their installed skills themselves — the skills CLI
+    // put them in every harness's native project scope, so each one advertises
+    // and loads them in its own words and the bare sandbox contributes nothing
+    // here. In-process agents have no filesystem, so their skills are advertised
+    // in the prompt and pulled on demand via the load_skill tool.
     const skillsPrompt = agentRunsInSandbox
       ? cliSandbox!.promptAddendum
       : buildToolsSkillsPrompt(toolsSkills);
     const systemPrompt = buildSystemPrompt(
+      exp.agent.id,
       'tools',
       session.promptAddendum,
       skillsPrompt
@@ -533,6 +532,7 @@ async function runOne(
     lastTranscript = run.transcript;
     lastAgentReport = run.agentReport;
     lastStoppedReason = run.stoppedReason;
+    lastSystemPrompt = systemPrompt;
     last = await (scorer as ToolScorer)({
       ...session.scoringContext,
       toolCalls: run.toolCalls,
@@ -554,6 +554,7 @@ async function runOne(
         transcript: run.transcript,
         agentReport: run.agentReport,
         stoppedReason: run.stoppedReason,
+        systemPrompt,
       };
     }
     logRetryAttempt(expName, ev, attempt, last);
@@ -568,6 +569,7 @@ async function runOne(
     transcript: lastTranscript,
     agentReport: lastAgentReport,
     stoppedReason: lastStoppedReason,
+    systemPrompt: lastSystemPrompt,
   };
 }
 

apps/framework/harness/system-prompt.test.ts

@@ -0,0 +1,93 @@
+import { describe, expect, it } from 'vitest';
+import type { AgentHarnessId } from '@supabase-evals/core';
+import {
+  buildSkillsPrompt,
+  buildToolSurfaceAddendum,
+  type SkillEntry,
+} from '@supabase-evals/sandbox';
+import { buildSystemPrompt } from './system-prompt.js';
+import type { EvalMode } from './types.js';
+
+const CLI_AGENTS: AgentHarnessId[] = ['claude-code', 'codex', 'opencode'];
+const MODES: EvalMode[] = ['tools', 'local-stack'];
+
+describe('buildSystemPrompt', () => {
+  it('gives the ai-sdk agent task framing in both modes', () => {
+    // ai-sdk is the one harness with no system prompt of its own, so it's the
+    // one harness the framework has to supply one for.
+    for (const mode of MODES) {
+      expect(buildSystemPrompt('ai-sdk', mode)).toContain(
+        'Use the provided tools'
+      );
+    }
+  });
+
+  it('gives no framing of our own to any CLI harness', () => {
+    // CLI harnesses ship their own system prompt; we're measuring that.
+    for (const agent of CLI_AGENTS) {
+      for (const mode of MODES) {
+        expect(buildSystemPrompt(agent, mode)).toBe('');
+      }
+    }
+  });
+
+  it('passes a CLI harness only the runtime blocks, with no base prompt', () => {
+    for (const agent of CLI_AGENTS) {
+      expect(
+        buildSystemPrompt(agent, 'local-stack', 'Addendum.', 'Skills listing.')
+      ).toBe('Addendum.\n\nSkills listing.');
+    }
+  });
+
+  it('assembles to nothing at all for a CLI harness, even with skills', () => {
+    // The real block producers, not stand-ins: with skills installed, a CLI
+    // harness must still receive an entirely empty system prompt. Codex and
+    // OpenCode find the skills through their own project-scope discovery and
+    // describe them to the model themselves.
+    const skills: SkillEntry[] = [
+      {
+        name: 'supabase',
+        description: 'Use for Supabase tasks.',
+        dir: '.claude/skills/supabase',
+      },
+    ];
+    for (const agent of CLI_AGENTS) {
+      expect(
+        buildSystemPrompt(
+          agent,
+          'local-stack',
+          buildToolSurfaceAddendum(agent),
+          buildSkillsPrompt(agent, skills)
+        )
+      ).toBe('');
+    }
+    // ai-sdk has no such mechanism — it only learns about skills from us.
+    const aiSdk = buildSystemPrompt(
+      'ai-sdk',
+      'local-stack',
+      buildToolSurfaceAddendum('ai-sdk'),
+      buildSkillsPrompt('ai-sdk', skills)
+    );
+    expect(aiSdk).toContain('## Available skills');
+    expect(aiSdk).toContain('- supabase: Use for Supabase tasks.');
+  });
+
+  it('never tells any agent how to end its turn', () => {
+    // Stopping behaviour is part of what an eval measures, so the harness must
+    // not coach it (e.g. "end your turn with a short summary").
+    for (const agent of [...CLI_AGENTS, 'ai-sdk' as const]) {
+      for (const mode of MODES) {
+        const prompt = buildSystemPrompt(agent, mode);
+        expect(prompt).not.toMatch(/summary/i);
+        expect(prompt).not.toMatch(/end your turn/i);
+      }
+    }
+  });
+
+  it('drops empty blocks instead of leaving blank gaps', () => {
+    expect(
+      buildSystemPrompt('claude-code', 'tools', '', 'Skills listing.')
+    ).toBe('Skills listing.');
+    expect(buildSystemPrompt('ai-sdk', 'tools', '', '')).not.toMatch(/\n\n$/);
+  });
+});

apps/framework/harness/system-prompt.ts

@@ -0,0 +1,52 @@
+/**
+ * System-prompt assembly, per agent harness.
+ *
+ * An eval measures out-of-the-box agent behaviour, so the harness injects as
+ * little prompt of its own as it can get away with: only the ai-sdk agent gets
+ * any base framing, because it is the only harness with no system prompt of its
+ * own (`aiSdkAgent` hands `systemPrompt` straight to the model's `system`). CLI
+ * agents ship their own coding-agent prompt, tool guidance, and stopping
+ * behaviour — and codex/opencode have no system-prompt flag at all, so anything
+ * we pass them lands on the *user* prompt.
+ */
+
+import type { AgentHarnessId } from '@supabase-evals/core';
+import type { EvalMode } from './types.js';
+
+/**
+ * Base framing for the ai-sdk harness: what it can't infer on its own — that it
+ * has tools, and what they act on. Deliberately silent on how to finish a turn
+ * (no "end with a summary"): stopping behaviour is part of what's measured.
+ * Empty for every CLI harness.
+ */
+function basePromptFor(agent: AgentHarnessId, mode: EvalMode): string {
+  if (agent !== 'ai-sdk') return '';
+  if (mode === 'local-stack') {
+    return (
+      'You are an agent solving a Supabase eval task in a Linux workspace. ' +
+      'Use the provided tools to inspect and modify the workspace and run commands.'
+    );
+  }
+  return (
+    'You are an agent solving a Supabase eval task. ' +
+    'Use the provided tools to inspect and modify the project.'
+  );
+}
+
+/**
+ * Assemble the system prompt handed to the agent. Every block is optional, and
+ * every one of them is ai-sdk-only (the base framing, the tool-surface addendum,
+ * the skills listing), so a CLI harness ends up with `''` — the CLI engine then
+ * stages no system-prompt file at all rather than an empty one.
+ */
+export function buildSystemPrompt(
+  agent: AgentHarnessId,
+  mode: EvalMode,
+  addendum?: string,
+  skillContext?: string
+): string {
+  const blocks = [basePromptFor(agent, mode), addendum, skillContext].filter(
+    Boolean
+  );
+  return blocks.join('\n\n');
+}

apps/framework/package.json

@@ -4,12 +4,13 @@
   "version": "0.0.1",
   "type": "module",
   "scripts": {
-    "check": "pnpm typecheck && pnpm test:framework && pnpm test:vercel-runner",
+    "check": "pnpm typecheck && pnpm test && pnpm test:framework && pnpm test:vercel-runner",
     "eval": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts",
     "eval:dry": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --dry",
     "eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke",
     "eval:vercel": "node --env-file=../../.env --import tsx/esm scripts/run-vercel-evals.ts",
     "typecheck": "tsc --noEmit",
+    "test": "vitest run harness",
     "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts",
     "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts lib/cli-args.test.ts",
     "export-results": "node --import tsx/esm scripts/export-results.ts",

packages/core/src/agents/claude-code/runner.test.ts

@@ -36,6 +36,46 @@ function streamJson(subtype: string, isError = false): string {
   ].join('\n');
 }
 
+/** The `claude` invocation from one exec, with a fake sandbox. */
+async function captureRunCommand(
+  systemPromptPath: string | undefined
+): Promise<string> {
+  let runCommand = '';
+  await claudeCodeRunner.exec({
+    sandbox: {
+      workspace: '/w',
+      exec: async (cmd) => {
+        if (cmd.includes('/bin/claude')) runCommand = cmd;
+        return ok;
+      },
+      readFile: async () => '',
+    },
+    model: 'claude-sonnet-4-6',
+    apiKey: 'k',
+    systemPromptPath,
+    userPromptPath: '"$HOME/.eval/user-prompt.txt"',
+    mcpServers: {},
+    timeoutSec: 1,
+  });
+  return runCommand;
+}
+
+describe('claudeCodeRunner.exec', () => {
+  it('appends the harness system prompt when there is one', async () => {
+    const command = await captureRunCommand('"$HOME/.eval/system-prompt.txt"');
+    expect(command).toContain(
+      '--append-system-prompt-file "$HOME/.eval/system-prompt.txt"'
+    );
+  });
+
+  it("omits the flag with no system prompt, leaving Claude Code's own intact", async () => {
+    const command = await captureRunCommand(undefined);
+    expect(command).not.toContain('--append-system-prompt-file');
+    // The task itself is still piped in.
+    expect(command).toContain('cat "$HOME/.eval/user-prompt.txt"');
+  });
+});
+
 describe('claudeCodeRunner.deriveStopReason', () => {
   const derive = claudeCodeRunner.deriveStopReason!;