diff --git a/README.md b/README.md index 31527982..0e815e96 100644 --- a/README.md +++ b/README.md @@ -143,10 +143,10 @@ Skills come from [`supabase/agent-skills`](https://github.com/supabase/agent-ski To use a skill in an experiment, reference its directory name in the experiment's `skills` array. -Both runtimes load skills lazily ([progressive disclosure](https://ai-sdk.dev/cookbook/guides/agent-skills)): only each skill's name+description is in the system prompt, and the agent pulls a skill's full instructions on demand. They differ only in how the body is fetched, because the tools-mode agent has no filesystem: +Skills are always loaded lazily ([progressive disclosure](https://ai-sdk.dev/cookbook/guides/agent-skills)) — a skill's full instructions are pulled on demand, never preloaded. How that happens depends on the harness: -- **Local-stack (sandbox) mode:** skills are installed into the workspace with [Vercel's `skills` CLI](https://github.com/vercel-labs/skills) (baked into the sandbox image, sourced from the local `skills/` directory — never the network) under `.claude/skills/`. When a task matches, the agent reads `.claude/skills//SKILL.md` (and any files it references) with its file tools. -- **Tools mode:** no filesystem, so a `load_skill` tool returns a skill's full instructions when the agent calls it with the skill's name. +- **CLI harnesses (Claude Code, Codex, OpenCode)** use their own built-in skills mechanism. Skills are installed into the sandbox workspace with [Vercel's `skills` CLI](https://github.com/vercel-labs/skills) (baked into the sandbox image, sourced from the local `skills/` directory — never the network), for each harness's own project scope: `.claude/skills/` for Claude Code, `.agents/skills/` for Codex and OpenCode. Each CLI then discovers, advertises and loads the skills itself. The framework injects nothing — an agent's real-world skill-following behaviour is part of what an eval measures. +- **The in-process `ai-sdk` harness** has no such mechanism, so the framework supplies one. In local-stack mode it lists each skill's name+description in the system prompt and the agent reads `.claude/skills//SKILL.md` with its file tools. In tools mode there is no filesystem at all, so a `load_skill` tool returns a skill's full instructions when the agent calls it with the skill's name. ## Framework Checks diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index f2a00641..d1bbbec7 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -29,6 +29,7 @@ import { } from '../lib/cli-args.js'; import { bootPlatformBackend } from './platform-backend.js'; import { viteBuild, vitestRun } from './project-runner.js'; +import { buildSystemPrompt } from './system-prompt.js'; import { buildDocsResult, buildSkillResult, @@ -237,9 +238,9 @@ function buildLoadSkillTool(skills: readonly ToolsSkill[]): ToolSet { } /** - * Local-stack skill sources: resolve each skill name to its host directory so - * the sandbox can install it with Vercel's `skills` CLI; the agent then - * discovers each skill by reading its SKILL.md with its file tools. The + * Sandbox skill sources: resolve each skill name to its host directory so the + * sandbox can install it with Vercel's `skills` CLI, which places it in every + * CLI harness's native project scope for that harness to discover. The * `skills/` entries are symlinks into the agent-skills submodule; realpath them * so `docker cp` copies real files, not dangling links. Missing skills are * skipped with a warning. @@ -298,31 +299,6 @@ function readSessionSeedArgs(ev: EvalManifest) { }; } -function basePromptFor(mode: EvalMode): string { - if (mode === 'local-stack') { - return ( - 'You are an agent solving a Supabase eval task in a Linux workspace. ' + - 'Use the provided tools to inspect and modify the workspace and run commands. ' + - 'When you are done, end your turn with a short summary of what you did.' - ); - } - return ( - 'You are an agent solving a Supabase eval task. ' + - 'Use the provided tools to inspect and modify the project. ' + - 'When you are done, end your turn with a short summary of what you did ' + - '(or for audit tasks, your findings).' - ); -} - -function buildSystemPrompt( - mode: EvalMode, - addendum?: string, - skillContext?: string -): string { - const blocks = [basePromptFor(mode), addendum, skillContext].filter(Boolean); - return blocks.join('\n\n'); -} - /** * Adapt a `{ close() }` resource to `AsyncDisposable` so it can be bound with * `await using` — cleanup then runs on scope exit (normal fall-through, `continue`, @@ -352,6 +328,13 @@ async function runOne( transcript: TranscriptPart[]; agentReport: string; stoppedReason: string; + /** + * The exact system prompt handed to the agent (`''` when it got none). CLI + * harnesses receive theirs as a file in the sandbox scratch dir, outside the + * exported workspace, so recording it here is the only way to verify from a + * run artifact what the agent was actually told. + */ + systemPrompt: string; } > { const prompt = parseEvalMarkdown( @@ -383,6 +366,7 @@ async function runOne( let lastTranscript: TranscriptPart[] = []; let lastAgentReport = ''; let lastStoppedReason = 'not_started'; + let lastSystemPrompt = ''; for (let attempt = 1; attempt <= RUNS; attempt += 1) { if (ev.mode === 'local-stack') { @@ -415,6 +399,7 @@ async function runOne( : undefined; await using session = disposable( await exp.localStack.startSession({ + agent: exp.agent.id, cliVersion: ev.metadata.cliVersion, localDir: ev.localDir, includeServices: ev.metadata.services, @@ -435,8 +420,13 @@ async function runOne( }) ); + const systemPrompt = buildSystemPrompt( + exp.agent.id, + 'local-stack', + session.promptAddendum + ); const run = await exp.agent.run({ - systemPrompt: buildSystemPrompt('local-stack', session.promptAddendum), + systemPrompt, userPrompt: prompt, tools: session.tools, sandbox: session.sandbox, @@ -447,6 +437,7 @@ async function runOne( lastTranscript = run.transcript; lastAgentReport = run.agentReport; lastStoppedReason = run.stoppedReason; + lastSystemPrompt = systemPrompt; // Export the agent's workspace to the host so scorers can run host // tooling (vite/vitest from the repo root) against the produced files @@ -488,6 +479,7 @@ async function runOne( transcript: run.transcript, agentReport: run.agentReport, stoppedReason: run.stoppedReason, + systemPrompt, }; } logRetryAttempt(expName, ev, attempt, last); @@ -500,7 +492,12 @@ async function runOne( // platform-lite via host.docker.internal (so platform-lite binds 0.0.0.0). // An in-process agent runs host-side with no sandbox. await using cliSandbox = agentRunsInSandbox - ? disposable(await createBareSandbox({ skills: skillSources })) + ? disposable( + await createBareSandbox({ + agent: exp.agent.id, + skills: skillSources, + }) + ) : undefined; await using session = disposable( await exp.runtime.startSession({ @@ -509,14 +506,16 @@ async function runOne( }) ); - // CLI agents read their installed skills from disk (the bare sandbox folds - // the discovery listing into its promptAddendum). In-process agents have - // no filesystem, so their skills are advertised in the prompt and pulled - // on demand via the load_skill tool. + // CLI agents discover their installed skills themselves — the skills CLI + // put them in every harness's native project scope, so each one advertises + // and loads them in its own words and the bare sandbox contributes nothing + // here. In-process agents have no filesystem, so their skills are advertised + // in the prompt and pulled on demand via the load_skill tool. const skillsPrompt = agentRunsInSandbox ? cliSandbox!.promptAddendum : buildToolsSkillsPrompt(toolsSkills); const systemPrompt = buildSystemPrompt( + exp.agent.id, 'tools', session.promptAddendum, skillsPrompt @@ -533,6 +532,7 @@ async function runOne( lastTranscript = run.transcript; lastAgentReport = run.agentReport; lastStoppedReason = run.stoppedReason; + lastSystemPrompt = systemPrompt; last = await (scorer as ToolScorer)({ ...session.scoringContext, toolCalls: run.toolCalls, @@ -554,6 +554,7 @@ async function runOne( transcript: run.transcript, agentReport: run.agentReport, stoppedReason: run.stoppedReason, + systemPrompt, }; } logRetryAttempt(expName, ev, attempt, last); @@ -568,6 +569,7 @@ async function runOne( transcript: lastTranscript, agentReport: lastAgentReport, stoppedReason: lastStoppedReason, + systemPrompt: lastSystemPrompt, }; } diff --git a/apps/framework/harness/system-prompt.test.ts b/apps/framework/harness/system-prompt.test.ts new file mode 100644 index 00000000..b1453678 --- /dev/null +++ b/apps/framework/harness/system-prompt.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from 'vitest'; +import type { AgentHarnessId } from '@supabase-evals/core'; +import { + buildSkillsPrompt, + buildToolSurfaceAddendum, + type SkillEntry, +} from '@supabase-evals/sandbox'; +import { buildSystemPrompt } from './system-prompt.js'; +import type { EvalMode } from './types.js'; + +const CLI_AGENTS: AgentHarnessId[] = ['claude-code', 'codex', 'opencode']; +const MODES: EvalMode[] = ['tools', 'local-stack']; + +describe('buildSystemPrompt', () => { + it('gives the ai-sdk agent task framing in both modes', () => { + // ai-sdk is the one harness with no system prompt of its own, so it's the + // one harness the framework has to supply one for. + for (const mode of MODES) { + expect(buildSystemPrompt('ai-sdk', mode)).toContain( + 'Use the provided tools' + ); + } + }); + + it('gives no framing of our own to any CLI harness', () => { + // CLI harnesses ship their own system prompt; we're measuring that. + for (const agent of CLI_AGENTS) { + for (const mode of MODES) { + expect(buildSystemPrompt(agent, mode)).toBe(''); + } + } + }); + + it('passes a CLI harness only the runtime blocks, with no base prompt', () => { + for (const agent of CLI_AGENTS) { + expect( + buildSystemPrompt(agent, 'local-stack', 'Addendum.', 'Skills listing.') + ).toBe('Addendum.\n\nSkills listing.'); + } + }); + + it('assembles to nothing at all for a CLI harness, even with skills', () => { + // The real block producers, not stand-ins: with skills installed, a CLI + // harness must still receive an entirely empty system prompt. Codex and + // OpenCode find the skills through their own project-scope discovery and + // describe them to the model themselves. + const skills: SkillEntry[] = [ + { + name: 'supabase', + description: 'Use for Supabase tasks.', + dir: '.claude/skills/supabase', + }, + ]; + for (const agent of CLI_AGENTS) { + expect( + buildSystemPrompt( + agent, + 'local-stack', + buildToolSurfaceAddendum(agent), + buildSkillsPrompt(agent, skills) + ) + ).toBe(''); + } + // ai-sdk has no such mechanism — it only learns about skills from us. + const aiSdk = buildSystemPrompt( + 'ai-sdk', + 'local-stack', + buildToolSurfaceAddendum('ai-sdk'), + buildSkillsPrompt('ai-sdk', skills) + ); + expect(aiSdk).toContain('## Available skills'); + expect(aiSdk).toContain('- supabase: Use for Supabase tasks.'); + }); + + it('never tells any agent how to end its turn', () => { + // Stopping behaviour is part of what an eval measures, so the harness must + // not coach it (e.g. "end your turn with a short summary"). + for (const agent of [...CLI_AGENTS, 'ai-sdk' as const]) { + for (const mode of MODES) { + const prompt = buildSystemPrompt(agent, mode); + expect(prompt).not.toMatch(/summary/i); + expect(prompt).not.toMatch(/end your turn/i); + } + } + }); + + it('drops empty blocks instead of leaving blank gaps', () => { + expect( + buildSystemPrompt('claude-code', 'tools', '', 'Skills listing.') + ).toBe('Skills listing.'); + expect(buildSystemPrompt('ai-sdk', 'tools', '', '')).not.toMatch(/\n\n$/); + }); +}); diff --git a/apps/framework/harness/system-prompt.ts b/apps/framework/harness/system-prompt.ts new file mode 100644 index 00000000..d042c57e --- /dev/null +++ b/apps/framework/harness/system-prompt.ts @@ -0,0 +1,52 @@ +/** + * System-prompt assembly, per agent harness. + * + * An eval measures out-of-the-box agent behaviour, so the harness injects as + * little prompt of its own as it can get away with: only the ai-sdk agent gets + * any base framing, because it is the only harness with no system prompt of its + * own (`aiSdkAgent` hands `systemPrompt` straight to the model's `system`). CLI + * agents ship their own coding-agent prompt, tool guidance, and stopping + * behaviour — and codex/opencode have no system-prompt flag at all, so anything + * we pass them lands on the *user* prompt. + */ + +import type { AgentHarnessId } from '@supabase-evals/core'; +import type { EvalMode } from './types.js'; + +/** + * Base framing for the ai-sdk harness: what it can't infer on its own — that it + * has tools, and what they act on. Deliberately silent on how to finish a turn + * (no "end with a summary"): stopping behaviour is part of what's measured. + * Empty for every CLI harness. + */ +function basePromptFor(agent: AgentHarnessId, mode: EvalMode): string { + if (agent !== 'ai-sdk') return ''; + if (mode === 'local-stack') { + return ( + 'You are an agent solving a Supabase eval task in a Linux workspace. ' + + 'Use the provided tools to inspect and modify the workspace and run commands.' + ); + } + return ( + 'You are an agent solving a Supabase eval task. ' + + 'Use the provided tools to inspect and modify the project.' + ); +} + +/** + * Assemble the system prompt handed to the agent. Every block is optional, and + * every one of them is ai-sdk-only (the base framing, the tool-surface addendum, + * the skills listing), so a CLI harness ends up with `''` — the CLI engine then + * stages no system-prompt file at all rather than an empty one. + */ +export function buildSystemPrompt( + agent: AgentHarnessId, + mode: EvalMode, + addendum?: string, + skillContext?: string +): string { + const blocks = [basePromptFor(agent, mode), addendum, skillContext].filter( + Boolean + ); + return blocks.join('\n\n'); +} diff --git a/apps/framework/package.json b/apps/framework/package.json index 5b282780..a78e9990 100644 --- a/apps/framework/package.json +++ b/apps/framework/package.json @@ -4,12 +4,13 @@ "version": "0.0.1", "type": "module", "scripts": { - "check": "pnpm typecheck && pnpm test:framework && pnpm test:vercel-runner", + "check": "pnpm typecheck && pnpm test && pnpm test:framework && pnpm test:vercel-runner", "eval": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts", "eval:dry": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --dry", "eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke", "eval:vercel": "node --env-file=../../.env --import tsx/esm scripts/run-vercel-evals.ts", "typecheck": "tsc --noEmit", + "test": "vitest run harness", "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts", "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts lib/cli-args.test.ts", "export-results": "node --import tsx/esm scripts/export-results.ts", diff --git a/packages/core/src/agents/claude-code/runner.test.ts b/packages/core/src/agents/claude-code/runner.test.ts index 07450363..28a42576 100644 --- a/packages/core/src/agents/claude-code/runner.test.ts +++ b/packages/core/src/agents/claude-code/runner.test.ts @@ -36,6 +36,46 @@ function streamJson(subtype: string, isError = false): string { ].join('\n'); } +/** The `claude` invocation from one exec, with a fake sandbox. */ +async function captureRunCommand( + systemPromptPath: string | undefined +): Promise { + let runCommand = ''; + await claudeCodeRunner.exec({ + sandbox: { + workspace: '/w', + exec: async (cmd) => { + if (cmd.includes('/bin/claude')) runCommand = cmd; + return ok; + }, + readFile: async () => '', + }, + model: 'claude-sonnet-4-6', + apiKey: 'k', + systemPromptPath, + userPromptPath: '"$HOME/.eval/user-prompt.txt"', + mcpServers: {}, + timeoutSec: 1, + }); + return runCommand; +} + +describe('claudeCodeRunner.exec', () => { + it('appends the harness system prompt when there is one', async () => { + const command = await captureRunCommand('"$HOME/.eval/system-prompt.txt"'); + expect(command).toContain( + '--append-system-prompt-file "$HOME/.eval/system-prompt.txt"' + ); + }); + + it("omits the flag with no system prompt, leaving Claude Code's own intact", async () => { + const command = await captureRunCommand(undefined); + expect(command).not.toContain('--append-system-prompt-file'); + // The task itself is still piped in. + expect(command).toContain('cat "$HOME/.eval/user-prompt.txt"'); + }); +}); + describe('claudeCodeRunner.deriveStopReason', () => { const derive = claudeCodeRunner.deriveStopReason!; diff --git a/packages/core/src/agents/claude-code/runner.ts b/packages/core/src/agents/claude-code/runner.ts index 37129ca1..84bc1297 100644 --- a/packages/core/src/agents/claude-code/runner.ts +++ b/packages/core/src/agents/claude-code/runner.ts @@ -70,7 +70,10 @@ export const claudeCodeRunner: AgentRunner = { ...(reasoningEffort ? [`--effort ${shellQuote(reasoningEffort)}`] : []), // Append (not replace), from a file (no ARG_MAX/shell-expansion surface), // so Claude Code keeps its default coding-agent prompt + tool guidance. - `--append-system-prompt-file ${systemPromptPath}`, + // Omitted when there's nothing to append, leaving that default untouched. + ...(systemPromptPath + ? [`--append-system-prompt-file ${systemPromptPath}`] + : []), ...mcpFlags, // The sandbox is the isolation boundary, so skip permission prompts and // give the agent its full native toolset (same in both modes). diff --git a/packages/core/src/agents/codex/runner.test.ts b/packages/core/src/agents/codex/runner.test.ts new file mode 100644 index 00000000..ba2c5adc --- /dev/null +++ b/packages/core/src/agents/codex/runner.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest'; +import type { CommandResult } from '../../index.js'; +import { codexRunner } from './runner.js'; + +const ok: CommandResult = { ok: true, exitCode: 0, stdout: '', stderr: '' }; + +/** The `codex exec` invocation from one exec, with a fake sandbox. */ +async function captureRunCommand( + systemPromptPath: string | undefined +): Promise { + let runCommand = ''; + await codexRunner.exec({ + sandbox: { + workspace: '/w', + exec: async (cmd) => { + if (cmd.includes(' exec ')) runCommand = cmd; + return ok; + }, + readFile: async () => '', + }, + model: 'gpt-5.4', + apiKey: 'k', + systemPromptPath, + userPromptPath: '"$HOME/.eval/user-prompt.txt"', + mcpServers: {}, + timeoutSec: 1, + }); + return runCommand; +} + +describe('codexRunner.exec', () => { + it('prepends the harness system prompt to the task when there is one', async () => { + // Codex has no system-prompt flag, so it lands on the user prompt. + const command = await captureRunCommand('"$HOME/.eval/system-prompt.txt"'); + expect(command).toContain( + `{ cat "$HOME/.eval/system-prompt.txt"; printf '\\n\\n'; cat "$HOME/.eval/user-prompt.txt"; }` + ); + }); + + it('sends the task alone with no system prompt (no leading blank block)', async () => { + const command = await captureRunCommand(undefined); + expect(command).not.toContain('system-prompt'); + expect(command).not.toContain("printf '\\n\\n'"); + expect(command.startsWith('cat "$HOME/.eval/user-prompt.txt" |')).toBe( + true + ); + }); +}); diff --git a/packages/core/src/agents/codex/runner.ts b/packages/core/src/agents/codex/runner.ts index f4e3837e..31ebd626 100644 --- a/packages/core/src/agents/codex/runner.ts +++ b/packages/core/src/agents/codex/runner.ts @@ -92,11 +92,15 @@ export const codexRunner: AgentRunner = { ].join(' '); // Codex has no system-prompt flag; prepend the system prompt to the task, - // both staged as files, fed on stdin. - const command = await sandbox.exec( - `{ cat ${systemPromptPath}; printf '\\n\\n'; cat ${userPromptPath}; } | ${codex} ${flags}`, - { timeoutMs: timeoutSec * 1000, env: { OPENAI_API_KEY: apiKey } } - ); + // both staged as files, fed on stdin. With no system prompt the task goes in + // alone — concatenating an empty one would open the prompt with a blank block. + const stdin = systemPromptPath + ? `{ cat ${systemPromptPath}; printf '\\n\\n'; cat ${userPromptPath}; }` + : `cat ${userPromptPath}`; + const command = await sandbox.exec(`${stdin} | ${codex} ${flags}`, { + timeoutMs: timeoutSec * 1000, + env: { OPENAI_API_KEY: apiKey }, + }); return { command, raw: command.stdout }; }, diff --git a/packages/core/src/agents/engine.test.ts b/packages/core/src/agents/engine.test.ts new file mode 100644 index 00000000..5b4a4c23 --- /dev/null +++ b/packages/core/src/agents/engine.test.ts @@ -0,0 +1,77 @@ +import { beforeEach, describe, expect, it } from 'vitest'; +import type { CommandResult } from '../index.js'; +import type { AgentTranscriptParser } from '../parsers/types.js'; +import { createCliAgent } from './engine.js'; +import { SYSTEM_PROMPT_PATH } from './shared.js'; +import type { AgentRunner } from './types.js'; + +const ok: CommandResult = { ok: true, exitCode: 0, stdout: '', stderr: '' }; + +const API_KEY_ENV_VAR = 'ENGINE_TEST_API_KEY'; + +/** A parser that reports one assistant message, so the engine stays quiet. */ +const parser: AgentTranscriptParser = { + parseTranscript: () => ({ + events: [{ type: 'message', role: 'assistant', content: 'done' }], + }), +}; + +/** + * Run a CLI agent against a fake sandbox, returning the `systemPromptPath` its + * runner was handed plus every command the engine ran in the sandbox. + */ +async function runWithSystemPrompt(systemPrompt: string): Promise<{ + systemPromptPath: string | undefined; + commands: string[]; +}> { + const commands: string[] = []; + let systemPromptPath: string | undefined; + const runner: AgentRunner = { + id: 'claude-code', + displayName: 'Fake CLI', + apiKeyEnvVar: API_KEY_ENV_VAR, + cliPackage: 'fake-cli', + defaultCliVersion: '1.0.0', + defaultModel: 'fake-model', + install: async () => undefined, + exec: async (args) => { + systemPromptPath = args.systemPromptPath; + return { command: ok, raw: '' }; + }, + }; + await createCliAgent(runner, parser, { model: 'fake-model' }).run({ + systemPrompt, + userPrompt: 'the task', + timeoutSec: 1, + sandbox: { + workspace: '/w', + exec: async (command) => { + commands.push(command); + return ok; + }, + readFile: async () => '', + }, + }); + return { systemPromptPath, commands }; +} + +describe('createCliAgent prompt staging', () => { + // The engine requires the runner's API key before it stages anything. + beforeEach(() => { + process.env[API_KEY_ENV_VAR] = 'k'; + }); + + it('stages a system prompt and hands its path to the runner', async () => { + const { systemPromptPath, commands } = await runWithSystemPrompt('Skills.'); + expect(systemPromptPath).toBe(SYSTEM_PROMPT_PATH); + expect(commands.some((c) => c.includes(SYSTEM_PROMPT_PATH))).toBe(true); + }); + + it('stages no file at all when the harness has no system prompt', async () => { + // The runner then omits its system-prompt plumbing, leaving the CLI's own + // prompt untouched instead of pointing it at an empty file. + const { systemPromptPath, commands } = await runWithSystemPrompt(''); + expect(systemPromptPath).toBeUndefined(); + expect(commands.some((c) => c.includes(SYSTEM_PROMPT_PATH))).toBe(false); + }); +}); diff --git a/packages/core/src/agents/engine.ts b/packages/core/src/agents/engine.ts index 41035a46..9749b91a 100644 --- a/packages/core/src/agents/engine.ts +++ b/packages/core/src/agents/engine.ts @@ -89,15 +89,25 @@ export function createCliAgent( await runner.install(sandbox, version, apiKey); // Stage the prompts into the sandbox scratch dir (outside the workspace). + // An empty system prompt is staged as no file at all — a CLI agent brings + // its own system prompt, and the harness only adds one when it has + // something real to say (e.g. an installed-skills listing). Runners then + // skip their system-prompt plumbing entirely rather than pointing a flag + // at an empty file or prepending a blank block to the user prompt. await sandbox.exec(`mkdir -p ${SCRATCH}`); - await writeSandboxFile(sandbox, SYSTEM_PROMPT_PATH, args.systemPrompt); + const systemPromptPath = args.systemPrompt + ? SYSTEM_PROMPT_PATH + : undefined; + if (systemPromptPath) { + await writeSandboxFile(sandbox, systemPromptPath, args.systemPrompt); + } await writeSandboxFile(sandbox, USER_PROMPT_PATH, args.userPrompt); const { command, raw } = await runner.exec({ sandbox, model: options.model, apiKey, - systemPromptPath: SYSTEM_PROMPT_PATH, + systemPromptPath, userPromptPath: USER_PROMPT_PATH, // Rewrite loopback hosts so in-container MCP servers can reach host-side // platform-lite; the runner writes them in its own config format. diff --git a/packages/core/src/agents/opencode/runner.test.ts b/packages/core/src/agents/opencode/runner.test.ts index 13a5b3af..315b1b7b 100644 --- a/packages/core/src/agents/opencode/runner.test.ts +++ b/packages/core/src/agents/opencode/runner.test.ts @@ -86,7 +86,7 @@ describe('opencode runner', () => { /** Capture the `--model` flag, run env, and written config from one exec. */ async function captureExec( model: string, - opts: { mcp?: boolean } = {} + opts: { mcp?: boolean; systemPrompt?: boolean } = {} ): Promise<{ runCommand: string; runEnv: Record | undefined; @@ -113,7 +113,7 @@ async function captureExec( }, model, apiKey: 'gw-key', - systemPromptPath: '/s', + systemPromptPath: opts.systemPrompt === false ? undefined : '/s', userPromptPath: '/u', mcpServers: opts.mcp ? { supabase: { command: 'srv' } } : {}, timeoutSec: 1, @@ -139,4 +139,18 @@ describe('opencode runner exec routing', () => { expect(config?.mcp).toEqual({}); expect(runCommand).toContain('OPENCODE_CONFIG='); }); + + it('prepends the harness system prompt to the message when there is one', async () => { + // opencode has no system-prompt flag, so it lands on the user message. + const { runCommand } = await captureExec('moonshotai/kimi-k3'); + expect(runCommand).toContain(`"$(cat /s; printf '\\n\\n'; cat /u)"`); + }); + + it('sends the task alone with no system prompt (no leading blank block)', async () => { + const { runCommand } = await captureExec('moonshotai/kimi-k3', { + systemPrompt: false, + }); + expect(runCommand).toContain('"$(cat /u)"'); + expect(runCommand).not.toContain("printf '\\n\\n'"); + }); }); diff --git a/packages/core/src/agents/opencode/runner.ts b/packages/core/src/agents/opencode/runner.ts index b7b437d9..051929bc 100644 --- a/packages/core/src/agents/opencode/runner.ts +++ b/packages/core/src/agents/opencode/runner.ts @@ -122,8 +122,11 @@ export function createOpencodeRunner( // opencode has no system-prompt flag, so prepend the system prompt to the // task; both are staged files, joined via command substitution into the - // single message argument. - const message = `"$(cat ${systemPromptPath}; printf '\\n\\n'; cat ${userPromptPath})"`; + // single message argument. With no system prompt the task is the whole + // message — concatenating an empty one would open it with a blank block. + const message = systemPromptPath + ? `"$(cat ${systemPromptPath}; printf '\\n\\n'; cat ${userPromptPath})"` + : `"$(cat ${userPromptPath})"`; await sandbox.exec(`mkdir -p ${SCRATCH}`); await writeSandboxFile( diff --git a/packages/core/src/agents/types.ts b/packages/core/src/agents/types.ts index baf91962..46d97674 100644 --- a/packages/core/src/agents/types.ts +++ b/packages/core/src/agents/types.ts @@ -54,8 +54,13 @@ export interface RunnerExecArgs { sandbox: AgentSandbox; model: M; apiKey: string; - /** Shell path to a file holding the system prompt (skills + task framing). */ - systemPromptPath: string; + /** + * Shell path to a file holding the system prompt (e.g. the installed-skills + * listing). Undefined when the harness has no system prompt for this agent — + * the runner must then leave the CLI's own prompt untouched: omit the flag, or + * for a CLI with no system-prompt flag, pass the user prompt on its own. + */ + systemPromptPath?: string; /** Shell path to a file holding the user prompt (the task). */ userPromptPath: string; /** MCP servers to expose, already loopback-rewritten. Empty when none. */ diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 347b26ef..a63491f8 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -365,6 +365,11 @@ export type LocalStackScorer = ( ) => Promise; export type AgentRunArgs = { + /** + * System prompt from the harness. Empty for a CLI agent the harness has + * nothing to add to — it keeps its own built-in prompt untouched, and no + * system prompt is staged into the sandbox at all. + */ systemPrompt: string; userPrompt: string; tools?: ToolSet; @@ -410,6 +415,13 @@ export type AgentHarness = { export type SkillSource = { name: string; dir: string }; export type LocalStackSessionArgs = { + /** + * The agent harness this session serves. Only `ai-sdk` calls the session's + * in-process `tools`, so only it gets the prompt addendum describing them — + * a CLI agent brings its own tools and would be told about tools it doesn't + * have. + */ + agent: AgentHarnessId; /** Supabase CLI version this scenario requires, overriding the runtime default. */ cliVersion?: string; /** diff --git a/packages/sandbox/src/agent-environment.ts b/packages/sandbox/src/agent-environment.ts index d17bc08d..cdd60983 100644 --- a/packages/sandbox/src/agent-environment.ts +++ b/packages/sandbox/src/agent-environment.ts @@ -38,7 +38,7 @@ export interface AgentEnvironmentOptions { cliVersion?: string; /** Host directory whose contents seed the workspace. */ localDir?: string; - /** Skills to install into the sandbox (the agent reads them with its file tools). */ + /** Skills to install into the sandbox (in every CLI harness's project scope). */ skills?: readonly SkillSource[]; /** * Run the Supabase local stack. Present → local-stack mode; omitted → tools @@ -85,7 +85,8 @@ export async function createAgentEnvironment( } else if (options.localDir) { await sandbox.copyToContainer(options.localDir, sandbox.workdir); } - // Skills are installed in both modes; the agent reads SKILL.md with its file tools. + // Skills are installed in both modes, into every CLI harness's native + // project scope (see installSkills) so each agent discovers them itself. const skills = await installSkills(sandbox, options.skills ?? []); return { sandbox, skills, close: () => sandbox.stop() }; } catch (err) { diff --git a/packages/sandbox/src/bare-sandbox.ts b/packages/sandbox/src/bare-sandbox.ts index 0ea025ef..e03e7f55 100644 --- a/packages/sandbox/src/bare-sandbox.ts +++ b/packages/sandbox/src/bare-sandbox.ts @@ -1,15 +1,32 @@ -import type { AgentSandbox, SkillSource } from '@supabase-evals/core'; +import type { + AgentHarnessId, + AgentSandbox, + SkillSource, +} from '@supabase-evals/core'; import { createAgentEnvironment } from './agent-environment.js'; import { toAgentSandbox } from './local-stack-runtime.js'; import { buildSkillsPrompt } from './skills.js'; export interface BareSandboxHandle { sandbox: AgentSandbox; - /** Skills-discovery text to fold into the agent's system prompt. */ + /** + * Skills-discovery text to fold into the agent's system prompt. Empty for + * every CLI harness — each discovers the installed skills natively and + * advertises them to the model itself (see `buildSkillsPrompt`). + */ promptAddendum: string; close(): Promise; } +export interface BareSandboxOptions { + /** Harness driving this sandbox; decides whether skills are advertised in the prompt. */ + agent: AgentHarnessId; + /** Supabase CLI version baked into the sandbox image. */ + cliVersion?: string; + /** Skills to install into the sandbox. */ + skills?: readonly SkillSource[]; +} + /** * The agent's execution environment for tools mode: the shared agent * environment (image, tooling, skills) **without** the Supabase local stack. @@ -20,7 +37,7 @@ export interface BareSandboxHandle { * platform-lite via `host.docker.internal` on the default bridge). */ export async function createBareSandbox( - options: { cliVersion?: string; skills?: readonly SkillSource[] } = {} + options: BareSandboxOptions ): Promise { const env = await createAgentEnvironment({ cliVersion: options.cliVersion, @@ -28,7 +45,7 @@ export async function createBareSandbox( }); return { sandbox: toAgentSandbox(env.sandbox), - promptAddendum: buildSkillsPrompt(env.skills), + promptAddendum: buildSkillsPrompt(options.agent, env.skills), close: env.close, }; } diff --git a/packages/sandbox/src/index.ts b/packages/sandbox/src/index.ts index cf486449..793a5e5e 100644 --- a/packages/sandbox/src/index.ts +++ b/packages/sandbox/src/index.ts @@ -19,13 +19,17 @@ export type { SetupSupabaseSandboxOptions } from './supabase.js'; export { buildLocalStackScoringContext, buildLocalStackTools, + buildToolSurfaceAddendum, localStackRuntime, toAgentSandbox, } from './local-stack-runtime.js'; export type { LocalStackRuntimeOptions } from './local-stack-runtime.js'; export { SKILLS_CLI_VERSION, + SKILLS_INSTALL_AGENTS, SKILLS_INSTALL_DIR, + SKILLS_INSTALL_DIRS, + buildSkillsAddCommand, buildSkillsPrompt, frontmatterDescription, installSkills, @@ -33,7 +37,7 @@ export { } from './skills.js'; export type { SkillEntry } from './skills.js'; export { createBareSandbox } from './bare-sandbox.js'; -export type { BareSandboxHandle } from './bare-sandbox.js'; +export type { BareSandboxHandle, BareSandboxOptions } from './bare-sandbox.js'; export { createAgentEnvironment } from './agent-environment.js'; export type { AgentEnvironment, diff --git a/packages/sandbox/src/local-stack-runtime.ts b/packages/sandbox/src/local-stack-runtime.ts index 67243bf3..40428abf 100644 --- a/packages/sandbox/src/local-stack-runtime.ts +++ b/packages/sandbox/src/local-stack-runtime.ts @@ -3,6 +3,7 @@ import { jsonSchema, tool, type ToolSet } from 'ai'; import { createClient } from '@supabase/supabase-js'; import { supabaseMcpServer, + type AgentHarnessId, type AgentSandbox, type HostedLink, type LocalStackRuntime, @@ -73,6 +74,7 @@ export function localStackRuntime( return { id: 'local-stack', async startSession({ + agent, cliVersion, localDir, includeServices, @@ -103,22 +105,14 @@ export function localStackRuntime( const mcpServers = await resolveMcpServers(options, hosted); - let baseAddendum = - 'docker, psql, git, and curl are installed in the workspace. ' + - 'Use the bash tool to run commands (the working directory is always the workspace root) ' + - 'and the files tools to inspect and modify files.'; - - if (!skipCliInstall) { - baseAddendum = 'The Supabase CLI (`supabase`), ' + baseAddendum; - baseAddendum += - ' Services started with `supabase start` are reachable on their default 127.0.0.1 ports.'; - } - return { tools: buildLocalStackTools(sandbox), sandbox: toAgentSandbox(sandbox), mcpServers, - promptAddendum: [baseAddendum, buildSkillsPrompt(env.skills)] + promptAddendum: [ + buildToolSurfaceAddendum(agent, { skipCliInstall }), + buildSkillsPrompt(agent, env.skills), + ] .filter(Boolean) .join('\n\n'), scoringContext: buildLocalStackScoringContext(sandbox, hosted), @@ -133,6 +127,33 @@ export function localStackRuntime( }; } +/** + * Describes the session's tool surface: the binaries installed in the workspace + * and the in-process `bash`/`files_*` tools from `buildLocalStackTools`. + * + * ai-sdk only. Those tools exist solely for `aiSdkAgent`, which the framework + * drives host-side; `createCliAgent` ignores `args.tools` entirely, so a CLI + * agent works the same workspace through its own built-in tools and this text + * would name tools it does not have. Empty string for every CLI agent. + */ +export function buildToolSurfaceAddendum( + agent: AgentHarnessId, + options: { skipCliInstall?: boolean } = {} +): string { + if (agent !== 'ai-sdk') return ''; + let addendum = + 'docker, psql, git, and curl are installed in the workspace. ' + + 'Use the bash tool to run commands (the working directory is always the workspace root) ' + + 'and the files tools to inspect and modify files.'; + + if (!options.skipCliInstall) { + addendum = 'The Supabase CLI (`supabase`), ' + addendum; + addendum += + ' Services started with `supabase start` are reachable on their default 127.0.0.1 ports.'; + } + return addendum; +} + /** * Build the MCP server map for a session. An explicit `options.mcpServers` * wins. Otherwise, when the eval links to a hosted project, expose a Supabase diff --git a/packages/sandbox/src/skills.ts b/packages/sandbox/src/skills.ts index 37f48248..f0ae34e1 100644 --- a/packages/sandbox/src/skills.ts +++ b/packages/sandbox/src/skills.ts @@ -1,34 +1,65 @@ /** - * Agent skills inside the local-stack sandbox. + * Agent skills inside the sandbox. * * Skills are reusable instruction sets (a SKILL.md plus bundled reference - * files) that the agent discovers and loads on demand — the AI SDK - * "agent skills" pattern (https://ai-sdk.dev/cookbook/guides/agent-skills). - * Rather than preloading every skill's full text into the system prompt, the - * sandbox advertises only each skill's name+description and tells the agent to - * read a skill's SKILL.md (with the existing file tools) when a task matches — - * progressive disclosure. This only works where the agent has a filesystem — - * the sandbox. Tools-mode evals (no filesystem) inject skills into the system - * prompt instead. + * files) that the agent discovers and loads on demand — progressive disclosure, + * rather than preloading every skill's full text into the system prompt. * * Skills are installed with Vercel's `skills` CLI (baked into the sandbox * image), sourcing from local directories — never the network. Each requested - * skill is staged outside the workspace, then `skills add` copies it into the - * workspace's `.claude/skills/` (claude-code project scope), where the agent's - * file tools can reach the SKILL.md and the files it references. + * skill is staged outside the workspace, then `skills add --agent …` copies it + * into every project scope the CLI harnesses discover natively: + * + * - `.claude/skills/` — Claude Code's project scope. + * - `.agents/skills/` — Codex's and OpenCode's project scope. + * + * Each CLI then advertises its own skills to the model, in its own words, with + * its own loading mechanism (Codex injects a `` block, + * OpenCode exposes a `skill` tool). The harness injects nothing: a synthetic + * "read this file with this tool" listing would both duplicate and contradict + * what the agent's own harness tells it. The one exception is the in-process + * `ai-sdk` agent, which has no such mechanism — `buildSkillsPrompt` renders the + * listing for it alone. */ -import type { SkillSource } from '@supabase-evals/core'; +import type { AgentHarnessId, SkillSource } from '@supabase-evals/core'; import type { DockerSandbox } from './docker-sandbox.js'; /** Version of Vercel's `skills` CLI baked into the sandbox image (pinned). */ export const SKILLS_CLI_VERSION = '1.5.11'; /** - * Where installed project-scoped skills are read from, relative to the - * workspace root (the CLI's cwd during install). We install for all agents - * (see installSkills), and `.claude/skills` is claude-code's project scope — - * the discovery listing points the agent here. + * `skills add --agent` ids we install for — the three CLI harnesses that run in + * the sandbox. Installed unconditionally rather than only for the experiment's + * own harness: the ids map onto just two directories (below), an unused one + * costs a directory copy of a few kilobytes, and keeping one code path means no + * agent id has to be threaded through `createAgentEnvironment` for correctness. + * + * Naming them explicitly also matters. With no `--agent` flag the CLI falls + * back to *every* agent it knows (71 at 1.5.11) when it can't detect an + * installed one, littering the scored, exported workspace with ~53 stray + * entries — and that fallback is order-dependent, so it would silently stop + * producing `.claude/skills` if an agent CLI were ever installed first. + */ +export const SKILLS_INSTALL_AGENTS = [ + 'claude-code', + 'codex', + 'opencode', +] as const; + +/** + * Every workspace-relative directory `SKILLS_INSTALL_AGENTS` populates, deduped + * (`codex` and `opencode` share `.agents/skills`). Verified after install. + */ +export const SKILLS_INSTALL_DIRS = [ + '.claude/skills', + '.agents/skills', +] as const; + +/** + * Claude Code's project scope, and the directory the `ai-sdk` discovery listing + * points at — that agent reads SKILL.md with the harness's own file tools, so it + * needs one concrete path. The CLI harnesses resolve their own paths. */ export const SKILLS_INSTALL_DIR = '.claude/skills'; @@ -39,7 +70,11 @@ const SKILLS_STAGING_DIR = '/tmp/skills-src'; export interface SkillEntry { name: string; description: string; - /** Workspace-relative directory of the installed skill. */ + /** + * Workspace-relative directory of the installed skill in the `.claude/skills` + * scope (`SKILLS_INSTALL_DIR`). The same tree exists under every entry of + * `SKILLS_INSTALL_DIRS`; this is the one the `ai-sdk` listing cites. + */ dir: string; } @@ -75,12 +110,22 @@ export function frontmatterDescription(markdown: string): string { } /** - * Render the discovery prompt: only names+descriptions enter the system + * Render the skills discovery listing: only names+descriptions enter the system * prompt, keeping context lean. When a task matches, the agent reads that * skill's SKILL.md with the existing file tools (progressive disclosure). - * Empty when no skills are installed. + * + * ai-sdk only. `files_read` is one of `buildLocalStackTools`' in-process tools, + * handed to the model by `aiSdkAgent` alone. Every CLI harness discovers the + * installed skills itself (see the module comment) and describes them to the + * model in its own words with its own loader, so injecting this would duplicate + * that listing and name a tool the agent does not have. Empty string for every + * CLI agent, and when no skills are installed. */ -export function buildSkillsPrompt(skills: readonly SkillEntry[]): string { +export function buildSkillsPrompt( + agent: AgentHarnessId, + skills: readonly SkillEntry[] +): string { + if (agent !== 'ai-sdk') return ''; if (skills.length === 0) return ''; return [ '## Available skills', @@ -94,12 +139,34 @@ export function buildSkillsPrompt(skills: readonly SkillEntry[]): string { ].join('\n'); } +/** + * The `skills add` invocation, as a string, so its shape is unit-testable + * without a container. + * + * Argument order is load-bearing: `--agent` is variadic (it consumes every + * following token that does not start with `-`), so the source directory must + * come *before* it — `skills add --agent codex ` swallows `` as an + * agent name and fails with "Missing required argument: source". `--skill` + * immediately after the agent list terminates it. + * + * `--copy` (rather than the default symlink) keeps the workspace self-contained + * once staging is gone, and skips the CLI's symlink-mode "only install if the + * agent's top-level directory already exists" branch. `--skill '*'` installs + * every staged skill; `--yes` is non-interactive. + */ +export function buildSkillsAddCommand( + stagingDir: string = SKILLS_STAGING_DIR +): string { + return `skills add ${stagingDir} --agent ${SKILLS_INSTALL_AGENTS.join(' ')} --skill '*' --copy --yes`; +} + /** * Install agent skills into the sandbox with Vercel's `skills` CLI, sourcing * from local directories (never the network). Each skill is staged under * `/skills/` (the collection layout the CLI expects), then - * `skills add` copies it into the workspace's `.claude/skills/`. Returns the - * discovered registry (name+description+dir) used for progressive disclosure. + * `skills add` copies it into every project scope in `SKILLS_INSTALL_DIRS`, so + * each CLI harness finds it through its own native discovery. Returns the + * installed registry (name+description+dir), used for the `ai-sdk` listing. * A no-op that returns `[]` when no skills are requested. */ export async function installSkills( @@ -120,38 +187,34 @@ export async function installSkills( ); } - // `skills add ` installs to the cwd's project scope, and runShell's cwd - // is the workspace, so skills land in //. --copy (not - // symlink) keeps the workspace self-contained once staging is gone; --skill - // '*' installs all staged skills; --yes is non-interactive. - // - // TODO: install only for the agent the experiment uses (--agent claude-code, - // codex, gemini-cli, …) once that is threaded through. We only run models via - // the AI SDK today, so for now we install for all agents and read claude-code's - // .claude/skills scope (SKILLS_INSTALL_DIR). - const install = await sandbox.runShell( - `skills add ${SKILLS_STAGING_DIR} --skill '*' --copy --yes` - ); + // `skills add ` installs into the cwd's project scopes, and runShell's + // cwd is the workspace, so skills land in /{.claude,.agents}/skills. + const install = await sandbox.runShell(buildSkillsAddCommand()); if (!install.ok) { throw new Error( `failed to install skills with the skills CLI: ${install.stderr || install.stdout}` ); } - // Confirm what landed and read each skill's description for the discovery - // listing. The name is the install directory; only the description is read. + // Confirm each skill landed in *every* agent scope — a missing one means the + // harness that reads it would silently see no skills at all — then read the + // description for the ai-sdk listing. The name is the install directory. const entries: SkillEntry[] = []; for (const source of sources) { - const dir = `${SKILLS_INSTALL_DIR}/${source.name}`; - const skillPath = `${dir}/SKILL.md`; - if (!(await sandbox.fileExists(skillPath))) { - throw new Error( - `skills CLI did not install "${source.name}" (no ${skillPath} in the sandbox)` - ); + for (const installDir of SKILLS_INSTALL_DIRS) { + const skillPath = `${installDir}/${source.name}/SKILL.md`; + if (!(await sandbox.fileExists(skillPath))) { + throw new Error( + `skills CLI did not install "${source.name}" (no ${skillPath} in the sandbox)` + ); + } } + const dir = `${SKILLS_INSTALL_DIR}/${source.name}`; entries.push({ name: source.name, - description: frontmatterDescription(await sandbox.readFile(skillPath)), + description: frontmatterDescription( + await sandbox.readFile(`${dir}/SKILL.md`) + ), dir, }); } diff --git a/packages/sandbox/test/docker.test.ts b/packages/sandbox/test/docker.test.ts index 5107b2a3..779aa754 100644 --- a/packages/sandbox/test/docker.test.ts +++ b/packages/sandbox/test/docker.test.ts @@ -18,7 +18,11 @@ import { SUPABASE_CLI_VERSION, teardownSupabaseProject, } from '../src/supabase.js'; -import { installSkills, SKILLS_INSTALL_DIR } from '../src/skills.js'; +import { + installSkills, + SKILLS_INSTALL_DIR, + SKILLS_INSTALL_DIRS, +} from '../src/skills.js'; const TEST_TIMEOUT_MS = 600_000; @@ -200,18 +204,30 @@ describe.runIf(process.env.SANDBOX_DOCKER_TESTS)( dir: `${SKILLS_INSTALL_DIR}/demo-skill`, }, ]); - // The full skill tree (including bundled references) is reachable in - // the workspace via the agent's file tools. - expect( - await sandbox.fileExists( - `${SKILLS_INSTALL_DIR}/demo-skill/SKILL.md` - ) - ).toBe(true); - expect( - await sandbox.readFile( - `${SKILLS_INSTALL_DIR}/demo-skill/references/extra.md` - ) - ).toBe('extra content'); + // The full skill tree (including bundled references) lands in every + // CLI harness's native project scope: .claude/skills for Claude Code, + // .agents/skills for Codex and OpenCode. + for (const installDir of SKILLS_INSTALL_DIRS) { + expect( + await sandbox.fileExists(`${installDir}/demo-skill/SKILL.md`) + ).toBe(true); + expect( + await sandbox.readFile( + `${installDir}/demo-skill/references/extra.md` + ) + ).toBe('extra content'); + } + // …and nowhere else. Without an explicit --agent the CLI installs for + // every one of the ~71 agents it knows, littering the exported, scored + // workspace with dozens of stray roots (including non-dotted ones). + for (const stray of [ + '.aider-desk', + '.factory', + '.windsurf', + 'data', + ]) { + expect(await sandbox.folderExists(stray)).toBe(false); + } } finally { rmSync(src, { recursive: true, force: true }); await sandbox.stop(); diff --git a/packages/sandbox/test/unit.test.ts b/packages/sandbox/test/unit.test.ts index bfbef903..db7986c5 100644 --- a/packages/sandbox/test/unit.test.ts +++ b/packages/sandbox/test/unit.test.ts @@ -2,6 +2,7 @@ import { readFileSync } from 'node:fs'; import { parseEvalMarkdown } from '@supabase-evals/core/eval-markdown'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { + buildToolSurfaceAddendum, resolveSandboxPath, truncateOutput, wrapSelectAsJson, @@ -17,9 +18,13 @@ import { import type { DockerSandbox } from '../src/docker-sandbox.js'; import { SKILLS_CLI_VERSION, + SKILLS_INSTALL_AGENTS, SKILLS_INSTALL_DIR, + SKILLS_INSTALL_DIRS, + buildSkillsAddCommand, buildSkillsPrompt, frontmatterDescription, + installSkills, } from '../src/skills.js'; import { ALL_SUPABASE_SERVICES } from '../src/types.js'; @@ -83,15 +88,17 @@ describe('frontmatterDescription', () => { }); describe('buildSkillsPrompt', () => { + const entries = [ + { name: 'supabase', description: 'Use for Supabase tasks.', dir: 'x' }, + { name: 'pg', description: 'Postgres tips.', dir: 'y' }, + ]; + it('is empty when no skills are installed', () => { - expect(buildSkillsPrompt([])).toBe(''); + expect(buildSkillsPrompt('ai-sdk', [])).toBe(''); }); it('lists name+description and points at the install dir for files_read', () => { - const prompt = buildSkillsPrompt([ - { name: 'supabase', description: 'Use for Supabase tasks.', dir: 'x' }, - { name: 'pg', description: 'Postgres tips.', dir: 'y' }, - ]); + const prompt = buildSkillsPrompt('ai-sdk', entries); expect(prompt).toContain(SKILLS_INSTALL_DIR); expect(prompt).toContain('files_read'); expect(prompt).toContain('SKILL.md'); @@ -100,6 +107,137 @@ describe('buildSkillsPrompt', () => { // Discovery only — the full body must not be inlined here. expect(prompt).not.toContain('# Body'); }); + + it('is empty for every CLI agent — each discovers its own skills natively', () => { + // The skills CLI installs into .claude/skills and .agents/skills, which + // Claude Code, Codex and OpenCode each walk themselves; they then advertise + // the skills in their own words, with their own loader. Injecting our + // listing would duplicate theirs and name `files_read`, an ai-sdk-only tool. + for (const agent of ['claude-code', 'codex', 'opencode'] as const) { + expect(buildSkillsPrompt(agent, entries)).toBe(''); + expect(buildSkillsPrompt(agent, [])).toBe(''); + } + }); +}); + +describe('buildSkillsAddCommand', () => { + it('installs for all three CLI harnesses, source before the variadic --agent', () => { + const command = buildSkillsAddCommand('/tmp/staging'); + expect(command).toBe( + "skills add /tmp/staging --agent claude-code codex opencode --skill '*' --copy --yes" + ); + // --agent is variadic: it eats every following non-flag token. The source + // dir must precede it (otherwise the CLI fails with "Missing required + // argument: source") and a flag must terminate the agent list. + expect(command.indexOf('/tmp/staging')).toBeLessThan( + command.indexOf('--agent') + ); + expect(command).toMatch(/--agent (?:[a-z-]+ )+--skill/); + }); + + it('names the agents explicitly rather than letting the CLI guess', () => { + // With no --agent the CLI falls back to all ~71 agents it knows, littering + // the exported workspace; the fallback is also install-order dependent. + expect(SKILLS_INSTALL_AGENTS).toEqual(['claude-code', 'codex', 'opencode']); + expect(buildSkillsAddCommand()).toContain('--agent'); + }); +}); + +describe('installSkills', () => { + /** A DockerSandbox stub that records shell commands and fakes the install. */ + function fakeSandbox(present: readonly string[]) { + const commands: string[] = []; + return { + commands, + sandbox: { + runShellAsRoot: async (command: string) => { + commands.push(command); + return { ok: true, exitCode: 0, stdout: '', stderr: '' }; + }, + runShell: async (command: string) => { + commands.push(command); + return { ok: true, exitCode: 0, stdout: '', stderr: '' }; + }, + copyToContainer: async () => {}, + fileExists: async (path: string) => present.includes(path), + readFile: async () => '---\ndescription: Demo skill.\n---\nbody', + } as unknown as DockerSandbox, + }; + } + + const installedEverywhere = SKILLS_INSTALL_DIRS.map( + (dir) => `${dir}/demo/SKILL.md` + ); + + it('is a no-op with no skills requested', async () => { + const { sandbox, commands } = fakeSandbox([]); + expect(await installSkills(sandbox, [])).toEqual([]); + expect(commands).toEqual([]); + }); + + it('runs the per-agent install and reports the .claude/skills tree', async () => { + const { sandbox, commands } = fakeSandbox(installedEverywhere); + const entries = await installSkills(sandbox, [ + { name: 'demo', dir: '/host/demo' }, + ]); + expect(entries).toEqual([ + { + name: 'demo', + description: 'Demo skill.', + dir: `${SKILLS_INSTALL_DIR}/demo`, + }, + ]); + expect(commands).toContain(buildSkillsAddCommand()); + }); + + it('installs into both .claude/skills and .agents/skills', () => { + // .claude/skills is Claude Code's project scope; .agents/skills is Codex's + // and OpenCode's. Codex does not read .claude/skills at all. + expect(SKILLS_INSTALL_DIRS).toEqual(['.claude/skills', '.agents/skills']); + expect(SKILLS_INSTALL_DIR).toBe('.claude/skills'); + }); + + it('throws when a skill is missing from any agent scope', async () => { + for (const missing of SKILLS_INSTALL_DIRS) { + const { sandbox } = fakeSandbox( + installedEverywhere.filter((p) => !p.startsWith(`${missing}/`)) + ); + await expect( + installSkills(sandbox, [{ name: 'demo', dir: '/host/demo' }]) + ).rejects.toThrow(`no ${missing}/demo/SKILL.md`); + } + }); +}); + +describe('buildToolSurfaceAddendum', () => { + it('describes the in-process tool surface for the ai-sdk agent', () => { + const addendum = buildToolSurfaceAddendum('ai-sdk'); + // These are the tools buildLocalStackTools actually provides. + expect(addendum).toContain('bash tool'); + expect(addendum).toContain('files tools'); + expect(addendum).toContain('The Supabase CLI (`supabase`)'); + expect(addendum).toContain('supabase start'); + }); + + it('drops the CLI sentence when the agent installs the CLI itself', () => { + const addendum = buildToolSurfaceAddendum('ai-sdk', { + skipCliInstall: true, + }); + expect(addendum).not.toContain('The Supabase CLI (`supabase`)'); + expect(addendum).not.toContain('supabase start'); + expect(addendum).toContain('docker, psql, git, and curl'); + }); + + it('is empty for every CLI agent — they never see these tools', () => { + // createCliAgent ignores `args.tools`, so a CLI agent works the workspace + // with its own built-in tools; naming ours would describe tools it lacks. + for (const agent of ['claude-code', 'codex', 'opencode'] as const) { + expect(buildToolSurfaceAddendum(agent)).toBe(''); + expect(buildToolSurfaceAddendum(agent, { skipCliInstall: true })).toBe( + '' + ); + } + }); }); describe('SKILLS_CLI_VERSION', () => {