diff --git a/.gitignore b/.gitignore index 66c7a855..22398d89 100644 --- a/.gitignore +++ b/.gitignore @@ -12,5 +12,7 @@ evals/*/local/supabase/.branches/ results/*/ .sync-tmp/ + # local-dev runner (apps/framework/scripts/local.ts) /results-local/ +/.local-docs/ diff --git a/apps/framework/scripts/docs/content-api-server.ts b/apps/framework/scripts/docs/content-api-server.ts new file mode 100644 index 00000000..d54a5a94 --- /dev/null +++ b/apps/framework/scripts/docs/content-api-server.ts @@ -0,0 +1,71 @@ +// fallow-ignore-file unused-file -- loaded at runtime (spawned/injected by local-docs.ts), never statically imported +/** + * Standalone docs content GraphQL API for `search_docs`. + * + * Serves the docs app's own route handler (apps/docs/app/api/graphql/route.ts + * in a supabase/supabase checkout) over plain node:http — no Next server. + * Launched by `pnpm local docs api` with the docs checkout's tsx so the + * route's TS + tsconfig conditions resolve; DOCS_ROUTE_PATH points at the + * checkout, PORT picks the listen port. + */ +import { createServer } from 'node:http'; +import { pathToFileURL } from 'node:url'; + +const routePath = process.env.DOCS_ROUTE_PATH; +if (!routePath) { + console.error( + 'DOCS_ROUTE_PATH not set — run this through `pnpm local docs api`' + ); + process.exit(1); +} +// The docs checkout location is user-supplied at runtime; a static import +// cannot name it. +const route = await import(pathToFileURL(routePath).href); +const handlers: Record Promise> = { + GET: route.GET, + OPTIONS: route.OPTIONS, + POST: route.POST, +}; +const port = Number(process.env.PORT ?? 3001); + +createServer(async (incoming, outgoing) => { + const url = new URL( + incoming.url ?? '/', + `http://${incoming.headers.host ?? `127.0.0.1:${port}`}` + ); + const handler = handlers[incoming.method ?? '']; + if (url.pathname !== '/docs/api/graphql' || !handler) { + outgoing.writeHead(404).end(); + return; + } + + const headers = new Headers(); + for (const [name, value] of Object.entries(incoming.headers)) { + if (Array.isArray(value)) + for (const item of value) headers.append(name, item); + else if (value !== undefined) headers.set(name, value); + } + + const chunks: Buffer[] = []; + for await (const chunk of incoming) chunks.push(Buffer.from(chunk)); + const body = + incoming.method === 'GET' || incoming.method === 'HEAD' + ? undefined + : Buffer.concat(chunks).toString('utf8'); + const response = await handler( + new Request(url, { method: incoming.method, headers, body }) + ); + + outgoing.writeHead( + response.status, + Object.fromEntries(response.headers.entries()) + ); + outgoing.end(Buffer.from(await response.arrayBuffer())); + // Bind every interface, advertise loopback — same rule as platform-lite in + // tools mode (see run-eval.ts: sandboxed CLI agents run their MCP servers + // INSIDE the container and reach host-side services via + // host.docker.internal, which arrives on the host's bridge interface, not + // loopback; a 127.0.0.1-only listener refuses those connections). +}).listen(port, '0.0.0.0', () => { + console.log(`Docs content API: http://127.0.0.1:${port}/docs/api/graphql`); +}); diff --git a/apps/framework/scripts/docs/sentry-stub-loader.mjs b/apps/framework/scripts/docs/sentry-stub-loader.mjs new file mode 100644 index 00000000..154e301f --- /dev/null +++ b/apps/framework/scripts/docs/sentry-stub-loader.mjs @@ -0,0 +1,11 @@ +// fallow-ignore-file unused-file -- registered at runtime by sentry-stub-register.mjs via module.register() +// Loader-thread resolve hook: '@sentry/nextjs' -> the no-op stub. +const stubUrl = new URL('./sentry-stub.mjs', import.meta.url).href; + +// fallow-ignore-next-line unused-export -- Node loader-hook contract: the module system calls `resolve` +export async function resolve(specifier, context, next) { + if (specifier === '@sentry/nextjs') { + return { url: stubUrl, shortCircuit: true }; + } + return next(specifier, context); +} diff --git a/apps/framework/scripts/docs/sentry-stub-register.mjs b/apps/framework/scripts/docs/sentry-stub-register.mjs new file mode 100644 index 00000000..0c425d1b --- /dev/null +++ b/apps/framework/scripts/docs/sentry-stub-register.mjs @@ -0,0 +1,9 @@ +// fallow-ignore-file unused-file -- loaded at runtime (spawned/injected by local-docs.ts), never statically imported +// Registers a resolve hook that short-circuits '@sentry/nextjs' to the local +// no-op stub. Injected via NODE_OPTIONS from `pnpm local docs api`; chains with tsx's +// own hooks (ours only intercepts the one specifier). Uses module.register() +// (Node 20.6+) rather than registerHooks() (22.15+) — mise pins node "22", +// which an older 22.x install satisfies. +import { register } from 'node:module'; + +register('./sentry-stub-loader.mjs', import.meta.url); diff --git a/apps/framework/scripts/docs/sentry-stub.mjs b/apps/framework/scripts/docs/sentry-stub.mjs new file mode 100644 index 00000000..d42bb83f --- /dev/null +++ b/apps/framework/scripts/docs/sentry-stub.mjs @@ -0,0 +1,9 @@ +// fallow-ignore-file unused-file -- loaded at runtime (spawned/injected by local-docs.ts), never statically imported +// No-op @sentry/nextjs stand-in for the standalone docs content API. +// The route handler calls Sentry.captureException/flush; under plain tsx +// (outside Next's Sentry instrumentation) the real package's ESM build +// resolves without those functions and every request crashes. A local dev +// adapter has no business sending telemetry anyway. Wired up by +// sentry-stub-register.mjs (see local-docs.ts). +export const captureException = () => ''; +export const flush = async () => true; diff --git a/apps/framework/scripts/local-docs.ts b/apps/framework/scripts/local-docs.ts new file mode 100644 index 00000000..7dcb99af --- /dev/null +++ b/apps/framework/scripts/local-docs.ts @@ -0,0 +1,331 @@ +/** + * local-docs.ts — minimal local docs loop for `search_docs` evals. + * + * pnpm local docs up --docs + * pnpm local docs seed # full embed via the docs app's own pipeline (~$0.12 OpenAI; asks first) + * pnpm local docs api [--port N] # serve the content GraphQL API (foreground; keep it running) + * pnpm local docs down + * + * Then point evals at it: + * pnpm local run --content-api http://127.0.0.1:3001/docs/api/graphql + * + * Design: + * - The docs checkout is YOURS (`--docs`), cloned wherever you like — no + * submodule, no patches. Edit pages there, re-seed, re-run. + * - The supabase stack runs from a generated workdir (.local-docs/) with its + * own project id and a port block off both the evals local-stack range + * (54321+) and the docs monorepo default, so it collides with neither. + * Files are COPIED, not symlinked (Windows-safe); `up` regenerates them. + * - Minimal on purpose: full seed only. The upstream pipeline's incremental + * mode has known bugs we found while building the previous iteration + * (guide checksums never set -> guides always re-embed; a skipped source's + * still-valid rows get purged). Incremental lands here once those fixes + * land upstream in supabase/supabase. + * - Some sources need production creds (e.g. DOCS_GITHUB_APP_* for + * lint-warnings); without them the upstream pipeline fails its run. Pass + * them through the environment if you have them. + */ +import { execFileSync, spawnSync } from 'node:child_process'; +import { + cpSync, + existsSync, + mkdirSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { createInterface } from 'node:readline/promises'; +import { dirname, isAbsolute, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs } from 'node:util'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const ROOT = resolve(__dirname, '..', '..', '..'); +const OVERLAY = join(ROOT, '.local-docs'); +const PROJECT_ID = 'evals-local-docs'; +const DB_CONTAINER = `supabase_db_${PROJECT_ID}`; +// stack ports: whatever block the checkout's config declares -> 443xx (off +// the evals local-stack range 54321-9, and below the macOS ephemeral range +// 49152+, where transient outbound sockets flakily steal listen ports) +const PORT_PREFIX_TO = '443'; +const STACK_EXCLUDES = + 'realtime,storage-api,imgproxy,mailpit,postgres-meta,studio,edge-runtime,logflare,vector,supavisor'; + +const onWindows = process.platform === 'win32'; + +type DocsOptions = { docs?: string; port?: string; yes?: boolean }; + +function fail(msg: string): never { + console.error(msg); + process.exit(1); +} + +/** + * Run a command, streaming output; fails loudly on nonzero exit. + * + * `quiet` buffers instead of streaming, because the supabase CLI reports the + * stack's ANON_KEY, PUBLISHABLE_KEY, SERVICE_ROLE_KEY, SECRET_KEY, and + * JWT_SECRET on every start, plus an update-notifier nag. That buries our own + * one-line status, and the keys are not something to leave on a screen + * recording. + * + * On failure it replays stderr only. Measured against `supabase start`: the key + * report goes to stdout (3 key lines there, 0 on stderr), while stderr carries + * the diagnostics you actually want (workdir, config warnings, per-service + * status). Dropping stdout is a stream boundary rather than a pattern match, so + * there is no redaction regex to keep in step with the CLI's output shapes. + */ +function run( + cmd: string, + args: string[], + opts: { + cwd?: string; + env?: Record; + shim?: boolean; + quiet?: boolean; + } = {} +) { + const res = spawnSync(cmd, args, { + stdio: opts.quiet ? 'pipe' : 'inherit', + encoding: opts.quiet ? 'utf8' : undefined, + cwd: opts.cwd ?? ROOT, + env: opts.env ? { ...process.env, ...opts.env } : process.env, + // .cmd shims (corepack, .bin/tsx) need a shell on Windows + shell: opts.shim ? onWindows : false, + }); + if (res.status !== 0) { + if (opts.quiet && res.stderr) process.stderr.write(res.stderr); + fail(`${cmd} ${args.join(' ')} failed (exit ${res.status})`); + } +} + +/** + * Capture stdout. stderr is swallowed rather than inherited: the supabase CLI + * writes its workdir line, deprecation warnings, stopped-service list, and + * update-notifier nag there on every invocation, and this runs inside helpers + * whose own output is one line. On failure execFileSync throws with the output + * attached, so nothing is lost when it matters. + */ +function capture(cmd: string, args: string[]): string { + return execFileSync(cmd, args, { + cwd: ROOT, + maxBuffer: 1 << 24, + stdio: ['ignore', 'pipe', 'pipe'], + }).toString(); +} + +function docsPath(docs: string | undefined): string { + const marker = join(OVERLAY, 'docs-path.txt'); + let p = + docs ?? + (existsSync(marker) ? readFileSync(marker, 'utf8').trim() : undefined); + if (!p) + fail( + 'no docs checkout configured — pass --docs (git clone https://github.com/supabase/supabase)' + ); + p = isAbsolute(p) ? p : resolve(process.cwd(), p); + if (!existsSync(join(p, 'apps', 'docs'))) + fail(`not a supabase monorepo checkout (apps/docs missing): ${p}`); + return p; +} + +/** Parse `supabase status -o env` output (KEY="value" lines). */ +function stackEnv(): Record { + const out = capture('supabase', [ + 'status', + '--workdir', + OVERLAY, + '-o', + 'env', + ]); + const env: Record = {}; + for (const m of out.matchAll(/^([A-Z_]+)="(.*)"$/gm)) env[m[1]] = m[2]; + if (!env.API_URL) + fail('could not read the local stack env — is it up? (pnpm local docs up)'); + return env; +} + +function cmdUp(opts: DocsOptions) { + const docs = docsPath(opts.docs); + const src = join(docs, 'supabase'); + existsSync(join(src, 'config.toml')) || + fail(`no supabase/config.toml in the docs checkout: ${docs}`); + + // regenerate the overlay workdir: rewritten config + copied stack files + rmSync(OVERLAY, { recursive: true, force: true }); + mkdirSync(join(OVERLAY, 'supabase'), { recursive: true }); + const config = readFileSync(join(src, 'config.toml'), 'utf8') + .replace(/^project_id = ".*"$/m, `project_id = "${PROJECT_ID}"`) + .replace( + /^port = \d{3}(\d{2})$/gm, + (_, tail) => `port = ${PORT_PREFIX_TO}${tail}` + ); + writeFileSync(join(OVERLAY, 'supabase', 'config.toml'), config); + for (const f of ['migrations', 'seed.sql', 'functions', 'buckets']) { + const from = join(src, f); + if (existsSync(from)) + cpSync(from, join(OVERLAY, 'supabase', f), { + recursive: true, + dereference: true, + }); + } + writeFileSync(join(OVERLAY, 'docs-path.txt'), `${docs}\n`); + + console.log(`starting the docs stack (project ${PROJECT_ID})...`); + run('supabase', ['start', '--workdir', OVERLAY, '-x', STACK_EXCLUDES], { + quiet: true, + }); + // Upstream page migrations grant service_role no CRUD on the content + // tables; the embedder authenticates as service_role and needs it. + run( + 'docker', + [ + 'exec', + DB_CONTAINER, + 'psql', + '-U', + 'postgres', + '-d', + 'postgres', + '-q', + '-c', + 'GRANT ALL ON public.page, public.page_section TO service_role; GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO service_role; GRANT SELECT ON public.page, public.page_section TO anon, authenticated;', + ], + { quiet: true } + ); + console.log( + `docs stack up on ${stackEnv().API_URL}; next: pnpm local docs seed` + ); +} + +async function cmdSeed(opts: DocsOptions) { + const docs = docsPath(opts.docs); + if (!process.env.OPENAI_API_KEY) + fail('OPENAI_API_KEY not set — add it to .env at the repo root'); + const docsApp = join(docs, 'apps', 'docs'); + if (!existsSync(join(docsApp, 'node_modules'))) { + fail( + `docs app dependencies not installed — run:\n corepack pnpm --dir ${docs} install --filter ./apps/docs...` + ); + } + const env = stackEnv(); + if (!opts.yes && !process.env.LOCAL_DOCS_YES) { + const rl = createInterface({ + input: process.stdin, + output: process.stdout, + }); + const answer = await rl.question( + "Full docs embed: ~1.2M tokens ≈ $0.12 OpenAI. Type 'seed' to proceed: " + ); + rl.close(); + if (answer !== 'seed') fail('cancelled.'); + } + run('corepack', ['pnpm', 'run', 'embeddings:refresh'], { + cwd: docsApp, + shim: true, + env: { + NEXT_PUBLIC_SUPABASE_URL: env.API_URL, + NEXT_PUBLIC_SUPABASE_ANON_KEY: env.PUBLISHABLE_KEY ?? env.ANON_KEY, + SUPABASE_SECRET_KEY: env.SECRET_KEY ?? env.SERVICE_ROLE_KEY, + // generate-embeddings.ts hard-requires these two before doing any work. + // It builds its own client from NEXT_PUBLIC_SUPABASE_URL + + // SUPABASE_SECRET_KEY, but sources/partner-integrations.ts reads the MISC + // pair to pull partner data from the hosted "misc" project. Pointed at the + // local stack they clear the gate; the partner source then finds no such + // tables, which is the right trade for a local docs index. + NEXT_PUBLIC_MISC_URL: env.API_URL, + NEXT_PUBLIC_MISC_ANON_KEY: env.PUBLISHABLE_KEY ?? env.ANON_KEY, + OPENAI_API_KEY: process.env.OPENAI_API_KEY, + NODE_ENV: 'development', + }, + }); + console.log( + 'seeded. next: pnpm local docs api (keep it running in a separate terminal)' + ); +} + +function cmdApi(opts: DocsOptions) { + const docs = docsPath(opts.docs); + const docsApp = join(docs, 'apps', 'docs'); + const port = opts.port ?? '3001'; + const env = stackEnv(); + const tsx = join( + docsApp, + 'node_modules', + '.bin', + onWindows ? 'tsx.cmd' : 'tsx' + ); + if (!existsSync(tsx)) + fail( + `tsx not installed in the docs app — run:\n corepack pnpm --dir ${docs} install --filter ./apps/docs...` + ); + const stub = join(__dirname, 'docs', 'sentry-stub-register.mjs'); + console.log( + `serving on http://127.0.0.1:${port}/docs/api/graphql — point evals at it with --content-api` + ); + // Runs with the DOCS app's tsx + tsconfig so the route's TS and its + // `react-server` condition resolve; the Sentry stub no-ops the route's + // telemetry (the real package crashes outside Next's instrumentation). + run( + tsx, + [ + '--conditions=react-server', + '--tsconfig', + 'tsconfig.json', + join(__dirname, 'docs', 'content-api-server.ts'), + ], + { + cwd: docsApp, + shim: true, + env: { + NODE_ENV: 'development', + PORT: port, + DOCS_ROUTE_PATH: join(docsApp, 'app', 'api', 'graphql', 'route.ts'), + NEXT_PUBLIC_SUPABASE_URL: env.API_URL, + NEXT_PUBLIC_SUPABASE_ANON_KEY: env.PUBLISHABLE_KEY ?? env.ANON_KEY, + OPENAI_API_KEY: process.env.OPENAI_API_KEY, + NODE_OPTIONS: `--import ${stub}${process.env.NODE_OPTIONS ? ` ${process.env.NODE_OPTIONS}` : ''}`, + }, + } + ); +} + +export async function main(argv: string[]) { + const usage = + 'usage: pnpm local docs [--docs ] [--port N] [--yes]'; + const parsed = (() => { + try { + return parseArgs({ + args: argv, + options: { + docs: { type: 'string' }, + port: { type: 'string' }, + yes: { type: 'boolean' }, + }, + allowPositionals: true, + }); + } catch (err) { + fail(`${err instanceof Error ? err.message : String(err)}\n${usage}`); + } + })(); + const { values } = parsed; + switch (parsed.positionals[0]) { + case 'up': + cmdUp(values); + break; + case 'seed': + await cmdSeed(values); + break; + case 'api': + cmdApi(values); + break; + case 'down': + run('supabase', ['stop', '--workdir', OVERLAY], { quiet: true }); + console.log( + `docs stack stopped (project ${PROJECT_ID}); the seeded index stays in its docker volume` + ); + break; + default: + fail(usage); + } +} diff --git a/apps/framework/scripts/local.ts b/apps/framework/scripts/local.ts index 472a5a7b..77988f4d 100755 --- a/apps/framework/scripts/local.ts +++ b/apps/framework/scripts/local.ts @@ -1,17 +1,18 @@ #!/usr/bin/env tsx /** * local.ts — local-dev runner. Run evals against YOUR inputs (an edited skills - * tree, a local MCP build) with provenance receipts. + * tree, a local MCP build, a custom docs content API) with provenance receipts. * - * pnpm local run [--experiment ] [--runs N] [--mcp ] + * pnpm local run [--experiment ] [--runs N] [--mcp ] [--content-api ] * pnpm local experiments + * pnpm local docs [--docs ] (see local-docs.ts) * * Design notes: * - Treatment-only: nothing here ever mutates a git tree, so concurrent * sessions/worktrees cannot interfere and in-flight work is never at risk. - * - Explicit over magic: this does not build your MCP checkout for you; it - * reports what world it measured. Build it with `pnpm build` in your mcp - * checkout and pass `--mcp`. + * - Explicit over magic: this does not build your MCP checkout or re-embed + * docs for you; it reports what world it measured. Build with + * `pnpm build` in your mcp checkout; serve docs with `pnpm local docs`. * - Gates run before any model call, because the harness SKIPs an experiment * with exit 0 on missing credentials and a wasted agent run costs real money. */ @@ -36,6 +37,11 @@ import { rawEvalResultSchema, type RawEvalResult, } from '@supabase-evals/core/eval-metadata'; +import { main as docsMain } from './local-docs.js'; +import { + CONTENT_API_FLAG_MIN_VERSION, + supportsContentApiFlag, +} from './mcp-capabilities.js'; import { parsePublishedLog, PUBLISHED_LOG_FORMAT } from './published-log.js'; const __dirname = dirname(fileURLToPath(import.meta.url)); @@ -78,10 +84,11 @@ type Provenance = { generatedAt: string; host: { sha?: string; branch?: string; dirtyFiles: number }; mcpOverride?: { path: string; sha?: string; dirtyFiles?: number }; + contentApiUrl?: string; platform: string; }; -function collectProvenance(mcpPath?: string): Provenance { +function collectProvenance(mcpPath?: string, contentApi?: string): Provenance { const dirty = (cwd: string) => (tryGit(['status', '--porcelain'], cwd) ?? '').split('\n').filter(Boolean) .length; @@ -102,6 +109,7 @@ function collectProvenance(mcpPath?: string): Provenance { dirtyFiles: inRepo ? dirty(inRepo) : undefined, }; } + if (contentApi) p.contentApiUrl = contentApi; return p; } @@ -251,6 +259,43 @@ function resolveMcpServerPath(raw: string): string { return p; } +/** + * `--content-api` only reaches the docs index if the server we launch honours + * `--content-api-url` (supabase/mcp#343). Otherwise `search_docs` silently + * queries PRODUCTION docs while collectProvenance still stamps contentApiUrl + * into the receipt: a paid run that measures the wrong world and reports the + * right one. + * + * Two ways to satisfy it. Either the MCP_SERVER_VERSION pin is new enough that + * the published package carries the flag, in which case no local build is + * needed (createConfig builds the flag list once and shares it between the npx + * and local-build launch paths), or `--mcp` supplies a build that has it. + * + * The probe looks for the FLAG, not the env var, because the flag is what we + * actually pass: createConfig reads SUPABASE_CONTENT_API_URL in this process + * and forwards it as `--content-api-url` in the server's argv (a CLI agent + * spawns that command inside the sandbox container, which inherits no + * environment from us, so the server's own env fallback never fires). A build + * with the flag but no env fallback works fine here; one with only the env + * fallback would not. + * + * Shape check, so it runs for fake runs too (like resolveMcpServerPath). + */ +function validateContentApi(contentApi: string, mcpServerPath?: string) { + if (!mcpServerPath) { + // No override: it rides on the pinned published package, so the pin decides. + if (supportsContentApiFlag(MCP_SERVER_VERSION)) return; + fail( + `--content-api needs --mcp : the harness launches the pinned v${MCP_SERVER_VERSION} package, which has no --content-api-url flag (added in v${CONTENT_API_FLAG_MIN_VERSION} via supabase/mcp#343), so search_docs would query production docs while the receipt claims ${contentApi}\n pass --mcp pointing at an mcp checkout carrying that change, built\n (raising MCP_SERVER_VERSION to v${CONTENT_API_FLAG_MIN_VERSION} also lifts this, but the platform-lite fixtures track the pin, so that is a harness-wide change and not a local fix)` + ); + } + const stdio = join(mcpServerPath, 'dist', 'transports', 'stdio.js'); + if (!readFileSync(stdio, 'utf8').includes('content-api-url')) + fail( + `the mcp build at ${mcpServerPath} has no --content-api-url flag (predates supabase/mcp#343) — search_docs would query production docs, not ${contentApi}\n update and rebuild the checkout: git pull && pnpm install && pnpm build` + ); +} + /** * The experiment's declared skills must exist in this checkout, or the * treatment silently runs skill-less against a skills-enabled published @@ -341,11 +386,11 @@ function runTreatment( id: string, experiment: string, runs: number, - opts: { env: Record; mcpPath?: string } + opts: { env: Record; mcpPath?: string; contentApi?: string } ): void { - const { env, mcpPath } = opts; + const { env, mcpPath, contentApi } = opts; console.log( - `== treatment: ${id} (${experiment}, runs=${runs}${mcpPath ? ', mcp override' : ''}) ==` + `== treatment: ${id} (${experiment}, runs=${runs}${mcpPath ? ', mcp override' : ''}${contentApi ? ', content-api override' : ''}) ==` ); const resultPath = process.env.LOCAL_EVAL_CMD ? fakeRun(id, experiment) @@ -361,7 +406,7 @@ function runTreatment( const result = parsed.data; const receipt = { ...result, - provenance: collectProvenance(mcpPath), + provenance: collectProvenance(mcpPath, contentApi), }; writeFileSync( join(OUT_DIR, `${id}.treatment.json`), @@ -398,7 +443,7 @@ async function cmdExperiments() { } const RUN_USAGE = - 'usage: pnpm local run [--experiment ] [--runs N] [--mcp ]'; + 'usage: pnpm local run [--experiment ] [--runs N] [--mcp ] [--content-api ]'; async function cmdRun(argv: string[]) { const parsed = (() => { @@ -409,6 +454,7 @@ async function cmdRun(argv: string[]) { experiment: { type: 'string' }, runs: { type: 'string' }, mcp: { type: 'string' }, + 'content-api': { type: 'string' }, }, allowPositionals: true, }); @@ -433,11 +479,16 @@ async function cmdRun(argv: string[]) { const env: Record = {}; const mcpPath = values.mcp ? resolveMcpServerPath(values.mcp) : undefined; if (mcpPath) env.SUPABASE_MCP_SERVER_PATH = mcpPath; + const contentApi = values['content-api']; + if (contentApi) { + validateContentApi(contentApi, mcpPath); + env.SUPABASE_CONTENT_API_URL = contentApi; + } mkdirSync(OUT_DIR, { recursive: true }); for (const id of evalIds) { const runs = Number(values.runs ?? 1); - runTreatment(id, experiment, runs, { env, mcpPath }); + runTreatment(id, experiment, runs, { env, mcpPath, contentApi }); } } @@ -451,8 +502,12 @@ switch (command) { case 'experiments': await cmdExperiments(); break; + case 'docs': + await docsMain(rest); + break; default: - fail(`usage: pnpm local ... - run run eval(s) in your world (skills tree as-is; --mcp override) - experiments list experiments (agent, model, effort, published availability)`); + fail(`usage: pnpm local ... + run run eval(s) in your world (skills tree as-is; --mcp / --content-api overrides) + experiments list experiments (agent, model, effort, published availability) + docs --docs local docs content API`); } diff --git a/apps/framework/scripts/mcp-capabilities.ts b/apps/framework/scripts/mcp-capabilities.ts new file mode 100644 index 00000000..71eb2378 --- /dev/null +++ b/apps/framework/scripts/mcp-capabilities.ts @@ -0,0 +1,68 @@ +/** + * Which MCP server capabilities a given published version has. + * + * Split out of local.ts so the smoke suite can drive the version comparison + * directly: the pin it actually gates on is a constant in @supabase-evals/core, + * so an end-to-end check can only ever exercise whichever side of the boundary + * today's pin happens to sit on. + */ + +/** + * First published release containing the `--content-api-url` flag + * (supabase/mcp#343, merged 2026-07-23 as 6fcaaa3). + * + * Verified against the tags rather than assumed: #343 is an ancestor of + * `mcp-server-supabase-v0.10.0` and is NOT contained in v0.9.0 or v0.8.3. + */ +export const CONTENT_API_FLAG_MIN_VERSION = '0.10.0'; + +/** `MAJOR.MINOR.PATCH` with an optional prerelease tail. */ +const VERSION_RE = /^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$/; + +/** + * Whether the published package at `version` honours `--content-api-url`. + * + * When it does, `--content-api` needs no local build: createConfig appends the + * flag to the npx invocation exactly as it does for a local one (the flag list + * is built once and shared by both launch paths), and `rewriteLoopback` still + * maps 127.0.0.1 to host.docker.internal for in-container agents. + * + * Only stable releases count. Semver defines precedence, not content: a + * `0.11.0-beta` sorts above `0.10.0` but could have been cut from a branch that + * forked before the flag landed, so its number does not prove it carries it. + * The costs are lopsided — wrongly trusting a pin buys a paid run that measures + * production docs while the receipt claims otherwise, whereas wrongly refusing + * one costs the user a `--mcp` flag — so any prerelease is treated as + * incapable. Add it here explicitly if a prerelease ever needs to qualify. + * + * Throws on a version it cannot parse. Silently treating `0.10.foo` as 0.10.0 + * would report "capable" and buy exactly that bad run, and silently treating it + * as incapable would send you chasing `--mcp` for the wrong reason. A malformed + * pin is a bug in this repo, so it should be loud. + */ +export function supportsContentApiFlag(version: string): boolean { + const parse = (v: string, label: string) => { + const m = VERSION_RE.exec(v.trim()); + if (!m) + throw new Error( + `${label} is not a MAJOR.MINOR.PATCH version: ${JSON.stringify(v)}` + ); + return { + nums: [Number(m[1]), Number(m[2]), Number(m[3])], + isPrerelease: m[4] !== undefined, + }; + }; + const got = parse(version, 'mcp server version'); + const min = parse( + CONTENT_API_FLAG_MIN_VERSION, + 'CONTENT_API_FLAG_MIN_VERSION' + ); + + // Before any comparison: a prerelease's number tells us nothing about whether + // the flag is in its tree. + if (got.isPrerelease) return false; + for (let i = 0; i < 3; i++) { + if (got.nums[i] !== min.nums[i]) return got.nums[i] > min.nums[i]; + } + return true; +} diff --git a/apps/framework/scripts/smoke-local.ts b/apps/framework/scripts/smoke-local.ts index 389cb118..dea815eb 100644 --- a/apps/framework/scripts/smoke-local.ts +++ b/apps/framework/scripts/smoke-local.ts @@ -19,6 +19,10 @@ import { import { dirname, join } from 'node:path'; import { tmpdir } from 'node:os'; import { fileURLToPath } from 'node:url'; +import { + CONTENT_API_FLAG_MIN_VERSION, + supportsContentApiFlag, +} from './mcp-capabilities.js'; import { parsePublishedLog, PUBLISHED_LOG_FORMAT } from './published-log.js'; const __dirname = dirname(fileURLToPath(import.meta.url)); @@ -96,6 +100,38 @@ function ck(name: string, fn: () => void) { } } +// --- content-api capability: version comparison, both sides of the boundary --- +// Driven directly because the pin it gates on is a constant in core, so an +// end-to-end check only ever exercises whichever side today's pin sits on. +{ + ck('versions below the flag release are not capable', () => { + for (const v of ['0.8.1', '0.9.0', '0.9.9', '0.1.0']) + assert.equal(supportsContentApiFlag(v), false, v); + }); + ck('the flag release and newer are capable', () => { + for (const v of [CONTENT_API_FLAG_MIN_VERSION, '0.10.1', '0.11.0', '1.0.0']) + assert.equal(supportsContentApiFlag(v), true, v); + }); + ck('no prerelease counts, however high its number', () => { + // A version number orders releases; it does not prove what is in the tree. + // A prerelease could be cut from a branch that forked before the flag, and + // wrongly trusting one costs a paid run against production docs, so the + // conservative answer is the right one on both sides of the minimum. + for (const v of ['0.10.0-dev.1', '0.11.0-beta', '1.0.0-rc.1', '0.9.0-x']) + assert.equal(supportsContentApiFlag(v), false, v); + }); + ck('malformed versions throw instead of reporting a capability', () => { + // The dangerous direction: `0.10.foo` parsed loosely reads as 0.10.0 and + // would wave a paid run through against production docs. + for (const v of ['0.10.foo', '0.10', '', 'latest', 'v0.10.0']) + assert.throws( + () => supportsContentApiFlag(v), + /not a MAJOR\.MINOR\.PATCH version/, + `expected a throw for ${JSON.stringify(v)}` + ); + }); +} + // --- published-log parsing: the merge-commit case origin/main cannot reach --- // Built here rather than asserted against real history: main has no merge that // touches a published export, so an end-to-end check would pass either way. @@ -252,6 +288,71 @@ function ck(name: string, fn: () => void) { /packages[/\\]mcp-server-supabase$/ ); }); + + // --- --content-api: refused unless a build that honours it is supplied --- + const noMcp = local(['run', EVAL, '--content-api', 'http://127.0.0.1:3001']); + ck('--content-api without --mcp refused pre-spend', () => { + assert.equal(noMcp.status, 1); + assert.match(noMcp.out, /--content-api needs --mcp/); + assert.match(noMcp.out, /production docs/); + }); + + // the fixture above is a bare stub, i.e. a build predating supabase/mcp#343 + const staleBuild = local([ + 'run', + EVAL, + '--content-api', + 'http://127.0.0.1:3001', + '--mcp', + fake, + ]); + ck('mcp build without the --content-api-url flag refused pre-spend', () => { + assert.equal(staleBuild.status, 1); + assert.match(staleBuild.out, /predates supabase\/mcp#343/); + }); + + // The discriminating case: a build carrying ONLY the env fallback, no flag. + // We forward the URL as `--content-api-url` in the server's argv (the server + // runs in-container and inherits no environment), so such a build would + // ignore the override — probing for the env var name alone would wave it + // through and buy a paid run against production docs. + writeFileSync( + join(pkg, 'dist', 'transports', 'stdio.js'), + '// smoke fixture: env fallback only\nprocess.env.SUPABASE_CONTENT_API_URL\n' + ); + const envOnly = local([ + 'run', + EVAL, + '--content-api', + 'http://127.0.0.1:3001', + '--mcp', + fake, + ]); + ck('mcp build with only the env fallback refused pre-spend', () => { + assert.equal(envOnly.status, 1); + assert.match(envOnly.out, /no --content-api-url flag/); + }); + + // A post-#343 build: registers the flag and keeps the env fallback. + writeFileSync( + join(pkg, 'dist', 'transports', 'stdio.js'), + "// smoke fixture: post-#343 build\n['content-api-url']:{type:'string'}\nprocess.env.SUPABASE_CONTENT_API_URL\n" + ); + const honoured = local([ + 'run', + EVAL, + '--content-api', + 'http://127.0.0.1:3001', + '--mcp', + fake, + ]); + ck('build with the --content-api-url flag is accepted and recorded', () => { + assert.equal(honoured.status, 0); + const receipt = JSON.parse( + readFileSync(join(OUT, `${EVAL}.treatment.json`), 'utf8') + ); + assert.equal(receipt.provenance.contentApiUrl, 'http://127.0.0.1:3001'); + }); rmSync(fake, { recursive: true, force: true }); } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index e5281926..06d884d5 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -971,6 +971,18 @@ export function supabaseMcpServer( // docs-only server runs standalone with no `--api-url`. if (apiUrl) serverArgs.push('--api-url', apiUrl); + // Docs override: point `search_docs` at a local content API instead of + // the public docs GraphQL. Baked into args rather than left to the parent + // environment because CLI agents spawn this command INSIDE the sandbox + // container, which inherits nothing from the harness process — and + // `rewriteLoopback` then maps 127.0.0.1 -> host.docker.internal so the + // host-side API is actually reachable from in there. Flag support landed + // in supabase/mcp#343 and shipped in v0.10.0, so it works on the npx path + // too once MCP_SERVER_VERSION reaches that; below it, `pnpm local` refuses + // --content-api unless --mcp supplies a build that has the flag. + const contentApiUrl = process.env.SUPABASE_CONTENT_API_URL; + if (contentApiUrl) serverArgs.push('--content-api-url', contentApiUrl); + const local = resolveLocalMcpServer(); if (local) { // `node`, not process.execPath: CLI agents run this command INSIDE the diff --git a/packages/core/src/mcp-server.test.ts b/packages/core/src/mcp-server.test.ts index ccb0b7fb..9f30397a 100644 --- a/packages/core/src/mcp-server.test.ts +++ b/packages/core/src/mcp-server.test.ts @@ -18,16 +18,19 @@ import { } from 'node:fs'; import { join, relative } from 'node:path'; import { tmpdir } from 'node:os'; +import { rewriteLoopback } from './agents/shared.js'; import { MCP_SERVER_VERSION, supabaseMcpServer, supabaseMcpServerMounts, } from './index.js'; -// Stub (not mutate) env so a pre-existing SUPABASE_MCP_SERVER_PATH is restored -// per test. +// Stub (not mutate) env so pre-existing SUPABASE_* values are restored per test. +// SUPABASE_CONTENT_API_URL is cleared too: it now adds server args, so an +// ambient value would leak into every assertion below. function clearEnv() { vi.stubEnv('SUPABASE_MCP_SERVER_PATH', undefined); + vi.stubEnv('SUPABASE_CONTENT_API_URL', undefined); } // A real on-disk build layout: the override path is existence-checked, so the @@ -120,6 +123,35 @@ describe('supabaseMcpServer().createConfig', () => { rmSync(linkDir, { recursive: true, force: true }); } }); + + it('passes SUPABASE_CONTENT_API_URL as --content-api-url so it survives into the sandbox', async () => { + clearEnv(); + vi.stubEnv( + 'SUPABASE_CONTENT_API_URL', + 'http://127.0.0.1:3001/docs/api/graphql' + ); + const { config } = await supabaseMcpServer().createConfig({}); + // In args, not env: a CLI agent spawns this inside the container, which + // inherits nothing from the harness process. + const flag = config.args.indexOf('--content-api-url'); + expect(flag).toBeGreaterThan(-1); + expect(config.args[flag + 1]).toBe( + 'http://127.0.0.1:3001/docs/api/graphql' + ); + + // ...and being in args is what lets the container reach the host-side API. + const rewritten = rewriteLoopback({ supabase: config }); + expect(rewritten.supabase.args).toContain( + 'http://host.docker.internal:3001/docs/api/graphql' + ); + }); + + it('omits the flag when no local docs API is configured', async () => { + clearEnv(); + vi.stubEnv('SUPABASE_CONTENT_API_URL', undefined); + const { config } = await supabaseMcpServer().createConfig({}); + expect(config.args).not.toContain('--content-api-url'); + }); }); describe('supabaseMcpServerMounts', () => {