From 225c849654eb42563d9041ad6311a080be6cdecf Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Fri, 14 Aug 2026 15:22:29 +0100 Subject: [PATCH 1/3] feat: parallel eval attempts with any-pass aggregation Adds --parallel-attempts to the runner: instead of one sandbox running a pair's `runs` attempts sequentially with stop-on-pass, fan every (pair x attempt) out to its own sandbox concurrently and pass the eval if at least one attempt passed. Each attempt is scored exactly as today (--runs 1), so run-eval.ts is unchanged; the any-pass is a pure OR-aggregation over the per-attempt result files (the representative passing tree is kept, attempts set to the number run). Since the bottleneck is agent time per run, parallelizing the attempts can cut wall-clock roughly in half when concurrency covers pairs x runs (one wave). The workflow enables it via vars.EVAL_PARALLEL_ATTEMPTS and sizes concurrency to pairs x runs; default (unset) keeps sequential stop-on-pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/eval-refresh.yml | 18 +- apps/framework/scripts/run-vercel-evals.ts | 213 +++++++++++++++++++-- 2 files changed, 216 insertions(+), 15 deletions(-) diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 13691d13..5e55565f 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -305,17 +305,31 @@ jobs: # Optional warm-boot snapshot to reuse across runs (built out-of-band). # Unset -> cold boots, so this can never regress the default path. EVAL_SNAPSHOT_ID: ${{ vars.EVAL_SNAPSHOT_ID }} + # When "true", run each attempt in its own sandbox concurrently and + # pass an eval if any attempt passes (any-pass), instead of one + # sandbox running the attempts sequentially with stop-on-pass. + EVAL_PARALLEL_ATTEMPTS: ${{ vars.EVAL_PARALLEL_ATTEMPTS }} + RUNS: ${{ needs.prepare.outputs.runs }} + CONCURRENCY: ${{ needs.prepare.outputs.sandbox_concurrency }} shell: bash run: | set -euo pipefail + concurrency="$CONCURRENCY" args=( --pairs-json "$EVAL_PAIRS" --revision "$EVAL_REVISION" - --runs "${{ needs.prepare.outputs.runs }}" + --runs "$RUNS" --timeout-sec "${{ needs.prepare.outputs.timeout_sec }}" - --concurrency "${{ needs.prepare.outputs.sandbox_concurrency }}" ) + if [ "${EVAL_PARALLEL_ATTEMPTS:-}" = "true" ]; then + args+=(--parallel-attempts) + # Size concurrency to pairs x runs so every attempt runs in one wave + # (the halving only shows up when attempts don't queue behind pairs). + npairs="$(jq 'length' <<< "$EVAL_PAIRS")" + concurrency=$(( npairs * RUNS )) + fi + args+=(--concurrency "$concurrency") if [ -n "${EVAL_SNAPSHOT_ID:-}" ]; then args+=(--snapshot-id "$EVAL_SNAPSHOT_ID") fi diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts index 9f52393a..a68486ea 100644 --- a/apps/framework/scripts/run-vercel-evals.ts +++ b/apps/framework/scripts/run-vercel-evals.ts @@ -1,7 +1,14 @@ #!/usr/bin/env tsx import { execFile, execFileSync } from 'node:child_process'; -import { mkdtempSync, mkdirSync, rmSync } from 'node:fs'; +import { + cpSync, + mkdtempSync, + mkdirSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; @@ -81,11 +88,26 @@ interface RunnerOptions { * Supabase images pre-baked). Undefined means a cold git-source boot. */ snapshotId?: string; + /** + * Run each of a pair's `runs` attempts in its own sandbox concurrently and + * pass the eval if any attempt passed, instead of one sandbox running the + * attempts sequentially with stop-on-pass. + */ + parallelAttempts?: boolean; } interface PairOptions extends RunnerOptions { pair: EvalPair; + /** Infra-retry attempt for this sandbox (name/tag only). */ attempt: number; + /** + * 1-based eval-attempt index in parallel mode — the sandbox runs a single + * eval attempt and downloads to `destDir`. Undefined runs `runs` attempts in + * one sandbox to the canonical artifact directory. + */ + evalAttempt?: number; + /** Overrides the download destination (parallel mode stages per attempt). */ + destDir?: string; } /** Runs every item while keeping at most `concurrency` promises active. */ @@ -101,6 +123,10 @@ export async function runBounded( /** Runs all pairs and reports failures only after independent work finishes. */ async function runPairs(options: RunnerOptions): Promise { const credentials = vercelCredentialsFromEnv(); + if (options.parallelAttempts) { + await runParallelAttempts(options, credentials); + return; + } const results = await runBounded( options.pairs, @@ -162,20 +188,176 @@ async function runPairs(options: RunnerOptions): Promise { } } +/** + * Parallel-attempts mode: fan every (pair × attempt) out to its own sandbox, + * bounded by `concurrency`, then aggregate each pair's attempts with any-pass + * (the eval passes if at least one attempt passed). + */ +async function runParallelAttempts( + options: RunnerOptions, + credentials: VercelCredentials +): Promise { + const staging = mkdtempSync(join(tmpdir(), 'vercel-eval-attempts-')); + const jobs = options.pairs.flatMap((pair, pairIndex) => + Array.from({ length: options.runs }, (_, index) => ({ + pair, + pairIndex, + evalAttempt: index + 1, + })) + ); + try { + const settled = await runBounded(jobs, options.concurrency, async (job) => { + const destDir = join(staging, `${job.pairIndex}-a${job.evalAttempt}`); + const label = `${pairLabel(job.pair)} a${job.evalAttempt}`; + try { + await pRetry( + (attempt) => + runPairOnce( + { + ...options, + pair: job.pair, + attempt, + evalAttempt: job.evalAttempt, + destDir, + }, + credentials + ), + { + retries: 2, + factor: 2, + minTimeout: 5_000, + maxTimeout: 30_000, + randomize: true, + onFailedAttempt: ({ + error, + attemptNumber, + retriesLeft, + retryDelay, + }) => { + const retrying = retriesLeft > 0; + console.warn( + `${label} attempt ${attemptNumber} failed${retrying ? `, retrying in ${Math.round(retryDelay / 1000)}s` : ', not retrying'}: ${firstLine(errorMessage(error))}` + ); + }, + } + ); + return { pairIndex: job.pairIndex, destDir }; + } catch (error) { + console.error(`SANDBOX FAILED ${label}: ${errorMessage(error)}`); + throw error; + } + }); + + // Group the surviving attempt directories by pair. + const dirsByPair = new Map(); + for (const result of settled) { + if (result.status !== 'fulfilled') continue; + const { pairIndex, destDir } = result.value; + const dirs = dirsByPair.get(pairIndex) ?? []; + dirs.push(destDir); + dirsByPair.set(pairIndex, dirs); + } + + const failures: string[] = []; + for (let index = 0; index < options.pairs.length; index += 1) { + const pair = options.pairs[index]; + if (!pair) continue; + const dirs = dirsByPair.get(index) ?? []; + if (dirs.length === 0) { + failures.push( + `${pairLabel(pair)}: all ${options.runs} attempt(s) failed` + ); + continue; + } + try { + aggregateAttempts(pair, dirs, options.outputDir); + console.log(`SANDBOX OK ${pairLabel(pair)}`); + } catch (error) { + failures.push(`${pairLabel(pair)}: ${errorMessage(error)}`); + } + } + + if (failures.length > 0) { + throw new AggregateError( + failures.map((message) => new Error(message)), + `${failures.length} Sandbox eval pair(s) failed` + ); + } + } finally { + rmSync(staging, { recursive: true, force: true }); + } +} + +/** + * Merges a pair's per-attempt result trees into the canonical artifact + * directory. The eval passes if any attempt passed; the representative tree (a + * passing attempt if there is one, else the first) is kept, with its `attempts` + * count set to the number of attempts that produced a result. + */ +function aggregateAttempts( + pair: EvalPair, + attemptDirs: string[], + outputDir: string +): void { + const resultName = `${pair.eval_id}.json`; + const parsed = attemptDirs + .map((dir) => { + try { + const json = JSON.parse( + readFileSync(join(dir, resultName), 'utf8') + ) as Record; + return { dir, json }; + } catch { + return undefined; + } + }) + .filter( + (entry): entry is { dir: string; json: Record } => + entry !== undefined + ); + if (parsed.length === 0) { + throw new Error('no attempt produced a result file'); + } + + const representative = + parsed.find((entry) => entry.json.passed === true) ?? parsed[0]; + const passedCount = parsed.filter( + (entry) => entry.json.passed === true + ).length; + + const destination = join(outputDir, artifactDirectory(pair)); + rmSync(destination, { recursive: true, force: true }); + mkdirSync(destination, { recursive: true }); + cpSync(representative.dir, destination, { recursive: true }); + writeFileSync( + join(destination, resultName), + JSON.stringify({ ...representative.json, attempts: parsed.length }, null, 2) + ); + console.log( + `${pairLabel(pair)} ${passedCount}/${parsed.length} attempt(s) passed → ${representative.json.passed === true ? 'PASS' : 'FAIL'}` + ); +} + /** Runs one pair in a fresh Sandbox and downloads its complete result tree. */ async function runPairOnce( options: PairOptions, credentials: VercelCredentials ): Promise { const { pair } = options; - const label = pairLabel(pair); + const label = options.evalAttempt + ? `${pairLabel(pair)} a${options.evalAttempt}` + : pairLabel(pair); + // In parallel mode each sandbox runs exactly one eval attempt; otherwise one + // sandbox runs all `runs` attempts sequentially (stop-on-pass). + const runs = options.evalAttempt ? 1 : options.runs; + const timeouts = { runs, timeoutSec: options.timeoutSec }; let sandbox: Sandbox | undefined; try { const warm = Boolean(options.snapshotId); sandbox = await createSandbox(label, { ...credentials, - name: sandboxName(pair), + name: sandboxName(pair, options.evalAttempt), // A snapshot source carries its own runtime + filesystem; a git source // needs the runtime named and clones the repo fresh. ...(options.snapshotId @@ -190,7 +372,7 @@ async function runPairOnce( }, }), resources: { vcpus: options.vcpus }, - timeout: evalCommandTimeoutMs(options) + SANDBOX_TIMEOUT_BUFFER_MS, + timeout: evalCommandTimeoutMs(timeouts) + SANDBOX_TIMEOUT_BUFFER_MS, persistent: false, tags: { runner: 'supabase-evals', @@ -251,12 +433,12 @@ async function runPairOnce( '--eval', pair.eval_id, '--runs', - String(options.runs), + String(runs), '--timeout-sec', String(options.timeoutSec), ], cwd: SANDBOX_CWD, - timeoutMs: evalCommandTimeoutMs(options), + timeoutMs: evalCommandTimeoutMs(timeouts), }, true ); @@ -279,7 +461,11 @@ async function runPairOnce( cwd: SANDBOX_CWD, timeoutMs: 3 * 60 * 1_000, }); - await downloadResults(sandbox, pair, options.outputDir); + await downloadResults( + sandbox, + pair, + options.destDir ?? join(options.outputDir, artifactDirectory(pair)) + ); } catch (error) { if (error instanceof Error) throw error; throw new Error(String(error)); @@ -288,15 +474,14 @@ async function runPairOnce( } } -/** Downloads and extracts one pair into the aggregate artifact directory. */ +/** Downloads and extracts one sandbox's result tree into `destination`. */ async function downloadResults( sandbox: Sandbox, pair: EvalPair, - outputDir: string + destination: string ): Promise { const staging = mkdtempSync(join(tmpdir(), 'vercel-eval-results-')); const archive = join(staging, 'results.tgz'); - const destination = join(outputDir, artifactDirectory(pair)); try { const downloaded = await sandbox.downloadFile( { path: '/tmp/eval-results.tgz' }, @@ -383,9 +568,10 @@ function pairLabel(pair: EvalPair): string { } /** Produces a unique dashboard-safe Sandbox name. */ -function sandboxName(pair: EvalPair): string { +function sandboxName(pair: EvalPair, evalAttempt?: number): string { const suffix = Math.random().toString(36).slice(2, 8); - return `${tagValue(pair.experiment).slice(0, 35)}--${tagValue(pair.eval_id).slice(0, 45)}--${suffix}`; + const attempt = evalAttempt ? `a${evalAttempt}-` : ''; + return `${tagValue(pair.experiment).slice(0, 35)}--${tagValue(pair.eval_id).slice(0, 42)}--${attempt}${suffix}`; } /** Keeps retry notices readable when the final summary carries full output. */ @@ -420,10 +606,11 @@ async function main(): Promise { // An explicit id reuses a prebuilt snapshot across runs; `--snapshot` // builds one for this run (resolved below, after the dry-run guard). snapshotId: readFlag(rawArgs, 'snapshot-id'), + parallelAttempts: rawArgs.includes('--parallel-attempts'), }; console.log( - `${options.pairs.length} pair(s), concurrency=${options.concurrency}, runs=${options.runs}, timeout=${options.timeoutSec}s, revision=${options.revision.slice(0, 8)}` + `${options.pairs.length} pair(s), concurrency=${options.concurrency}, runs=${options.runs}, timeout=${options.timeoutSec}s, revision=${options.revision.slice(0, 8)}${options.parallelAttempts ? ', parallel-attempts (any-pass)' : ''}` ); for (const pair of options.pairs) console.log(`PLAN ${pairLabel(pair)}`); if (rawArgs.includes('--dry-run')) return; From 0c241c83d38b513fb4c0e1cdbd8254883a3ba040 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 14:43:24 +0000 Subject: [PATCH 2/3] chore: refresh eval results --- apps/web/src/data/eval-results.json | 14944 ++++++++-------- .../web/src/data/regression-eval-results.json | 450 +- 2 files changed, 7684 insertions(+), 7710 deletions(-) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index 006be91c..3be60779 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -31,7 +31,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user d7fb398a-0a59-40f5-aaa8-bc0b26722470, signUp returned {\"userId\":\"d7fb398a-0a59-40f5-aaa8-bc0b26722470\"}" + "notes": "db user 3fd405fe-2a14-4f81-9a19-217d645a1c9e, signUp returned {\"userId\":\"3fd405fe-2a14-4f81-9a19-217d645a1c9e\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -46,7 +46,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"d7fb398a-0a59-40f5-aaa8-bc0b26722470\"}" + "notes": "{\"userId\":\"3fd405fe-2a14-4f81-9a19-217d645a1c9e\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -75,17 +75,27 @@ }, "docs": { "calls": [ + { + "source": "shell_fetch", + "query": "curl -s --max-time 25 https://supabase.com/changelog.md | head -60", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 3161 + }, { "source": "search_docs", - "query": "{ searchDocs(query: \"signUp email password options data user metadata javascript\", limit: 4) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"signUp with email password and user metadata options data javascript\", limit: 4) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" + "url": "https://supabase.com/docs/reference/javascript/auth-signup" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" + "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", + "title": "Customizing Emails by Language" }, { "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" @@ -96,22 +106,12 @@ } ], "resultChars": 22309 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md | head -60", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 3161 } ] }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5/build-auth-001-email-password-flow.json" }, { @@ -148,7 +148,7 @@ { "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "found 2 rows" + "notes": "found 3 rows" }, { "name": "row level security is enabled on todos", @@ -166,7 +166,7 @@ { "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "2 rows" + "notes": "3 rows" } ], "skills": { @@ -183,29 +183,29 @@ "calls": [ { "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md 2>&1 | grep -iE 'breaking|rls|policy|grant' | head -30", + "query": "curl -s https://supabase.com/changelog.md 2>&1 | head -60", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 5391 + "resultChars": 3161 }, { "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog/45329-breaking-change-tables-not-exposed-to-data-and-graphql-api-automatically.md 2>&1 | head -60", + "query": "curl -s https://supabase.com/changelog/45329-breaking-change-tables-not-exposed-to-data-and-graphql-api-automatically.md 2>&1 | head -80", "pages": [ { "url": "https://supabase.com/changelog/45329-breaking-change-tables-not-exposed-to-data-and-graphql-api-automatically.md" } ], - "resultChars": 3976 + "resultChars": 5583 } ] }, "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5/build-cli-001-bootstrap-app.json" }, { @@ -261,19 +261,19 @@ "calls": [ { "source": "shell_fetch", - "query": "curl -s https://supabase.com/docs/guides/local-development/declarative-database-schemas.md | head -120", + "query": "curl -s --max-time 20 https://supabase.com/changelog.md 2>&1 | head -40 || echo \"NO NETWORK\"", "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas.md" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 4170 + "resultChars": 2110 } ] }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5/build-cli-002-declarative-schema.json" }, { @@ -309,12 +309,12 @@ { "name": "cron command enqueues to the 'tasks' queue", "passed": true, - "notes": "queue depth 0 -> 1" + "notes": "queue depth 1 -> 2" }, { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 83) from the queue" + "notes": "function removed the seeded message (id 30) from the queue" } ], "skills": { @@ -331,74 +331,66 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"pg_cron schedule job invoke edge function pgmq queue\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"pg_cron schedule job every minute cron.schedule\", limit: 4) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, { "url": "https://supabase.com/docs/guides/database/extensions/pg_net", "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" + "url": "https://supabase.com/docs/guides/auth/auth-hooks/send-sms-hook", + "title": "Send SMS Hook" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" + "url": "https://supabase.com/docs/guides/auth/auth-hooks/send-email-hook", + "title": "Send Email Hook" }, { "url": "https://supabase.com/docs/guides/cron", "title": "Cron" } ], - "resultChars": 62989 + "resultChars": 81247 }, { "source": "web_fetch", - "query": "How do you create a cron job that runs every minute using SQL (cron.schedule)? Show the exact SQL syntax, including how to name a job, how to run SQL vs invoke an edge function, and how to unschedule/replace an existing job. Include any notes about job names and idempotency.", + "query": "Show the exact SQL syntax for creating a cron job with cron.schedule, including how to name a job, use a '* * * * *' every-minute schedule, and how to unschedule/update a job. Also note any guidance about job names, sql payloads, and whether cron.schedule is idempotent.", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/cron/quickstart.md" } ], - "resultChars": 1310 + "resultChars": 1088 }, { "source": "web_fetch", - "query": "How do you create a pgmq queue and send/read/delete messages? Show exact SQL function signatures for pgmq.create, pgmq.send, pgmq.read, pgmq.delete, pgmq.pop. Also note anything about queue types (basic/unlogged/partitioned) and the pgmq_public wrappers for the Data API.", + "query": "Show exact SQL and JS for: creating a queue (pgmq.create), sending a message, reading messages, and deleting/archiving messages. Include the pgmq_public wrapper function names and their parameter names as used from supabase-js .schema('pgmq_public').rpc(...). Note the parameter names for read (queue_name, sleep_seconds, n) and delete (queue_name, message_id).", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/queues/quickstart.md" } ], - "resultChars": 1391 + "resultChars": 1323 }, { "source": "web_fetch", - "query": "List any entries tagged breaking-change or otherwise notable that relate to: Queues / pgmq, Cron / pg_cron, or Edge Functions (deployment, config.toml verify_jwt, serving). Include dates and links.", + "query": "List any entries tagged breaking-change, or any entries related to Queues (pgmq), Cron (pg_cron), Edge Functions runtime/env vars, or API keys (publishable/secret vs anon/service_role). Include dates and links.", "hasContent": true, "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 3048 + "resultChars": 4642 } ] }, "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5/build-cli-003-pg-cron-queue-workflow.json" }, { @@ -461,9 +453,20 @@ }, "docs": { "calls": [ + { + "source": "web_fetch", + "query": "List any breaking changes or notable changes relating to: API keys (secret/publishable keys, service_role), supabase-js v2 client usage in Node backends, PostgREST embedded resource / nested select behavior, or row limits (max_rows). Include dates and links.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 2525 + }, { "source": "search_docs", - "query": "{ searchDocs(query: \"secret key service role server-side supabase-js createClient API keys\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"secret key service_role supabase-js server-side createClient persistSession\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -471,29 +474,28 @@ "title": "Performing administration tasks on the server side with a secret key" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" + "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", + "title": "Build a Social Auth App with Expo React Native" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/getting-started/quickstarts/refine", + "title": "Use Supabase with Refine" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" }, { - "url": "https://supabase.com/docs/guides/getting-started/quickstarts/sveltekit", - "title": "Use Supabase with SvelteKit" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-refine", + "title": "Build a User Management App with Refine" } ], - "resultChars": 94242 + "resultChars": 107503 } ] }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5/build-dataapi-001-relational-report.json" }, { @@ -551,8 +553,7 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { @@ -560,7 +561,7 @@ }, "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5/build-dataapi-002-restock-alert-report.json" }, { @@ -616,11 +617,44 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"migrate existing postgres database to Supabase pg_restore dump\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", + "title": "Migrate from Vercel Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/neon", + "title": "Migrate from Neon to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/heroku", + "title": "Migrate from Heroku to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase", + "title": "Migrating to Supabase" + } + ], + "resultChars": 45371 + } + ] }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5/build-database-001-migrate-postgres-to-supabase.json" }, { @@ -685,73 +719,31 @@ }, "docs": { "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Function auth user Authorization header RLS createClient\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/functions/http-methods", - "title": "Routing" - } - ], - "resultChars": 73347 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/docs/guides/functions/auth.md | head -120", - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth.md" - } - ], - "resultChars": 5299 - }, { "source": "shell_fetch", - "query": "curl -s https://supabase.com/docs/guides/functions/auth-headers.md", + "query": "curl -s https://supabase.com/changelog.md | grep -iE 'breaking|edge function|getClaims|auth' | head -40", "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth-headers.md" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 3330 + "resultChars": 6945 }, { "source": "shell_fetch", - "query": "curl -s https://supabase.com/docs/guides/functions/secrets.md | head -60", + "query": "curl -s https://supabase.com/docs/guides/functions/auth.md", "pages": [ { - "url": "https://supabase.com/docs/guides/functions/secrets.md" + "url": "https://supabase.com/docs/guides/functions/auth.md" } ], - "resultChars": 2599 + "resultChars": 7274 } ] }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5/build-functions-004-service-role-bypass.json" }, { @@ -793,17 +785,17 @@ { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"07c687de-fcad-4acd-9271-a501b42d02be\",\"metric\":\"steps_a_msj188pa\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"7493bf83-24cc-4e2a-9e2e-cadf7c66e3cf\",\"metric\":\"steps_a_mst1vuru\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: [{\"user_id\":\"07c687de-fcad-4acd-9271-a501b42d02be\",\"metric\":\"steps_a_msj188pa\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"7493bf83-24cc-4e2a-9e2e-cadf7c66e3cf\",\"metric\":\"steps_a_mst1vuru\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"831c5088-6bc6-4c0e-8c1f-3fe53e8a0852\",\"metric\":\"steps_b_msj188pa\",\"value\":222}]" + "notes": "status 200: [{\"user_id\":\"53b9238e-71fc-4dbb-a1d1-f81061c9dd12\",\"metric\":\"steps_b_mst1vuru\",\"value\":222}]" }, { "name": "non-service key is not granted service access", @@ -838,28 +830,59 @@ "docs": { "calls": [ { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md 2>/dev/null | head -60 || echo \"changelog fetch failed\"", + "source": "web_fetch", + "query": "List any entries (especially breaking-change tagged) related to: Edge Functions, API keys (publishable/secret keys, sb_secret_, sb_publishable_), verify_jwt, JWT signing keys, or a @supabase/server package. Include dates and links.", + "hasContent": true, "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 3161 + "resultChars": 2477 }, { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog/45702-developer-update-may-2026.md 2>/dev/null | head -100", + "source": "search_docs", + "query": "{ searchDocs(query: \"Edge Functions authentication verify_jwt service role key user JWT getUser\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-getclaims" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-getuser" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-getuser", + "title": "user()" + } + ], + "resultChars": 11940 + }, + { + "source": "web_fetch", + "query": "Full content: how to secure Edge Functions. Include the recommended patterns for (a) verifying a user JWT in the handler, (b) authenticating a trusted service call with a secret/service-role key in the apikey header, and (c) any dual-auth pattern. List exact env var names (e.g. SUPABASE_SECRET_KEYS, SUPABASE_PUBLISHABLE_KEYS, SUPABASE_SERVICE_ROLE_KEY) and any code samples verbatim.", + "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog/45702-developer-update-may-2026.md" + "url": "https://supabase.com/docs/guides/functions/auth.md" } ], - "resultChars": 6160 + "resultChars": 2112 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"@supabase/server SDK edge functions auth service role\", limit: 12) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"SUPABASE_SECRET_KEYS SUPABASE_PUBLISHABLE_KEYS edge function environment variables secret key rotation\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -867,57 +890,80 @@ "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/auth/third-party/auth0", - "title": "Auth0" + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" }, { - "url": "https://supabase.com/docs/guides/auth/third-party/firebase-auth", - "title": "Firebase Auth" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + } + ], + "resultChars": 56147 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"@supabase/server withSupabase ctx authMode supabaseAdmin userClaims reference\", limit: 8) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/auth/third-party/aws-cognito", - "title": "Amazon Cognito (Amplify)" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", + "title": "Which package to use" }, { - "url": "https://supabase.com/docs/guides/auth/third-party/workos", - "title": "WorkOS" + "url": "https://supabase.com/docs/guides/database/postgres/roles", + "title": "Postgres Roles" }, { - "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", - "title": "Custom Roles" + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" }, { - "url": "https://supabase.com/docs/reference/cli/supabase-functions", - "title": "Manage Supabase Edge functions" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/guides/auth/architecture", - "title": "Auth architecture" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-react", + "title": "Build a User Management App with React" }, { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" + } + ], + "resultChars": 12 + }, + { + "source": "web_fetch", + "query": "Reproduce the content verbatim, especially anything about @supabase/server: withSupabase options, auth modes ('user', 'secret', arrays), the ctx object fields (supabase, supabaseAdmin, authMode, userClaims), required env vars, and when to use it vs supabase-js / @supabase/ssr.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package.md" } ], - "resultChars": 75829 + "resultChars": 1511 } ] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5/build-functions-005-dual-auth-user-secret.json" }, { @@ -958,7 +1004,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019fdc90-5069-75c6-93dc-d42f28e61c92/receipt-alpha.pdf, 019fdc90-5069-75c6-93dc-d42f28e61c92/receipt-beta.pdf" + "notes": "saw: 01a000b1-1c9c-7751-a925-d472bd643170/receipt-alpha.pdf, 01a000b1-1c9c-7751-a925-d472bd643170/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -979,7 +1025,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Meets all required criteria: private user-files bucket, owner-scoped SELECT and INSERT RLS policies on storage.objects for authenticated users with RLS kept enabled, and supabase-js createSignedUrl code with expiry for temporary sharing. No disqualifying public bucket, permissive policies, public URL sharing, or client service-role usage." + "judgeNotes": "Creates private user-files bucket, keeps RLS, defines authenticated owner-scoped SELECT and INSERT WITH CHECK policies (plus update/delete), and provides supabase-js code using createSignedUrl/createSignedUrls with expiry. No public bucket/getPublicUrl/service-role client misuse." } ], "skills": { @@ -993,75 +1039,66 @@ }, "docs": { "calls": [ + { + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md 2>&1 | head -100", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 5516 + }, { "source": "search_docs", - "query": "{ searchDocs(query: \"storage RLS policies private bucket user folder owner uid\", limit: 6) { nodes { title href content } } }", + "query": "{\n storage: searchDocs(query: \"storage RLS policies restrict users to their own folder\", limit: 5) {\n nodes { title href content }\n }\n signed: searchDocs(query: \"createSignedUrl signed URL expiring private bucket\", limit: 5) {\n nodes { title href content }\n }\n}", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, { "url": "https://supabase.com/docs/guides/storage/security/access-control", "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", + "title": "Storage Helper Functions" }, { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" + "url": "https://supabase.com/docs/guides/auth/third-party/clerk", + "title": "Clerk" }, { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - } - ], - "resultChars": 36724 - }, - { - "source": "web_fetch", - "query": "List any entries tagged breaking-change, and any entries related to Storage buckets, storage RLS policies, signed URLs, or the storage.objects schema (e.g. owner/owner_id columns, new bucket types). Include dates and links.", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" + }, { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 8117 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"createSignedUrl expires in temporary link download share\", limit: 4) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" + }, { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" + "url": "https://supabase.com/docs/guides/storage/cdn/smart-cdn", + "title": "Smart CDN" }, { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" }, { - "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" }, { - "url": "https://supabase.com/docs/reference/csharp/storage-from-createsignedurl", - "title": "From().CreateSignedUrl()" + "url": "https://supabase.com/docs/reference/dart/file-buckets-createsigneduploadurl" } ], - "resultChars": 3099 + "resultChars": 37689 } ] }, "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5/build-storage-001-private-bucket-access.json" }, { @@ -1089,17 +1126,17 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "3 file(s): supabase/tests/002_tenant_isolation_writes.test.sql, supabase/tests/001_tenant_isolation_select.test.sql, supabase/tests/003_tenant_isolation_baseline.test.sql" + "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "7 passed, 5 failed" + "notes": "14 passed, 8 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as having the broken tenant isolation policy, explicitly noting authenticated users in any org can read every post because the policy is missing `m.org_id = posts.org_id`. It grounds this in pgTAP results, distinguishes `notes` as passing read isolation, and treats test results as authoritative rather than trusting migration comments." + "judgeNotes": "The agent correctly identifies `posts` as having the tenant isolation flaw: its SELECT policy only checks membership in any org and omits correlation to `posts.org_id`, allowing authenticated members to read posts from organizations they do not belong to. This is grounded in pgTAP failures and manual verification, while distinguishing that `notes` is correctly isolated for reads." } ], "skills": { @@ -1117,7 +1154,7 @@ }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5/build-tests-001-rls-tenant-isolation.json" }, { @@ -1181,49 +1218,92 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"semantic search pgvector gte-small edge function embeddings match_document_sections RLS\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"pgvector semantic search match function hnsw embedding RLS\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/ai/hybrid-search", "title": "Hybrid search" }, - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, { "url": "https://supabase.com/docs/guides/ai/semantic-search", "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" + "url": "https://supabase.com/docs/guides/ai/vector-indexes/hnsw-indexes", + "title": "HNSW indexes" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" + "url": "https://supabase.com/docs/guides/ai/going-to-prod", + "title": "Going to Production" }, { "url": "https://supabase.com/docs/guides/ai/vector-columns", "title": "Vector columns" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + } + ], + "resultChars": 83277 + }, + { + "source": "web_fetch", + "query": "Show the full recommended SQL for semantic search: enabling the vector extension (which schema), the embedding column type/dimensions for gte-small, the match/search function definition (including security settings, set search_path, distance operator), and the HNSW index definition including which ops class to use for normalized embeddings. Quote SQL verbatim.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/ai/semantic-search.md" + } + ], + "resultChars": 1214 + }, + { + "source": "web_fetch", + "query": "Quote the HNSW index creation SQL and explain which operator class to use for inner product / cosine, and what operator (#<#, <=>) pairs with each.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/ai/vector-indexes/hnsw-indexes.md" + } + ], + "resultChars": 732 + }, + { + "source": "web_fetch", + "query": "What are the output dimensions of the gte-small model in Supabase.ai.Session? Quote any SQL showing the vector column size used with gte-small.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/ai-models.md" + } + ], + "resultChars": 576 + }, + { + "source": "web_fetch", + "query": "List any entries tagged breaking-change, and any entries related to pgvector, the vector extension, extensions schema, or database functions / RLS.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 84107 + "resultChars": 4010 } ] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5/build-vectors-001-rag-with-permissions.json" }, { @@ -1253,12 +1333,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "Meets requirements: HTTPS Supabase Metrics API scrape for evalshostedprojectxy.supabase.co at /customer/v1/privileged/metrics, uses HTTP Basic Auth with password_file, preserves existing app scrape, and docker-compose mounts the secrets directory matching the password_file path." + "judgeNotes": "Meets all rubric requirements: HTTPS Supabase Metrics API scrape for the project ref, correct path, basic_auth with password_file, app scrape preserved, and docker-compose mounts the password_file location." }, { "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "README includes Secret API key creation, matching password_file setup, reload/Compose rollout steps, and concrete verification via curl, Prometheus targets/query API, and Grafana dashboard." + "judgeNotes": "README includes Secret API key creation, matching password_file placement, compose start/reload steps, and concrete verification via curl, Prometheus targets API, and Grafana dashboard checks." } ], "skills": { @@ -1274,7 +1354,7 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"Prometheus metrics endpoint project observability scrape\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Prometheus metrics endpoint project observability\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -1285,37 +1365,30 @@ "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", "title": "Vendor-agnostic Metrics API setup" }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" + }, { "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" + "url": "https://supabase.com/docs/reference/api/v1-scrape-project-metrics", + "title": "Scrape a project's metrics" }, { "url": "https://supabase.com/docs/guides/platform/read-replicas", "title": "Read Replicas" } ], - "resultChars": 29057 - }, - { - "source": "web_fetch", - "query": "What is the exact metrics endpoint URL format, what authentication does it use (username/password), and what is the recommended Prometheus scrape config and scrape interval? Include any notes about rate limits or which key to use.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/telemetry/metrics.md" - } - ], - "resultChars": 1186 + "resultChars": 29685 } ] }, "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5/deploy-database-001-prometheus-metrics.json" }, { @@ -1370,81 +1443,30 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Functions secrets environment variables managing\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - } - ], - "resultChars": 71163 - }, - { - "source": "web_fetch", - "query": "List any breaking changes or notable changes related to Edge Functions, function deployment, function secrets/environment variables, or the CLI's secrets/functions commands.", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md | head -100", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 1947 - }, - { - "source": "web_fetch", - "query": "How do you set and access environment variables/secrets in Supabase Edge Functions? Include the exact CLI commands for setting production secrets from a .env file, local .env handling, and reserved secret names. Quote commands verbatim.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets.md" - } - ], - "resultChars": 1274 - }, - { - "source": "web_fetch", - "query": "Show the current recommended boilerplate for a Supabase Edge Function index.ts (Deno.serve vs std serve import), the deploy command, and any config.toml settings for functions like verify_jwt. Quote code verbatim.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/quickstart.md" - } - ], - "resultChars": 879 + "resultChars": 5516 }, { "source": "shell_fetch", - "query": "curl -s https://supabase.com/docs/guides/functions/quickstart.md | head -120", + "query": "curl -s https://supabase.com/docs/guides/functions/secrets.md", "pages": [ { - "url": "https://supabase.com/docs/guides/functions/quickstart.md" + "url": "https://supabase.com/docs/guides/functions/secrets.md" } ], - "resultChars": 5029 + "resultChars": 5105 } ] }, "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5/deploy-functions-001-edge-function-secrets.json" }, { @@ -1500,31 +1522,31 @@ "calls": [ { "source": "web_fetch", - "query": "Extract the full self-hosting with Docker guide verbatim-ish: the exact steps to get the docker setup, all the secrets/env vars that must be changed before going to production (JWT secret, anon/service keys, postgres password, dashboard user/pass, secret_key_base, vault enc key, SITE_URL, API_EXTERNAL_URL, SUPABASE_PUBLIC_URL, pooler tenant id, etc.), securing services, and any notes about restarting/regenerating keys.", + "query": "Extract the full self-hosting with Docker instructions verbatim as much as possible: the steps to get started, which files are needed, the full list of environment variables in .env, how to generate API keys / JWT secret, and all \"securing your setup\" recommendations. Include any notes about which services are included.", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/self-hosting/docker.md" } ], - "resultChars": 3113 + "resultChars": 3648 }, { "source": "web_fetch", - "query": "List any entries tagged breaking-change or otherwise relevant to self-hosting, Docker, JWT signing keys / API key format (publishable/secret vs anon/service_role), Supavisor/pooler, or Studio, from 2025 and 2026. Include dates and links.", + "query": "List any entries related to self-hosting, docker, docker-compose, analytics/logflare, API keys (publishable/secret keys, anon key deprecation), or JWT signing keys. Include dates and any breaking-change tags.", "hasContent": true, "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 3915 + "resultChars": 3280 } ] }, "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5/deploy-self-hosting-001-docker-compose.json" }, { @@ -1577,8 +1599,8 @@ }, { "name": "diagnosed and explained session revocation", - "passed": true, - "judgeNotes": "The answer identifies the original soft-delete-only bug and lack of session/refresh-token revocation, implements real auth/session revocation via banning and deleting sessions plus RLS checks, and explains the remaining stateless JWT caveat consistently: Data API notes/profiles are closed by RLS, but purely local JWT validation remains valid until exp with mitigations. It also correctly distinguishes publishable frontend keys with RLS from server-only secret/service_role keys that bypass RLS and must not ship to clients." + "passed": false, + "judgeNotes": "The answer diagnoses the soft-delete/session problem, implements real session revocation and an Edge Function using Admin API deleteUser, explains publishable vs secret keys correctly, and adds an RLS/session guard that closes the Data API stale-JWT window. However, its own final memory states the frontend still calls the RPC and until cutover “deleted accounts remain as zombie auth users that can still authenticate,” meaning the actual user-facing delete-account flow is not fixed to delete the auth user. The rubric requires fixing the delete-account flow so the auth user is deleted or identity/sessions are removed; leaving the shipped flow dependent on a pending frontend cutover fails that requirement." } ], "skills": { @@ -1592,11 +1614,29 @@ }, "docs": { "calls": [ + { + "source": "shell_fetch", + "query": "curl -s --max-time 25 https://supabase.com/changelog.md | head -60", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 3161 + }, { "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key new API keys anon service_role RLS\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"publishable key secret key API keys anon service_role RLS\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" + }, { "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", "title": "Creating a Supabase client for SSR" @@ -1606,27 +1646,84 @@ "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + } + ], + "resultChars": 113282 + }, + { + "source": "shell_fetch", + "query": "for u in guides/auth/sessions guides/auth/signout guides/auth/jwts guides/auth/jwt-fields; do echo \"=== $u ===\"; curl -sL \"https://supabase.com/docs/$u.md\" | head -300; done", + "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" + "url": "https://supabase.com/docs/$u.md" + } + ], + "resultChars": 38570 + }, + { + "source": "shell_fetch", + "query": "for u in reference/javascript/auth-admin-deleteuser reference/javascript/auth-admin-signout guides/auth/auth-hooks/custom-access-token-hook; do echo \"=== $u ===\"; curl -sL \"https://supabase.com/docs/$u.md\" | head -200; done", + "pages": [ + { + "url": "https://supabase.com/docs/$u.md" + } + ], + "resultChars": 6688 + }, + { + "source": "shell_fetch", + "query": "echo \"--- deleteuser (no .md) ---\"; curl -sL \"https://supabase.com/docs/reference/javascript/auth-admin-deleteuser\" -o /tmp/du.html -w \"%{http_code} %{size_download}\\n\"\necho \"--- try llms ---\"; curl -sL \"https://supabase.com/docs/reference/javascript/auth-admin-deleteuser.md\" -w \"\\nHTTP:%{http_code} size:%{size_download}\\n\" | tail -5\necho \"--- search docs for delete user ---\"; curl -sL \"https://supabase.com/docs/guides/auth/users.md\" | head -200", + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser.md" }, { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" + "url": "https://supabase.com/docs/guides/auth/users.md" + } + ], + "resultChars": 14634 + }, + { + "source": "shell_fetch", + "query": "cd /tmp && for p in auth-admin-signout auth-admin-updateuserbyid; do echo \"=== $p ===\"; curl -sL \"https://supabase.com/docs/reference/javascript/$p\" | python3 -c \"\nimport re,html,sys\ns=sys.stdin.read()\ns=re.sub(r'','',s,flags=re.S); s=re.sub(r'','',s,flags=re.S)\nt=html.unescape(re.sub(r'<[^>]+>',' ',s)); t=re.sub(r'[ \\t]+',' ',t); t=re.sub(r'\\n\\s*\\n+','\\n',t); print(t)\n\" | grep -in -B3 -A6 \"scope\\|ban\\|revok\\|jwt\" | head -80; done", + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/$p" + } + ], + "resultChars": 152 + }, + { + "source": "web_fetch", + "query": "Quote verbatim everything about deleteUser: parameters (id, shouldSoftDelete), what shouldSoftDelete does, any notes/warnings about sessions, access tokens, cascade, requires service_role. Include the full parameter descriptions.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + } + ], + "resultChars": 726 + }, + { + "source": "web_fetch", + "query": "Quote verbatim the description, parameters (jwt, scope), and any notes about whether access tokens remain valid or only refresh tokens are revoked.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-signout" } ], - "resultChars": 190908 + "resultChars": 549 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"delete user does not invalidate access token revoke sessions sign out\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"access token JWT expiry limit maximum 604800 configure session settings\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -1634,28 +1731,272 @@ "title": "User sessions" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" + "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", + "title": "Should I set a shorter Max-Age parameter on the cookies?" }, { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" + "url": "https://supabase.com/docs/guides/auth/server-side/advanced-guide", + "title": "Advanced guide" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/platform/temporary-access", + "title": "Temporary access" }, { "url": "https://supabase.com/docs/guides/resources/glossary", "title": "Glossary" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", - "title": "Should I set a shorter Max-Age parameter on the cookies?" + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-setsession" } ], - "resultChars": 66489 - } - ] + "resultChars": 82133 + }, + { + "source": "shell_fetch", + "query": "cd /tmp && for u in guides/database/postgres/roles-superuser guides/troubleshooting guides/auth/auth-hooks guides/database/postgres/configuration; do :; done\necho \"=== search: modifying auth schema ===\"\nfor u in \"guides/auth/managing-user-data\" \"guides/database/overview\" \"guides/auth/auth-hooks\"; do echo \"--- $u\"; curl -sL \"https://supabase.com/docs/$u.md\" | grep -in -B2 -A6 \"auth schema\" | head -40; done", + "pages": [ + { + "url": "https://supabase.com/docs/$u.md" + } + ], + "resultChars": 1111 + }, + { + "source": "web_search", + "query": "Supabase docs \"auth schema\" \"not recommended\" modify tables managed by Supabase triggers auth.users", + "hasContent": false, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/dashboard-errors-when-managing-users-N1ls4A", + "title": "Supabase Docs | Troubleshooting | Errors when creating / updating / deleting users" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/resolving-500-status-authentication-errors-7bU5U8", + "title": "Supabase Docs | Troubleshooting | Resolving 500 Status Authentication Errors" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/database-error-saving-new-user-RU_EwB", + "title": "Supabase Docs | Troubleshooting | Database error saving new user" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting", + "title": "Supabase Docs | Troubleshooting" + }, + { + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth | Supabase Docs" + }, + { + "url": "https://supabase.com/changelog/9314-security-patch-notice", + "title": "Security Patch Notice · Changelog" + }, + { + "url": "https://supabase.com/docs/guides/auth/architecture", + "title": "Auth architecture | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/reference/javascript/v1/auth-update", + "title": "JavaScript API Reference | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-hooks", + "title": "Auth Hooks | Supabase Docs" + } + ], + "resultChars": 2445 + }, + { + "source": "shell_fetch", + "query": "cd /tmp && echo \"=== troubleshooting: dashboard-errors-when-managing-users ===\"; curl -sL \"https://supabase.com/docs/guides/troubleshooting/dashboard-errors-when-managing-users-N1ls4A.md\" | head -60; echo; echo \"=== auth/architecture ===\"; curl -sL \"https://supabase.com/docs/guides/auth/architecture.md\" | grep -in -B3 -A10 \"auth schema\\|do not\\|permission\\|supabase_auth_admin\" | head -60", + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/dashboard-errors-when-managing-users-N1ls4A.md" + }, + { + "url": "https://supabase.com/docs/guides/auth/architecture.md" + } + ], + "resultChars": 4350 + }, + { + "source": "web_search", + "query": "Supabase \"delete my account\" Edge Function service role deleteUser recommended pattern docs", + "hasContent": false, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser", + "title": "JavaScript: deleteUser | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-admin-deleteuser", + "title": "Swift API Reference | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser", + "title": "Flutter: deleteUser | Supabase Docs" + } + ], + "resultChars": 2916 + }, + { + "source": "shell_fetch", + "query": "cd /tmp && grep -rn \"customizations to the auth schema\" -A6 -B6 mud.md 2>/dev/null; echo \"=== try troubleshooting search ===\"; curl -sL \"https://supabase.com/docs/guides/troubleshooting/resolving-500-status-authentication-errors-7bU5U8.md\" | grep -in -B6 -A10 \"auth schema\" | head -50", + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/resolving-500-status-authentication-errors-7bU5U8.md" + } + ], + "resultChars": 2593 + }, + { + "source": "web_search", + "query": "\"customizations to the auth schema\" supabase \"break migrations done by the Auth\"", + "hasContent": false, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/resolving-500-status-authentication-errors-7bU5U8", + "title": "Supabase Docs | Troubleshooting | Resolving 500 Status Authentication Errors" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/migrating-auth-users-between-projects", + "title": "Supabase Docs | Troubleshooting | Migrating Auth Users Between Supabase Projects" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-hooks", + "title": "Auth Hooks | Supabase Docs" + }, + { + "url": "https://supabase.com/changelog?next=Y3Vyc29yOnYyOpK0MjAyNC0wOC0zMFQxNjowMzo0NVrOAGyKbA%3D%3D&restPage=2", + "title": "Changelog" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage | Supabase Docs" + }, + { + "url": "https://supabase.com/features/cli", + "title": "CLI | Supabase Features" + }, + { + "url": "https://supabase.com/docs/guides/local-development/overview?access-method=postgres&=&queryGroups=access-method", + "title": "Local development with schema migrations | Supabase Docs" + } + ], + "resultChars": 2511 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"secret key with user access token which role applies apikey header Authorization precedence Data API\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" + } + ], + "resultChars": 83398 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"create multiple publishable keys name rotate publishable key expiry never expire\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + } + ], + "resultChars": 74216 + }, + { + "source": "web_search", + "query": "supabase authenticated role permission denied for table auth.sessions RLS policy security definer function check session_id", + "hasContent": false, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security | Supabase Docs" + } + ], + "resultChars": 2965 + }, + { + "source": "web_search", + "query": "Supabase dashboard \"Access token (JWT) expiry time\" maximum 604800 seconds one week minimum", + "hasContent": false, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/guides/auth/jwts", + "title": "JSON Web Token (JWT) | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/jwt-expired-error-in-supabase-dashboard-F06k3x" + } + ], + "resultChars": 2428 + } + ] }, "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", @@ -1708,7 +2049,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "The assistant correctly identified the root cause as orders missing from the supabase_realtime publication despite SUBSCRIBED succeeding, fixed it with ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, and did not weaken RLS/policies or disrupt courier_locations. It checked RLS/grants but did not blame them or change them." + "judgeNotes": "Diagnosed missing public.orders membership in supabase_realtime despite SUBSCRIBED channel, fixed with ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, and did not weaken RLS/policies or disrupt courier_locations." } ], "skills": { @@ -1721,11 +2062,36 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"postgres changes enable realtime publication supabase_realtime add table\", limit: 4) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", + "title": "Subscribing to Database Changes" + }, + { + "url": "https://supabase.com/docs/guides/realtime/postgres-changes", + "title": "Postgres Changes" + }, + { + "url": "https://supabase.com/docs/guides/realtime/concepts", + "title": "Realtime Concepts" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", + "title": "Backup and Restore using the CLI" + } + ], + "resultChars": 95291 + } + ] }, "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5/investigate-realtime-001-subscribed-no-events.json" }, { @@ -1752,17 +2118,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform as affected and described the 8 HTTP 503 gateway failures recurring across the morning of 2026-04-28 from 07:00Z to 12:00Z, distinguishing them from unrelated billing-webhook 503s." + "judgeNotes": "Identified image-transform as the affected function and described the recurring pattern of 8 gateway HTTP 503s spread through the morning of 2026-04-28, with retries succeeding. Did not incorrectly center the old billing-webhook 503s." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "The assistant clearly attributes the recurring image-transform 503s to the gateway/Edge platform layer, not function code, and grounds this in valid observations: gateway 503s had no matching edge-function invocations, nearby invocations succeeded, deployment/version stayed unchanged, and avatar-upload's function-level 500 is distinguished as a separate class of error." + "judgeNotes": "Attributes the image-transform 503s to the gateway/platform layer before function invocation, not function code. Grounds this in valid observations: 503s only in gateway logs with no edge-function execution rows, nearby function-level errors do appear in both layers, deployment/version unchanged, successful invocations nearby, and distinguishes avatar-upload's logged 500 as application-level/unrelated." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended concrete next steps, including opening/support escalation to the platform with the specific time window and evidence, investigating the scheduled caller, adding jitter/retries, and separately triaging a code-path error." + "judgeNotes": "The assistant recommended concrete next steps: verify the correct project/ref and evidence, determine the upload path, get Edge Function gateway/platform metrics for the incident window, add retry/backoff, and investigate the avatar-upload 500 separately." } ], "skills": { @@ -1779,7 +2145,7 @@ }, "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5/investigate-reliability-003-edge-function-5xx-correlation.json" }, { @@ -1833,7 +2199,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed RLS enabled with no policies causing deny-all Data API results, kept RLS enabled, and created authenticated-only owner-scoped SELECT and INSERT policies using auth.uid() with WITH CHECK for inserts. Did not use permissive policies or disable RLS." + "judgeNotes": "Diagnosed RLS enabled with no policies/default-deny, kept RLS enabled, and added authenticated SELECT and INSERT owner-scoped policies enforcing auth.uid() = user_id via USING/WITH CHECK." } ], "skills": { @@ -1846,51 +2212,11 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"row level security policy auth.uid() select insert performance index\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0003_auth_rls_initplan", - "title": "Database Advisor: Lint 0003_auth_rls_initplan" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/rls-performance-and-best-practices-Z5Jjwv", - "title": "RLS Performance and Best Practices" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" - } - ], - "resultChars": 67906 - }, - { - "source": "web_fetch", - "query": "List any entries tagged breaking-change related to RLS, row level security, policies, the Data API / PostgREST, grants, or auth.uid(). Just list titles, dates and one-line summaries.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 1403 - } - ] + "calls": [] }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5/resolve-dataapi-001-empty-results.json" }, { @@ -1937,7 +2263,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "Avatar migration was applied through `supabase db push` in action #21, with output showing `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration was reconciled by adding the missing local migration file `supabase/migrations/20240115000000_add_profile_bio.sql` in action #18, after which `supabase migration list` showed local/remote alignment and the push proceeded. I saw read-only psql inspections and a rolled-back verification schema, but no prohibited workaround or direct application of the avatar migration outside the CLI." + "judgeNotes": "Applied pending avatar_url migration with `supabase db push --include-all`, which output `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` Reconciled the orphan bio migration by adding local file `supabase/migrations/20240115000000_add_profile_bio.sql`, after which `supabase migration list` showed local and remote aligned. Read-only psql inspections were used; no disallowed workaround or direct mutation of remote DB/history observed." } ], "skills": { @@ -1954,7 +2280,7 @@ }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5/resolve-database-001-migration-history-mismatch.json" }, { @@ -1994,7 +2320,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_desc_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -2007,7 +2333,6 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", "supabase-postgres-best-practices" ] }, @@ -2016,7 +2341,7 @@ }, "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5/resolve-performance-001-slow-query-cpu-spike.json" }, { @@ -2093,7 +2418,7 @@ }, "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5/resolve-security-002-rls-cross-tenant-leak.json" }, { @@ -2128,7 +2453,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 5a5d5991-7eae-49b4-8f13-2487fba8366c, signUp returned {\"userId\":\"5a5d5991-7eae-49b4-8f13-2487fba8366c\"}" + "notes": "db user b3edfec4-0277-4646-8f47-d5304308159b, signUp returned {\"userId\":\"b3edfec4-0277-4646-8f47-d5304308159b\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -2143,7 +2468,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"5a5d5991-7eae-49b4-8f13-2487fba8366c\"}" + "notes": "{\"userId\":\"b3edfec4-0277-4646-8f47-d5304308159b\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -2170,7 +2495,7 @@ }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5-no-skills/build-auth-001-email-password-flow.json" }, { @@ -2207,7 +2532,7 @@ { "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "found 3 rows" + "notes": "found 2 rows" }, { "name": "row level security is enabled on todos", @@ -2225,7 +2550,7 @@ { "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "3 rows" + "notes": "2 rows" } ], "skills": { @@ -2233,19 +2558,48 @@ "loaded": [] }, "docs": { - "calls": [] - }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-cli-001-bootstrap-app.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"row level security policy authenticated users select only read-only table\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0024_permissive_rls_policy", + "title": "Database Advisor: Lint 0024_permissive_rls_policy" + } + ], + "resultChars": 78282 + } + ] + }, + "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", + "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-opus-5-no-skills/build-cli-001-bootstrap-app.json" + }, + { + "experiment": "claude-code-opus-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", "modelId": "claude-opus-5", "reasoningEffort": "high" }, @@ -2284,36 +2638,11 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{searchDocs(query:\"declarative database schemas migration workflow db diff\", limit:4){nodes{title href content}}}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-pull", - "title": "Pull schema from the remote database" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" - } - ], - "resultChars": 62949 - } - ] + "calls": [] }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5-no-skills/build-cli-002-declarative-schema.json" }, { @@ -2354,7 +2683,7 @@ { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 13) from the queue" + "notes": "function removed the seeded message (id 3) from the queue" } ], "skills": { @@ -2365,73 +2694,33 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"pg_cron schedule job every minute queues pgmq send\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - }, - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" - } - ], - "resultChars": 69432 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Supabase Queues create queue read delete messages edge function\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"queues quickstart create queue send read delete pgmq\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, { "url": "https://supabase.com/docs/guides/queues/quickstart", "title": "Quickstart" }, { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pgmq", - "title": "pgmq: Queues" + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" }, { - "url": "https://supabase.com/docs/guides/functions/architecture", - "title": "Edge Functions Architecture" + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" } ], - "resultChars": 55770 + "resultChars": 36950 } ] }, "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5-no-skills/build-cli-003-pg-cron-queue-workflow.json" }, { @@ -2492,7 +2781,7 @@ }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5-no-skills/build-dataapi-001-relational-report.json" }, { @@ -2553,7 +2842,7 @@ }, "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5-no-skills/build-dataapi-002-restock-alert-report.json" }, { @@ -2607,7 +2896,7 @@ }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5-no-skills/build-database-001-migrate-postgres-to-supabase.json" }, { @@ -2670,7 +2959,7 @@ }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5-no-skills/build-functions-004-service-role-bypass.json" }, { @@ -2707,37 +2996,37 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" + "notes": "status 401: {\"error\":\"Unauthorized\"}" }, { "name": "user with JWT reads only their own rows", - "passed": false, - "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" + "passed": true, + "notes": "status 200: [{\"user_id\":\"1fafaf6a-59b6-4ad3-92e4-723686d496e7\",\"metric\":\"steps_a_mst1zvtv\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": false, - "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" + "notes": "status 403: {\"error\":\"Forbidden\"}" }, { "name": "service key bypasses RLS to read the target user's rows", - "passed": false, - "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" + "passed": true, + "notes": "status 200: [{\"user_id\":\"d5281832-0aee-46f7-a46e-38417fc7980f\",\"metric\":\"steps_b_mst1zvtv\",\"value\":222}]" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" + "notes": "status 401: {\"error\":\"Unauthorized\"}" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" + "notes": "status 401: {\"error\":\"Unauthorized\"}" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" + "notes": "status 401: {\"error\":\"Unauthorized\"}" }, { "name": "implementation uses @supabase/server", @@ -2750,36 +3039,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge function environment variables SUPABASE_SERVICE_ROLE_KEY SB_SECRET_KEY publishable\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" - } - ], - "resultChars": 94601 - } - ] + "calls": [] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", @@ -2824,7 +3084,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019fdc8e-c692-723e-917b-7953678ca0d1/receipt-alpha.pdf, 019fdc8e-c692-723e-917b-7953678ca0d1/receipt-beta.pdf" + "notes": "saw: 01a000af-ef63-76bf-890d-9db18bc4cfc8/receipt-alpha.pdf, 01a000af-ef63-76bf-890d-9db18bc4cfc8/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -2845,7 +3105,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Meets all requirements: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies with WITH CHECK, RLS kept enabled, and supabase-js createSignedUrl/createSignedUrls with expiry for temporary sharing." + "judgeNotes": "Creates private user-files bucket, keeps RLS enabled, defines authenticated owner-scoped SELECT and INSERT policies using first path segment = auth.uid(), avoids public/anon/service-role pitfalls, and provides supabase-js createSignedUrl with expiry." } ], "skills": { @@ -2853,11 +3113,85 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"storage RLS policy user folder owner uid first folder name\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", + "title": "Storage Helper Functions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" + }, + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/auth/third-party/clerk", + "title": "Clerk" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", + "title": "Column Level Security" + } + ], + "resultChars": 64798 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"createSignedUrl signed url expires storage\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" + }, + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" + }, + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" + }, + { + "url": "https://supabase.com/docs/guides/storage/cdn/smart-cdn", + "title": "Smart CDN" + }, + { + "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" + } + ], + "resultChars": 11623 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"storage access control ownership policies private bucket per user folder\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" + }, + { + "url": "https://supabase.com/docs/guides/storage/cdn/fundamentals", + "title": "Storage CDN" + }, + { + "url": "https://supabase.com/docs/guides/storage/uploads/file-limits", + "title": "Limits" + } + ], + "resultChars": 6837 + } + ] }, "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5-no-skills/build-storage-001-private-bucket-access.json" }, { @@ -2885,17 +3219,17 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "3 file(s): supabase/tests/database/02_posts_tenant_isolation.test.sql, supabase/tests/database/03_memberships_and_writes.test.sql, supabase/tests/database/01_notes_tenant_isolation.test.sql" + "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "4 passed, 2 failed" + "notes": "6 passed, 3 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as the broken tenant isolation policy, explains that authenticated members can read posts from orgs they are not members of, and grounds this in pgTAP/direct test results. It also distinguishes `notes` as passing isolation." + "judgeNotes": "The agent correctly identifies `posts` as having the tenant isolation flaw: authenticated members can read posts from organizations they do not belong to, and grounds this in pgTAP result 6 plus independent confirmation. It also correctly distinguishes `notes` as scoped for reads and treats the test results as authoritative rather than dismissing them." } ], "skills": { @@ -2907,7 +3241,7 @@ }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5-no-skills/build-tests-001-rls-tenant-isolation.json" }, { @@ -2941,12 +3275,12 @@ { "name": "HNSW index on the embedding column", "passed": true, - "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" + "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { "name": "index operator class matches the search operator", "passed": true, - "notes": "function operators: <#>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { "name": "user A search returns only own sections, best match first", @@ -2974,7 +3308,7 @@ }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5-no-skills/build-vectors-001-rag-with-permissions.json" }, { @@ -3004,12 +3338,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "Supabase scrape uses HTTPS, correct metrics path, Basic Auth with password_file, targets a supabase.co project ref, preserves the app job, and docker-compose mounts the secrets directory containing the password file path." + "judgeNotes": "Meets all requirements: preserves app scrape, adds HTTPS Supabase Metrics API scrape for a project ref on supabase.co with correct metrics_path, uses HTTP Basic Auth with password_file, and docker-compose mounts the password file path via a read-only secrets volume." }, { "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "README includes Secret API key creation, matching secret file path, Compose restart/up and reload guidance, plus concrete verification via Prometheus targets, PromQL API, and Grafana dashboard checks." + "judgeNotes": "README includes correct Secret API key creation, matching password_file placement, reload/restart guidance, and concrete verification via Prometheus targets and Grafana/PromQL." } ], "skills": { @@ -3020,7 +3354,7 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"Prometheus metrics endpoint scrape project metrics observability\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Prometheus metrics endpoint project observability scrape\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -3031,6 +3365,10 @@ "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", "title": "Vendor-agnostic Metrics API setup" }, + { + "url": "https://supabase.com/docs/reference/api/v1-scrape-project-metrics", + "title": "Scrape a project's metrics" + }, { "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", "title": "Metrics API with Prometheus & Grafana (self-hosted)" @@ -3040,13 +3378,13 @@ "title": "Metrics API with Grafana Cloud" } ], - "resultChars": 19943 + "resultChars": 20571 } ] }, "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5-no-skills/deploy-database-001-prometheus-metrics.json" }, { @@ -3094,40 +3432,11 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge functions environment variables secrets manage\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-secrets", - "title": "Manage Supabase secrets" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - } - ], - "resultChars": 61195 - } - ] + "calls": [] }, "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5-no-skills/deploy-functions-001-edge-function-secrets.json" }, { @@ -3179,7 +3488,7 @@ }, "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5-no-skills/deploy-self-hosting-001-docker-compose.json" }, { @@ -3233,7 +3542,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "The answer diagnoses the original flow as only soft-deleting public.profiles and not touching auth.users/sessions/refresh tokens. It implements real revocation by banning/soft-deleting the auth user and deleting refresh_tokens/sessions, plus RLS gating. It consistently explains JWT access tokens remain cryptographically valid until expiry for local validation, while its implemented RLS/session-existence check closes the Data API window. It also correctly distinguishes publishable frontend keys under anon/authenticated + RLS from secret/service_role backend-only keys that bypass RLS." + "judgeNotes": "Meets the rubric: correctly diagnoses soft-delete-only flow, implements real session/refresh-token revocation plus sign-in blocking, adds RLS gating to close the data-path stale-JWT window, explains remaining JWT validity for locally validated/unprotected surfaces, and correctly distinguishes publishable frontend/RLS-enforced keys from secret server-only/RLS-bypassing keys." } ], "skills": { @@ -3244,25 +3553,9 @@ "calls": [ { "source": "search_docs", - "query": "{\n banned: searchDocs(query: \"ban user banned_until revoke session delete user admin API\", limit: 4) {\n nodes { title href content }\n }\n keys: searchDocs(query: \"publishable key secret key legacy anon service_role API keys migration RLS\", limit: 6) {\n nodes { title href content }\n }\n}", + "query": "{ searchDocs(query: \"publishable key secret key new API keys anon service_role migration RLS\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" - }, { "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", "title": "Build a User Management App with RedwoodJS" @@ -3288,11 +3581,11 @@ "title": "Build a User Management App with SolidJS" } ], - "resultChars": 245869 + "resultChars": 195336 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key API keys\", limit: 3) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"API keys publishable secret anon service_role\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -3304,130 +3597,112 @@ "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" } ], - "resultChars": 61983 + "resultChars": 59592 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"sb_secret sb_publishable new API keys migration\", limit: 2) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Understanding API keys publishable key anon role authenticated role RLS bypass\", limit: 2) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" } ], - "resultChars": 29035 + "resultChars": 40726 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"Understanding API keys publishable key anon role RLS\", limit: 1) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"delete user admin deleteUser soft delete\", limit: 2) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - } - ], - "resultChars": 13676 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"secret key bypasses Row Level Security service_role postgres role\", limit: 2) { nodes { title href } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" } ], - "resultChars": 329 + "resultChars": 908 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"ban user ban_duration admin updateUserById\", limit: 3) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"sessions refresh token revoke sign out access token expiry JWT TTL\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-updateuserbyid" + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" }, { - "url": "https://supabase.com/docs/reference/dart/auth-admin-updateuserbyid" + "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", + "title": "Should I set a shorter Max-Age parameter on the cookies?" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-updateuser" + "url": "https://supabase.com/docs/guides/auth/server-side/advanced-guide", + "title": "Advanced guide" } ], - "resultChars": 3573 + "resultChars": 23429 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"deleteUser soft delete admin\", limit: 3) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"banned_until ban user auth schema do not modify auth.users directly\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0021_fkey_to_auth_unique", + "title": "Database Advisor: Lint 0021_fkey_to_auth_unique" }, { - "url": "https://supabase.com/docs/reference/swift/auth-admin-deleteuser", - "title": "deleteUser()" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" } ], - "resultChars": 1237 + "resultChars": 23573 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"disable legacy anon service_role key dashboard rotate JWT secret\", limit: 2) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"managing user data delete user account permanently auth.users foreign key\", limit: 2) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/rotating-anon-service-and-jwt-secrets-1Jq6yd", - "title": "Rotating Anon, Service, and JWT Secrets" + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" } ], - "resultChars": 34590 + "resultChars": 21497 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"JWT expiry access token lifetime sessions\", limit: 3) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"ban_duration updateUserById ban user banned_until\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" + "url": "https://supabase.com/docs/reference/javascript/auth-admin-updateuserbyid" } ], - "resultChars": 59617 + "resultChars": 2179 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"auth.admin.signOut revoke sessions scope global\", limit: 3) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"signOut scope global others local revoke all refresh tokens\", limit: 2) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -3436,32 +3711,33 @@ }, { "url": "https://supabase.com/docs/reference/javascript/auth-signout" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-signout" } ], - "resultChars": 5307 + "resultChars": 5109 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"legacy API keys deprecation end of 2026 removal timeline\", limit: 2) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"auth schema managed by Supabase do not modify triggers on auth.users unsupported\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/auth/architecture", + "title": "Auth architecture" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/auth-error-500-database-error-querying-schema-eb6b44", + "title": "Auth error: '500: Database error querying schema'" } ], - "resultChars": 45967 + "resultChars": 11948 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"banned_until auth.users table columns\", limit: 3) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Users guide auth.users table columns banned_until\", limit: 2) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -3477,269 +3753,92 @@ }, { "source": "search_docs", - "query": "{ searchDocs(query: \"delete account user edge function service role\", limit: 2) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Users identities auth.users columns is_anonymous email_confirmed_at user model\", limit: 2) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" + "url": "https://supabase.com/docs/guides/auth/users", + "title": "Users" } ], - "resultChars": 20425 + "resultChars": 26091 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"do not modify auth schema managed by Supabase\", limit: 2) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"troubleshooting invalidate JWT immediately after ban or delete user cannot use token\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/prisma/prisma-troubleshooting", - "title": "Troubleshooting prisma errors" + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" }, { - "url": "https://supabase.com/docs/guides/auth/architecture", - "title": "Auth architecture" - } - ], - "resultChars": 16231 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"verifying a JWT from Supabase getClaims asymmetric local verification\", limit: 2) { nodes { title href } subsections: nodes { title href } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" + "url": "https://supabase.com/docs/guides/platform/sso/testing-best-practices", + "title": "SSO Testing and Best Practices" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" } ], - "resultChars": 490 + "resultChars": 75665 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"JSON Web Token JWT verifying revocation\", limit: 1) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"ban a user prevent sign in banned account\", limit: 2) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" - } - ], - "resultChars": 14157 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"banned user cannot sign in ban\", limit: 3) { nodes { title href } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", - "title": "Passwordless email logins" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-identity-linking", - "title": "Identity Linking" - } - ], - "resultChars": 392 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"cannot delete user foreign key constraint troubleshooting\", limit: 3) { nodes { title href } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0021_fkey_to_auth_unique", - "title": "Database Advisor: Lint 0021_fkey_to_auth_unique" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/cascade-deletes", - "title": "Cascade Deletes" + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - } - ], - "resultChars": 468 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"user management managing users admin api\", limit: 3) { nodes { title href } } }", - "hasContent": false, - "pages": [ { "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", "title": "Multiple SSO Providers" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-refine", - "title": "Build a User Management App with Refine" - }, - { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" - } - ], - "resultChars": 406 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Users guide auth users table permanent anonymous\", limit: 1) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" - } - ], - "resultChars": 11521 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"managing user data profiles trigger accessing user data\", limit: 1) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", - "title": "Build a User Management App with Angular" } ], - "resultChars": 34746 + "resultChars": 34007 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"signOut jwt scope admin server side revoke refresh tokens\", limit: 3) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"should_soft_delete admin users delete endpoint auth API\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" - }, - { - "url": "https://supabase.com/docs/reference/javascript/oauth-server-revokegrant" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" } ], - "resultChars": 38073 + "resultChars": 6927 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"JWT claims reference session_id is_anonymous fields\", limit: 1) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"ban_duration none unban user duration format\", limit: 3) { nodes { title href content } } }", "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - } - ], - "resultChars": 12679 + "pages": [], + "resultChars": 71 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"shouldSoftDelete\", limit: 3) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"auth admin signOut jwt revoke session server side\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", - "title": "Deleting data and dropping objects safely" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/soft-deletes-with-supabase-js", - "title": "Implementing soft deletes with supabase-js" - } - ], - "resultChars": 12330 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"service role key never expose server side edge function admin client\", limit: 2) { nodes { title href } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - } - ], - "resultChars": 326 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Data API PostgREST how requests are authorized JWT role\", limit: 2) { nodes { title href } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" + "url": "https://supabase.com/docs/reference/swift/auth-admin-signout", + "title": "signOut()" }, { - "url": "https://supabase.com/docs/guides/database/postgres/roles", - "title": "Postgres Roles" - } - ], - "resultChars": 273 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"access token expiry setting jwt_exp configuration auth\", limit: 2) { nodes { title href } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" + "url": "https://supabase.com/docs/reference/javascript/auth-signout" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", - "title": "Should I set a shorter Max-Age parameter on the cookies?" + "url": "https://supabase.com/docs/reference/javascript/auth-admin-signout" } ], - "resultChars": 358 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"user_banned error code\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - } - ], - "resultChars": 11900 - }, - { - "source": "search_docs", - "query": "{ error(code: \"user_banned\", service: AUTH) { code httpStatusCode message } }", - "hasContent": false, - "pages": [], - "resultChars": 280 - }, - { - "source": "search_docs", - "query": "{ a: error(code: \"session_not_found\", service: AUTH) { code message } b: error(code: \"user_not_found\", service: AUTH) { code message } c: error(code: \"refresh_token_not_found\", service: AUTH) { code message } d: error(code: \"refresh_token_already_used\", service: AUTH) { code message } e: error(code: \"session_expired\", service: AUTH) { code message } }", - "hasContent": false, - "pages": [], - "resultChars": 971 + "resultChars": 2213 } ] }, @@ -3794,7 +3893,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "The answer correctly identifies that the channel can reach SUBSCRIBED while INSERT events are silent because public.orders was missing from the supabase_realtime publication. It fixes exactly that with ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, preserves courier_locations, keeps RLS/policies intact, and explicitly does not blame RLS or client code." + "judgeNotes": "Diagnosed the silent subscription as orders missing from supabase_realtime, applied ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, and did not weaken RLS/policies or disrupt courier_locations." } ], "skills": { @@ -3806,7 +3905,7 @@ }, "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5-no-skills/investigate-realtime-001-subscribed-no-events.json" }, { @@ -3833,17 +3932,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "The assistant clearly identified `image-transform` as the affected function and described the recurring pattern of eight HTTP 503 gateway failures spread through the morning of 2026-04-28, while distinguishing them from older billing-webhook 503s." + "judgeNotes": "Identified image-transform as the affected function and described the recurring 8 HTTP 503 gateway failures across the morning of 2026-04-28 (07:00Z–12:00Z), distinguishing them from older billing-webhook 503s." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "Attributes the recurring image-transform 503s to the gateway/platform layer, not function code, and grounds this in valid observations: 503s only in gateway logs with no runtime invocations, nearby successful invocations, unchanged deployment/version, and distinction from avatar-upload's function-level 500." + "judgeNotes": "Attributes the recurring image-transform 503s to the gateway/platform/boot-dispatch layer before function code, not application code. Grounds this in valid observations: 503s appear in gateway/api logs but are absent from edge-function logs while nearby invocations succeed; deployment is unchanged and successful invocations are healthy; distinguishes gateway-only 503s from avatar-upload's function-level logged 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended multiple concrete next steps: investigate half-hour scheduled jobs/concurrency, query function_edge_logs for status>=500, check Edge Function limits/metrics, open a support ticket with timestamps, and examine avatar-upload error output." + "judgeNotes": "The assistant recommended concrete next steps including re-pulling current api/edge-function logs, checking Edge Function boot/deployment logs for failing timestamps, pinning/vendorizing dependencies, and fixing logging blind spots." } ], "skills": { @@ -3855,7 +3954,7 @@ }, "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" }, { @@ -3909,7 +4008,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed RLS enabled with zero policies as the cause of empty Data API results, kept RLS enabled, and created authenticated-only owner-scoped SELECT and INSERT policies using auth.uid() with WITH CHECK for inserts." + "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies causing deny-all Data API results, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts. It did not use permissive public/anon policies or disable RLS." } ], "skills": { @@ -3921,7 +4020,7 @@ }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5-no-skills/resolve-dataapi-001-empty-results.json" }, { @@ -3968,7 +4067,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "Avatar migration was applied through the Supabase CLI with `supabase db push` in action #19, showing `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration was reconciled by adding the local file `20240115000000_add_profile_bio.sql` in action #16, after which Supabase CLI migration list showed it matched remote (#17) and the subsequent `db push` succeeded. No disallowed direct-SQL mutation or prepared-statement workaround was used." + "judgeNotes": "Avatar migration was applied through the Supabase CLI with `supabase db push` in #21, which showed `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration was reconciled by adding the local file `20240115000000_add_profile_bio.sql` in #15, after which `supabase migration list` (#20) showed it matched remote and `db push` proceeded. No disallowed workaround such as direct remote ALTER, schema_migrations edit, or DEALLOCATE was seen." } ], "skills": { @@ -3980,7 +4079,7 @@ }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5-no-skills/resolve-database-001-migration-history-mismatch.json" }, { @@ -4036,7 +4135,7 @@ }, "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5-no-skills/resolve-performance-001-slow-query-cpu-spike.json" }, { @@ -4108,7 +4207,7 @@ }, "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5-no-skills/resolve-security-002-rls-cross-tenant-leak.json" }, { @@ -4143,7 +4242,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 0ac704f8-596e-4f98-84c9-00595301873d, signUp returned {\"userId\":\"0ac704f8-596e-4f98-84c9-00595301873d\"}" + "notes": "db user 39427fca-ebe9-4ace-b5a2-b03243a78e53, signUp returned {\"userId\":\"39427fca-ebe9-4ace-b5a2-b03243a78e53\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -4158,7 +4257,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"0ac704f8-596e-4f98-84c9-00595301873d\"}" + "notes": "{\"userId\":\"39427fca-ebe9-4ace-b5a2-b03243a78e53\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -4190,7 +4289,7 @@ }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/build-auth-001-email-password-flow.json" }, { @@ -4227,7 +4326,7 @@ { "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "found 2 rows" + "notes": "found 3 rows" }, { "name": "row level security is enabled on todos", @@ -4245,7 +4344,7 @@ { "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "2 rows" + "notes": "3 rows" } ], "skills": { @@ -4263,7 +4362,7 @@ }, "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/build-cli-001-bootstrap-app.json" }, { @@ -4319,7 +4418,7 @@ }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/build-cli-002-declarative-schema.json" }, { @@ -4355,12 +4454,12 @@ { "name": "cron command enqueues to the 'tasks' queue", "passed": true, - "notes": "queue depth 0 -> 1" + "notes": "queue depth 1 -> 2" }, { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 3) from the queue" + "notes": "function removed the seeded message (id 5) from the queue" } ], "skills": { @@ -4375,102 +4474,76 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "{ searchDocs(query: \"cron.schedule pgmq send queue example\", limit: 5) { nodes { title href content } } }", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md | head -100", "pages": [ { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 35564 + "resultChars": 5516 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Functions read from queue pgmq delete message worker\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"pg_cron schedule job cron.schedule syntax\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/recursive-functions", - "title": "Recursive / Nested Function Calls" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", - "title": "Backup and Restore using the CLI" - } - ], - "resultChars": 93380 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Functions default environment variables SUPABASE_SERVICE_ROLE_KEY SUPABASE_URL automatically available\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", + "title": "pg_cron debugging guide" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" } ], - "resultChars": 28165 + "resultChars": 32837 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"Queues schedule cron job to process messages Edge Function example\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Queues pgmq read send delete pop edge function worker example\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, { "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", "title": "Consuming Supabase Queue Messages with Edge Functions" }, { - "url": "https://supabase.com/docs/guides/auth/auth-hooks/send-email-hook", - "title": "Send Email Hook" + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" }, { - "url": "https://supabase.com/docs/guides/functions/examples/slack-bot-mention", - "title": "Slack Bot Mention Edge Function" + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" }, { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", + "title": "Resumable WebSockets with Edge Functions" } ], - "resultChars": 81933 + "resultChars": 37478 } ] }, "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/build-cli-003-pg-cron-queue-workflow.json" }, { @@ -4536,7 +4609,7 @@ }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/build-dataapi-001-relational-report.json" }, { @@ -4593,16 +4666,14 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [ - "supabase" - ] + "loaded": [] }, "docs": { "calls": [] }, "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/build-dataapi-002-restock-alert-report.json" }, { @@ -4661,7 +4732,7 @@ }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/build-database-001-migrate-postgres-to-supabase.json" }, { @@ -4729,7 +4800,7 @@ }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/build-functions-004-service-role-bypass.json" }, { @@ -4771,17 +4842,17 @@ { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"9dfb45f7-a584-480f-9567-31a82085a4db\",\"metric\":\"steps_a_msj1bz52\",\"value\":111}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"2bfd7226-9f06-45a8-a19b-e8c34da68e68\",\"metric\":\"steps_a_mst1spq9\",\"value\":111}]}" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"9dfb45f7-a584-480f-9567-31a82085a4db\",\"metric\":\"steps_a_msj1bz52\",\"value\":111}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"2bfd7226-9f06-45a8-a19b-e8c34da68e68\",\"metric\":\"steps_a_mst1spq9\",\"value\":111}]}" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"7ef32f13-71a0-4757-84bc-00f9632c3443\",\"metric\":\"steps_b_msj1bz52\",\"value\":222}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"6216d098-cec2-4343-9d5c-3ad291e3d5af\",\"metric\":\"steps_b_mst1spq9\",\"value\":222}]}" }, { "name": "non-service key is not granted service access", @@ -4817,7 +4888,7 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Functions environment variables SUPABASE_URL SUPABASE_SERVICE_ROLE_KEY SUPABASE_ANON_KEY secret key\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Edge Functions environment variables SUPABASE_URL SUPABASE_ANON_KEY SUPABASE_SERVICE_ROLE_KEY secret key publishable key\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -4832,190 +4903,16 @@ "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", "title": "Migrating to publishable and secret API keys" }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/storage-caching", - "title": "Integrating with Supabase Storage" - } - ], - "resultChars": 36932 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"new API keys publishable secret key edge functions verify_jwt\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, { "url": "https://supabase.com/docs/guides/functions/auth", "title": "Securing Edge Functions" }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - } - ], - "resultChars": 70715 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Securing Edge Functions auth modes user secret publishable multiple auth withSupabase array\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, { "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - } - ], - "resultChars": 45970 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Authorization headers edge functions apikey header verify_jwt disable\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-hooks/send-email-hook", - "title": "Send Email Hook" - } - ], - "resultChars": 95133 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"combining auth modes user secret verify_jwt false config.toml example\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" - }, - { - "url": "https://supabase.com/docs/guides/functions/deploy", - "title": "Deploy to Production" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - } - ], - "resultChars": 24076 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"@supabase/server package version pin npm install deno.json import map\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/security/npm-security", - "title": "Securing npm installs" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", - "title": "Building an MCP Server with mcp-lite" - }, - { - "url": "https://supabase.com/docs/guides/functions/dependencies", - "title": "Managing dependencies" - } - ], - "resultChars": 34326 - }, - { - "source": "web_search", - "query": "\"failed to determine entrypoint\" supabase edge-runtime worker boot error", - "pages": [], - "resultChars": 2031 - }, - { - "source": "web_search", - "query": "supabase cli issue 4190 podman edge functions serve entrypoint bind mount", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/troubleshooting", - "title": "Supabase Docs | Edge Functions Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/issues-serving-edge-functions-locally", - "title": "Supabase Docs | Troubleshooting | Issues serving Edge Functions locally" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration | Supabase Docs" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips | Supabase Docs" - } - ], - "resultChars": 2355 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Issues serving Edge Functions locally troubleshooting entrypoint bind mount docker context\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/issues-serving-edge-functions-locally", - "title": "Issues serving Edge Functions locally" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" } ], - "resultChars": 45231 + "resultChars": 41771 } ] }, @@ -5062,7 +4959,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019fdc8d-2b1c-70b8-a74d-7a4097cf4fa6/receipt-alpha.pdf, 019fdc8d-2b1c-70b8-a74d-7a4097cf4fa6/receipt-beta.pdf" + "notes": "saw: 01a000ac-b660-717c-8919-6b34492e9b64/receipt-alpha.pdf, 01a000ac-b660-717c-8919-6b34492e9b64/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -5083,7 +4980,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "The answer creates a private user-files bucket, defines authenticated SELECT and INSERT policies on storage.objects scoped to bucket_id and the user's UID folder with WITH CHECK for uploads, keeps RLS enabled, avoids public/anon/service-role pitfalls, and provides supabase-js createSignedUrl with an expiry for temporary sharing." + "judgeNotes": "Meets all rubric requirements: private user-files bucket, owner-scoped authenticated SELECT and INSERT policies with WITH CHECK on storage.objects, RLS not disabled and verified, plus supabase-js createSignedUrl with expiry for short-lived sharing links." } ], "skills": { @@ -5096,65 +4993,11 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"storage RLS policy owner folder path user id\", limit: 5) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/ownership", - "title": "Ownership" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" - } - ], - "resultChars": 61599 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"createSignedUrl expiring share link\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" - }, - { - "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - }, - { - "url": "https://supabase.com/docs/reference/csharp/storage-from-createsignedurl", - "title": "From().CreateSignedUrl()" - } - ], - "resultChars": 7873 - } - ] + "calls": [] }, "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/build-storage-001-private-bucket-access.json" }, { @@ -5187,12 +5030,12 @@ { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "6 passed, 4 failed" + "notes": "12 passed, 0 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as having a broken tenant isolation SELECT policy, specifically that members of any org can read posts from other orgs, and grounds this in pgTAP test 5 failing. It treats test results as authoritative and contrasts `posts` with `notes`, which passed isolation tests. It also notes additional membership-table issues, but does not blame `notes` instead of `posts`." + "judgeNotes": "The agent correctly identifies `posts` as having the tenant isolation flaw: authenticated users who are members of some org can read posts from other orgs because the policy lacks `posts.org_id` matching. It grounds this in pgTAP/test results, explicitly says `notes` passed and is correctly scoped, and does not dismiss the tests or blame `notes`. Extra mention of `memberships` RLS being disabled does not invalidate the required conclusion." } ], "skills": { @@ -5206,11 +5049,40 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"pgTAP RLS testing row level security authenticated role\", limit: 5) { totalCount nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/testing/overview", + "title": "Testing Overview" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pgtap", + "title": "pgTAP: Unit Testing" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/roles", + "title": "Postgres Roles" + } + ], + "resultChars": 12 + } + ] }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/build-tests-001-rls-tenant-isolation.json" }, { @@ -5274,15 +5146,14 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"semantic search edge functions pgvector gte-small match_document_sections\", limit: 3) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"semantic search edge functions gte-small match_document_sections\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -5290,19 +5161,25 @@ "title": "Semantic Search" }, { - "url": "https://supabase.com/docs/guides/database/full-text-search", - "title": "Full Text Search" + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" + "url": "https://supabase.com/docs/reference/javascript/using-filters-textsearch" + }, + { + "url": "https://supabase.com/docs/reference/javascript/using-filters-gte" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search", + "title": "Full Text Search" } ], - "resultChars": 61699 + "resultChars": 62362 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"vector extension schema best practice extensions schema\", limit: 3) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"pgvector hnsw index cosine distance best practices\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -5310,21 +5187,29 @@ "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started", - "title": "Getting Started with Realtime" + "url": "https://supabase.com/docs/guides/ai/vector-indexes/hnsw-indexes", + "title": "HNSW indexes" }, { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/ai/hybrid-search", + "title": "Hybrid search" + }, + { + "url": "https://supabase.com/docs/guides/ai/vector-indexes", + "title": "Vector indexes" + }, + { + "url": "https://supabase.com/docs/guides/ai/vector-indexes/ivf-indexes", + "title": "IVFFlat indexes" } ], - "resultChars": 46198 + "resultChars": 42868 } ] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/build-vectors-001-rag-with-permissions.json" }, { @@ -5354,12 +5239,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "Meets requirements: HTTPS Supabase Metrics API scrape at the correct path with project target, basic_auth using password_file, app scrape preserved, and docker-compose mounts the secrets directory containing the password file." + "judgeNotes": "Meets requirements: preserves app scrape, adds HTTPS Supabase Metrics API scrape for the project target with correct path and Basic Auth password_file, and docker-compose mounts the secrets directory containing that password file." }, { "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "README includes steps to create/use a Supabase Secret API key, place it in the mounted secret file matching password_file, reload/restart Prometheus via Compose or lifecycle reload, and verify via Prometheus targets showing the supabase job UP." + "judgeNotes": "README includes creating/copying a Supabase Secret API key, placing it in the password_file path mounted into Prometheus, restarting/reloading the Compose stack, and verifying the scrape via Prometheus Status → Targets with concrete failure modes." } ], "skills": { @@ -5367,21 +5252,27 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [ - "supabase" - ] + "loaded": [] }, "docs": { "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"project metrics prometheus endpoint observability\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"project metrics endpoint prometheus\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/reference/api/v1-scrape-project-metrics", + "title": "Scrape a project's metrics" + }, { "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", "title": "Metrics API" }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" + }, { "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", "title": "Vendor-agnostic Metrics API setup" @@ -5389,23 +5280,15 @@ { "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" } ], - "resultChars": 29057 + "resultChars": 20571 } ] }, "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/deploy-database-001-prometheus-metrics.json" }, { @@ -5458,68 +5341,11 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Functions environment variables secrets Deno.env deploy\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - } - ], - "resultChars": 38791 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"management API invoke edge function endpoint\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/architecture", - "title": "Edge Functions Architecture" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-functions", - "title": "Manage Supabase Edge functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - } - ], - "resultChars": 66572 - } - ] + "calls": [] }, "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/deploy-functions-001-edge-function-secrets.json" }, { @@ -5572,11 +5398,40 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"self-hosting with docker docker-compose setup\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting", + "title": "Self-Hosting" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17", + "title": "Upgrade to Postgres 17" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" + } + ], + "resultChars": 91121 + } + ] }, "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/deploy-self-hosting-001-docker-compose.json" }, { @@ -5599,7 +5454,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "victim session active before delete-account", @@ -5620,8 +5475,7 @@ }, { "name": "deleted user cannot sign back in", - "passed": false, - "notes": "deleted account can still sign in" + "passed": true }, { "name": "other users keep their sessions and access", @@ -5630,7 +5484,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "The answer identifies the soft-delete-only bug, implements real revocation by deleting auth.sessions and banning the auth user, adds RLS checks to close the data-access stale-JWT window, and consistently explains that stateless JWTs remain locally valid until expiry. It also correctly distinguishes frontend publishable/anon keys from server-only secret/service_role keys and notes secret bypasses RLS." + "judgeNotes": "Meets all rubric requirements: identifies soft-delete-only root cause, changes flow to delete auth.users and cascade/revoke sessions/refresh tokens, consistently explains remaining stateless JWT access-token window and mitigations, and correctly distinguishes publishable vs secret keys and RLS behavior." } ], "skills": { @@ -5643,44 +5497,112 @@ ] }, "docs": { - "calls": [] - }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-sonnet-5/investigate-auth-001-deleted-user-access.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "investigate-realtime-001-subscribed-no-events", - "stage": "investigate", - "product": [ - "realtime", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "orders table added to supabase_realtime publication", - "passed": true - }, - { - "name": "courier_locations still in supabase_realtime publication", - "passed": true - }, - { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"delete user account security definer function auth.users\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", + "title": "Build a User Management App with SvelteKit" + }, + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pgaudit", + "title": "PGAudit: Postgres Auditing" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/push-notifications", + "title": "Sending Push Notifications" + } + ], + "resultChars": 72582 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"revoke sessions sign out user access token still valid after delete\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", + "title": "OAuth 2.1 Flows" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-admin-signout", + "title": "signOut()" + } + ], + "resultChars": 49194 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"publishable key secret key vs anon key service_role migration RLS\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" + } + ], + "resultChars": 58595 + } + ] + }, + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5/investigate-auth-001-deleted-user-access.json" + }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "investigate-realtime-001-subscribed-no-events", + "stage": "investigate", + "product": [ + "realtime", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "orders table added to supabase_realtime publication", + "passed": true + }, + { + "name": "courier_locations still in supabase_realtime publication", + "passed": true + }, + { "name": "publication still publishes INSERT events", "passed": true }, @@ -5696,7 +5618,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "The assistant correctly identified the root cause as orders missing from the supabase_realtime publication despite SUBSCRIBED status, added only public.orders to the existing publication, preserved courier_locations/RLS/policies, and did not blame or alter client code, RLS, grants, or networking." + "judgeNotes": "Diagnosed the silent subscription as orders missing from the supabase_realtime publication despite SUBSCRIBED, added public.orders to the existing publication, and preserved courier_locations/RLS/policies. No disallowed fixes were applied." } ], "skills": { @@ -5713,7 +5635,7 @@ }, "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/investigate-realtime-001-subscribed-no-events.json" }, { @@ -5740,17 +5662,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform as the main affected function and described the recurring HTTP 503 pattern across the 2026-04-28 morning window, including the eight gateway failures from about 07:00Z to 12:00Z. Also correctly distinguished older billing-webhook 503s as unrelated." + "judgeNotes": "Identified image-transform as affected and described the recurring pattern of 8 HTTP 503 gateway failures across the morning of 2026-04-28, while distinguishing it from older billing-webhook 503s." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "Attributes image-transform 503s to upstream gateway/platform rejection rather than handler code, grounded in missing edge-function runtime logs for 503s while 200s appear, and distinguishes avatar-upload's runtime 500 as a separate function-level error." + "judgeNotes": "Attributes recurring image-transform 503s to gateway/infrastructure layer, not function code, and grounds this in valid observations: 503s only in gateway logs with no corresponding execution logs, nearby successful invocations, and distinction from avatar-upload's function-level 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant provided multiple concrete actionable next steps, including instrumenting memory/input sizes, checking for a recurring scheduler/batch job, adding retry/backoff, investigating the separate avatar-upload stack trace, and considering architectural changes for heavy image processing." + "judgeNotes": "The assistant recommended concrete next steps including checking Supabase status for the time window, adding retries, setting up alerting, reviewing concurrency/memory limits, and investigating package issues." } ], "skills": { @@ -5758,16 +5680,14 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [ - "supabase" - ] + "loaded": [] }, "docs": { "calls": [] }, "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/investigate-reliability-003-edge-function-5xx-correlation.json" }, { @@ -5821,7 +5741,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies causing deny-all Data API results, kept RLS enabled, created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for insert, and verified behavior." + "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies causing deny-all Data API results, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid(), with INSERT enforced via WITH CHECK. Verification was also performed." } ], "skills": { @@ -5838,7 +5758,7 @@ }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/resolve-dataapi-001-empty-results.json" }, { @@ -5885,7 +5805,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "Applied avatar_url via `supabase db push` in #11, which showed `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` Reconciled the orphan bio migration by adding local file `20240115000000_add_profile_bio.sql` in #9, after which `supabase migration list` matched local/remote and `db push` proceeded. No disallowed workaround or direct mutation observed; psql usage was read-only inspection." + "judgeNotes": "Avatar migration was applied through `supabase db push` in #16, with output `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` History was reconciled by adding the missing local migration file `20240115000000_add_profile_bio.sql` in #14, after which `supabase migration list` showed local/remote alignment in #15. No disallowed workaround or direct SQL mutation was used; psql commands were read-only inspection." } ], "skills": { @@ -5893,16 +5813,14 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [ - "supabase" - ] + "loaded": [] }, "docs": { "calls": [] }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/resolve-database-001-migration-history-mismatch.json" }, { @@ -5955,7 +5873,6 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", "supabase-postgres-best-practices" ] }, @@ -5964,7 +5881,7 @@ }, "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/resolve-performance-001-slow-query-cpu-spike.json" }, { @@ -6041,7 +5958,7 @@ }, "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/resolve-security-002-rls-cross-tenant-leak.json" }, { @@ -6076,7 +5993,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 863566c0-fe1b-4264-9906-53f26c545be8, signUp returned {\"userId\":\"863566c0-fe1b-4264-9906-53f26c545be8\"}" + "notes": "db user a835d031-9776-4a55-a5fa-90ba1f9484bc, signUp returned {\"userId\":\"a835d031-9776-4a55-a5fa-90ba1f9484bc\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -6091,7 +6008,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"863566c0-fe1b-4264-9906-53f26c545be8\"}" + "notes": "{\"userId\":\"a835d031-9776-4a55-a5fa-90ba1f9484bc\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -6118,7 +6035,7 @@ }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/build-auth-001-email-password-flow.json" }, { @@ -6155,7 +6072,7 @@ { "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "found 2 rows" + "notes": "found 3 rows" }, { "name": "row level security is enabled on todos", @@ -6173,7 +6090,7 @@ { "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "2 rows" + "notes": "3 rows" } ], "skills": { @@ -6185,7 +6102,7 @@ }, "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-001-bootstrap-app.json" }, { @@ -6236,7 +6153,7 @@ }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-002-declarative-schema.json" }, { @@ -6277,7 +6194,7 @@ { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 4) from the queue" + "notes": "function removed the seeded message (id 5) from the queue" } ], "skills": { @@ -6285,11 +6202,40 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"pg_cron trigger edge function with pgmq queue worker\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" + } + ], + "resultChars": 75234 + } + ] }, "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-003-pg-cron-queue-workflow.json" }, { @@ -6313,7 +6259,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": true, + "passed": false, "checks": [ { "name": "report runs and prints JSON", @@ -6332,8 +6278,8 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/report.mjs" + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" }, { "name": "report queries via the Data API, not raw SQL", @@ -6465,7 +6411,7 @@ }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/build-database-001-migrate-postgres-to-supabase.json" }, { @@ -6570,17 +6516,17 @@ { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"6f5b2171-8fac-4f4d-a612-8f5898da836f\",\"metric\":\"steps_a_msj0y6nu\",\"value\":111}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"c3c5a126-bf85-44dd-abdf-7c8576ca12b1\",\"metric\":\"steps_a_mst1sq80\",\"value\":111}]}" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"6f5b2171-8fac-4f4d-a612-8f5898da836f\",\"metric\":\"steps_a_msj0y6nu\",\"value\":111}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"c3c5a126-bf85-44dd-abdf-7c8576ca12b1\",\"metric\":\"steps_a_mst1sq80\",\"value\":111}]}" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"36482d33-6a70-43e3-bd27-459a7428cf03\",\"metric\":\"steps_b_msj0y6nu\",\"value\":222}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"9008d804-7ca3-4e8d-aef0-ce630f9cb2fa\",\"metric\":\"steps_b_mst1sq80\",\"value\":222}]}" }, { "name": "non-service key is not granted service access", @@ -6609,19 +6555,9 @@ }, "docs": { "calls": [ - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md 2>&1 | head -100", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 5516 - }, { "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Functions environment variables SUPABASE_SERVICE_ROLE_KEY SUPABASE_URL default secrets\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"edge function environment variables SUPABASE_URL SUPABASE_SERVICE_ROLE_KEY SUPABASE_ANON_KEY secret key publishable key\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -6636,44 +6572,20 @@ "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", "title": "Migrating to publishable and secret API keys" }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, { "url": "https://supabase.com/docs/guides/functions/auth", "title": "Securing Edge Functions" - } - ], - "resultChars": 41771 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"new API keys publishable secret key edge functions env var migration from anon service_role\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", - "title": "Backup and Restore using the CLI" + "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", + "title": "Build a Social Auth App with Expo React Native" } ], - "resultChars": 95976 + "resultChars": 94601 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"Authorization headers verify_jwt combining auth modes user secret apikey Edge Functions\", limit: 3) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"withSupabase @supabase/server edge function authMode supabaseAdmin ctx.user publishable secret\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -6681,21 +6593,29 @@ "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", + "title": "Which package to use" }, { "url": "https://supabase.com/docs/guides/getting-started/api-keys", "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" } ], - "resultChars": 24799 + "resultChars": 45543 } ] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/build-functions-005-dual-auth-user-secret.json" }, { @@ -6736,7 +6656,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019fdc8c-98ed-736c-a6e0-4477d1da6052/receipt-alpha.pdf, 019fdc8c-98ed-736c-a6e0-4477d1da6052/receipt-beta.pdf" + "notes": "saw: 01a000ac-ef89-709c-befe-181f338a8b2d/receipt-alpha.pdf, 01a000ac-ef89-709c-befe-181f338a8b2d/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -6757,7 +6677,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Meets requirements: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies on storage.objects with WITH CHECK for uploads, no RLS disabling or public access, and supabase-js createSignedUrl with expiry for temporary sharing." + "judgeNotes": "Meets requirements: creates a private user-files bucket, adds authenticated owner-scoped SELECT and INSERT policies on storage.objects with WITH CHECK for uploads, does not disable RLS or use permissive/public policies, and provides supabase-js createSignedUrl code with an expiry for temporary sharing." } ], "skills": { @@ -6769,7 +6689,7 @@ }, "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/build-storage-001-private-bucket-access.json" }, { @@ -6802,12 +6722,12 @@ { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "10 passed, 5 failed" + "notes": "3 passed, 3 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as having the tenant isolation flaw, explains the missing `org_id` predicate, and grounds the conclusion in the pgTAP failures showing cross-org post visibility. It does not blame `notes` and treats the test results as authoritative." + "judgeNotes": "The agent correctly identifies `posts` as having a broken tenant isolation SELECT policy, grounded in pgTAP results showing the cross-org posts read test failed (`have: 1, want: 0`). It also correctly distinguishes `notes` as passing isolation and treats the test results as authoritative." } ], "skills": { @@ -6819,7 +6739,7 @@ }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/build-tests-001-rls-tenant-isolation.json" }, { @@ -6882,11 +6802,32 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"gte-small embedding dimensions Supabase.ai Session\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", + "title": "Choosing your Compute Add-on" + }, + { + "url": "https://supabase.com/docs/guides/functions/ai-models", + "title": "Running AI Models" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" + } + ], + "resultChars": 52120 + } + ] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/build-vectors-001-rag-with-permissions.json" }, { @@ -6916,12 +6857,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "Meets all requirements: Supabase Metrics API scrape uses HTTPS, correct metrics path, Basic Auth with password_file, valid supabase.co project target, app scrape is preserved, and docker-compose mounts the secrets directory containing the password file." + "judgeNotes": "Meets all requirements: HTTPS Supabase metrics endpoint with correct path, Basic Auth using password_file, project target on supabase.co, existing app scrape preserved, and docker-compose mounts the password file." }, { "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "README includes Secret API key creation, matching secret file placement, Compose start/reload, and concrete verification via Prometheus targets." + "judgeNotes": "README includes Secret API key creation, placing it in the mounted secret file, recreating Prometheus/Compose, and verifying the Supabase target via Prometheus /targets." } ], "skills": { @@ -6935,6 +6876,10 @@ "query": "{ searchDocs(query: \"project metrics endpoint prometheus\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/reference/api/v1-scrape-project-metrics", + "title": "Scrape a project's metrics" + }, { "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", "title": "Metrics API" @@ -6950,19 +6895,15 @@ { "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/database/replication/manual-replication-monitoring", - "title": "Manual replication monitoring" } ], - "resultChars": 23548 + "resultChars": 20571 } ] }, "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/deploy-database-001-prometheus-metrics.json" }, { @@ -7010,26 +6951,55 @@ "loaded": [] }, "docs": { - "calls": [] - }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/deploy-functions-001-edge-function-secrets.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", - "product": [ - "database", + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"management API invoke edge function\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-functions", + "title": "Manage Supabase Edge functions" + }, + { + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", + "title": "Backup and Restore using the CLI" + } + ], + "resultChars": 93393 + } + ] + }, + "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", + "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5-no-skills/deploy-functions-001-edge-function-secrets.json" + }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "deploy-self-hosting-001-docker-compose", + "stage": "deploy", + "product": [ + "database", "auth", "storage" ], @@ -7066,7 +7036,7 @@ }, "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/deploy-self-hosting-001-docker-compose.json" }, { @@ -7120,7 +7090,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "Meets all rubric requirements: identifies soft-delete/no auth revocation root cause, implements session/refresh-token revocation plus sign-in blocking and RLS enforcement, explains remaining stateless JWT/local-validation window consistently with the RLS fix, and correctly distinguishes publishable vs secret keys and RLS behavior." + "judgeNotes": "Meets the rubric: diagnoses soft-delete-only cause, implements real auth revocation via ban plus session/refresh token deletion and RLS live-state checks, explains JWT expiry/local-validation window consistently with the RLS fix, and correctly distinguishes publishable frontend keys from server-only secret keys that bypass RLS." } ], "skills": { @@ -7181,7 +7151,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "The assistant correctly identified that the channel can reach SUBSCRIBED while INSERT events do not fire because public.orders was missing from the supabase_realtime publication. It fixed exactly that with ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, verified courier_locations remained included, and did not disable RLS or weaken policies." + "judgeNotes": "Identifies the missing orders table in supabase_realtime as the root cause, explains SUBSCRIBED vs Postgres Changes delivery, fixes with ALTER PUBLICATION ADD TABLE public.orders, and preserves courier_locations/RLS/policies without blaming or changing them." } ], "skills": { @@ -7193,7 +7163,7 @@ }, "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/investigate-realtime-001-subscribed-no-events.json" }, { @@ -7220,17 +7190,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "The assistant correctly identified `image-transform` as the affected function and described the recurring HTTP 503 pattern across the morning of 2026-04-28, including all 8 failures from 07:00Z through 12:00Z. It also avoided misattributing the main issue to the older `billing-webhook` 503s." + "judgeNotes": "Assistant identified `image-transform` as the affected function and described the recurring pattern of 8 HTTP 503 gateway responses across 07:00Z–12:00Z on 2026-04-28. It also correctly treated the older billing-webhook 503s as unrelated." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "Attributes the recurring 503s to gateway/platform/boot-level failure rather than application code, grounded in valid observations: no corresponding execution logs for failed requests, nearby successful invocations, unchanged deployment/version, and distinction from avatar-upload's function-level 500." + "judgeNotes": "Attributes image-transform 503s to the gateway/platform layer before function invocation, not application code, and grounds this in missing edge-function execution logs for the 503s, nearby successful invocations with same deployment/version, and distinction from avatar-upload's runtime 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended multiple concrete next steps: checking Edge Function dashboard logs for BOOT_ERROR/WORKER_RESOURCE_LIMIT in a specific time window, reviewing recent deployment img-deploy-42 and considering rollback, investigating resource limits/input size/concurrency, and adding alerting." + "judgeNotes": "The assistant provided specific actionable next steps, including investigating scaling behavior, correlating traffic volume for a precise time window, adding monitoring, and escalating to Supabase support with exact timestamps." } ], "skills": { @@ -7238,40 +7208,11 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge function 503 error worker boot\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-503-response", - "title": "Edge Function 503 error response" - }, - { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-504-error-response", - "title": "Edge Function 504 error response" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-546-error-response", - "title": "546 - WORKER_RESOURCE_LIMIT Exceeded / WORKER_LIMIT Exceeded" - } - ], - "resultChars": 43344 - } - ] + "calls": [] }, "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" }, { @@ -7325,7 +7266,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed RLS deny-all due to no policies, kept RLS enabled, and added authenticated-only SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." + "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies causing deny-all Data API results, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid(), with INSERT enforced via WITH CHECK." } ], "skills": { @@ -7337,7 +7278,7 @@ }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-dataapi-001-empty-results.json" }, { @@ -7384,7 +7325,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "PASS: avatar_url was applied through `supabase db push` in #14, with output `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration was reconciled by adding the missing local migration file in #12 (`20240115000000_add_profile_bio.sql`), after which `supabase migration list` in #13 showed local and remote histories aligned. psql usage was read-only inspection only; no disallowed workaround seen." + "judgeNotes": "Avatar migration was applied through `supabase db push` in action #21, which showed `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration history was reconciled by adding the missing local migration file `20240115000000_add_profile_bio.sql` in action #19, after which `supabase migration list` showed local/remote alignment. No prohibited workaround or direct remote mutation was used." } ], "skills": { @@ -7396,7 +7337,7 @@ }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-database-001-migration-history-mismatch.json" }, { @@ -7452,7 +7393,7 @@ }, "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-performance-001-slow-query-cpu-spike.json" }, { @@ -7524,7 +7465,7 @@ }, "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-security-002-rls-cross-tenant-leak.json" }, { @@ -7559,7 +7500,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 3d73a23b-4cc8-4789-9fd3-9abe325e1baa, signUp returned {\"userId\":\"3d73a23b-4cc8-4789-9fd3-9abe325e1baa\"}" + "notes": "db user d87c208e-12d1-4c6c-bf12-84e9c6a39792, signUp returned {\"userId\":\"d87c208e-12d1-4c6c-bf12-84e9c6a39792\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -7574,7 +7515,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"3d73a23b-4cc8-4789-9fd3-9abe325e1baa\"}" + "notes": "{\"userId\":\"d87c208e-12d1-4c6c-bf12-84e9c6a39792\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -7606,82 +7547,109 @@ "calls": [ { "source": "search_docs", - "query": "query {\n searchDocs(query: \"supabase-js signUp signInWithPassword getUser getSession auth admin createUser user_metadata display_name profiles\", limit: 5) {\n nodes {\n title\n href\n content\n ... on ClientLibraryFunctionReference {\n language\n methodName\n }\n }\n totalCount\n }\n}", + "query": "query { searchDocs(query: \"supabase-js signUp signIn getUser getSession select from profiles auth uid\", limit: 5) { nodes { title href content ... on ClientLibraryFunctionReference { language methodName } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" + "url": "https://supabase.com/docs/reference/javascript/auth-getuser" }, { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-react", + "title": "Build a User Management App with React" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" + "url": "https://supabase.com/docs/guides/auth/social-login/auth-google", + "title": "Login with Google" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-apple", - "title": "Login with Apple" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", + "title": "Build a User Management App with Angular" } ], - "resultChars": 79187 + "resultChars": 155442 }, { "source": "search_docs", - "query": "query {\n searchDocs(query: \"signInWithPassword supabase-js reference signUp options data getUser getSession auth.currentUser\", limit: 10) {\n nodes {\n title\n href\n content\n ... on ClientLibraryFunctionReference {\n language\n methodName\n }\n }\n totalCount\n }\n}", + "query": "query { searchDocs(query: \"JavaScript signInWithPassword auth.signInWithPassword\", limit: 5) { nodes { title href content ... on ClientLibraryFunctionReference { language methodName } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/auth-getuser" + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" }, { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" + "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" }, { "url": "https://supabase.com/docs/guides/auth/passwords", "title": "Password-based Auth" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-google", - "title": "Login with Google" + "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", + "title": "signInWithPassword()" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpasskey" + } + ], + "resultChars": 50333 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"JavaScript select single profiles auth.getUser\", limit: 5) { nodes { title href content ... on ClientLibraryFunctionReference { language methodName } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-getuser" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-azure", - "title": "Login with Azure (Microsoft)" + "url": "https://supabase.com/docs/reference/javascript/auth-admin-getuserbyid" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" + "url": "https://supabase.com/docs/reference/javascript/auth-admin-listusers" }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-createuser" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-updateuser" + } + ], + "resultChars": 3832 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"JavaScript signUp auth.signUp signInWithPassword\", limit: 5) { nodes { title href content ... on ClientLibraryFunctionReference { language methodName } } } }", + "hasContent": true, + "pages": [ { "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" }, + { + "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", + "title": "signInWithPassword()" + }, { "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", - "title": "Login with Figma" + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpasskey" } ], - "resultChars": 193019 + "resultChars": 50333 } ] }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini/build-auth-001-email-password-flow.json" }, { @@ -7753,68 +7721,87 @@ "calls": [ { "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,120p'\"", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | rg -n \"breaking-change|''^#|''^##\" -m 80'", "hasContent": true, "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 6745 + "resultChars": 6416 }, { "source": "search_docs", - "query": "query {\n searchDocs(query: \"local development migrations RLS select authenticated anon expose table data api\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", + "query": "query { searchDocs(query: \"row level security authenticated select policy public schema anon authenticated\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } totalCount } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" + "url": "https://supabase.com/docs/guides/realtime/authorization", + "title": "Realtime Authorization" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes", + "title": "Postgres Changes" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter", - "title": "Build a User Management App with Flutter" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" } ], - "resultChars": 152845 + "resultChars": 136371 }, { "source": "search_docs", - "query": "query {\n searchDocs(query: \"Securing your API grants authenticated anon RLS select policy\", limit: 3) {\n nodes { title href content }\n }\n}", + "query": "query { searchDocs(query: \"Exposing a Table to the Data API RLS authenticated anon select policy\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } totalCount } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/api/securing-your-api", "title": "Securing your API" }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - }, { "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0017_foreign_table_in_api", + "title": "Database Advisor: Lint 0017_foreign_table_in_api" + }, + { + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0026_pg_graphql_anon_table_exposed", + "title": "Database Advisor: Lint 0026_pg_graphql_anon_table_exposed" + }, + { + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0027_pg_graphql_authenticated_table_exposed", + "title": "Database Advisor: Lint 0027_pg_graphql_authenticated_table_exposed" + } + ], + "resultChars": 68596 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog/45329-breaking-change-tables-not-exposed-to-data-and-graphql-api-automatically | sed -n '1,220p'\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog/45329-breaking-change-tables-not-exposed-to-data-and-graphql-api-automatically" } ], - "resultChars": 63887 + "resultChars": 157740 } ] }, "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini/build-cli-001-bootstrap-app.json" }, { @@ -7870,7 +7857,7 @@ "calls": [ { "source": "search_docs", - "query": "query SearchDocs {\n searchDocs(query: \"declarative database schemas local development migration generate supabase schema_paths\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", + "query": "query { searchDocs(query: \"declarative database schemas migration local development\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -7882,32 +7869,25 @@ "title": "Local development workflow" }, { - "url": "https://supabase.com/docs/guides/deployment/branching/working-with-branches", - "title": "Working with branches" + "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", + "title": "Building an MCP Server with mcp-lite" }, { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - } - ], - "resultChars": 65847 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,120p'\"", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" + }, { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", + "title": "Build a User Management App with SvelteKit" } ], - "resultChars": 6745 + "resultChars": 115077 } ] }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini/build-cli-002-declarative-schema.json" }, { @@ -7948,7 +7928,7 @@ { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 36) from the queue" + "notes": "function removed the seeded message (id 5) from the queue" } ], "skills": { @@ -7964,92 +7944,21 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"pg_cron create cron job every minute run function Supabase local queue pgmq pop delete\", limit: 5) {\n nodes {\n title\n href\n ... on Guide { content }\n ... on Subsection { content }\n ... on CLICommandReference { content }\n ... on ClientLibraryFunctionReference { content }\n ... on TroubleshootingGuide { content }\n ... on ManagementApiReference { content }\n }\n }\n}", - "hasContent": true, - "pages": [] - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"pg_cron create cron job every minute Supabase queue pgmq pop delete\", limit: 10) {\n nodes {\n title\n href\n content\n }\n }\n}", + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -n 40'", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - }, - { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_cron", - "title": "pg_cron: Schedule Recurring Jobs with Cron Syntax in Postgres" - }, - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 69751 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"pg_cron unschedule jobname cron.schedule same name idempotent\", limit: 5) {\n nodes { title href content }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/pg_cron-launcher-crashes-with-duplicate-key-value-violates-unique-constraint-cc6472", - "title": "`pg_cron launcher crashes with 'duplicate key value violates unique constraint'`" - }, - { - "url": "https://supabase.com/docs/guides/platform/upgrading", - "title": "Upgrading" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" - } - ], - "resultChars": 42853 + "resultChars": 2161 } ] }, "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini/build-cli-003-pg-cron-queue-workflow.json" }, { @@ -8114,15 +8023,152 @@ "docs": { "calls": [ { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,120p'\"", + "source": "search_docs", + "query": "query { searchDocs(query: \"service role REST API select rows Supabase\", limit: 5) { nodes { __typename title href content ... on Guide { subsections { nodes { title href content } } } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/api/creating-routes", + "title": "Creating API Routes" + }, + { + "url": "https://supabase.com/docs/guides/api/creating-routes#api-url-and-keys", + "title": "API URL and keys" + }, + { + "url": "https://supabase.com/docs/guides/api/creating-routes#using-the-api", + "title": "Using the API" + }, + { + "url": "https://supabase.com/docs/guides/api/creating-routes#create-a-table", + "title": "Create a table" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api#default-privileges-for-new-tables-and-functions", + "title": "Default privileges for new tables and functions" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api#accessing-request-information", + "title": "Accessing request information" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api#enforce-additional-rules-on-each-request", + "title": "Enforce additional rules on each request" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api#add-rls-policies", + "title": "Add RLS policies" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api#examples", + "title": "Examples" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api#disable-the-data-api", + "title": "Disable the Data API" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api#use-a-dedicated-api-schema", + "title": "Use a dedicated API schema" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api#grant-access-explicitly", + "title": "Grant access explicitly" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data#edge-functions", + "title": "Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data#data-api", + "title": "Data API" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data#connecting-your-app-securely", + "title": "Connecting your app securely" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data#more-information", + "title": "More information" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data#frontend-access", + "title": "Frontend access" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data#direct-database-connections", + "title": "Direct database connections" + }, + { + "url": "https://supabase.com/docs/guides/api", + "title": "Data REST API" + }, + { + "url": "https://supabase.com/docs/guides/api#rest-api-overview", + "title": "Features " + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-1-create-the-new-api-keys", + "title": "Step 1: Create the new API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-2-swap-the-publishable-key-in-client-code", + "title": "Step 2: Swap the publishable key in client code" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-3-swap-the-secret-key-in-backend-code", + "title": "Step 3: Swap the secret key in backend code" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#database-webhooks-and-pg_net", + "title": "Database Webhooks and pg_net" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-4-update-edge-functions", + "title": "Step 4: Update Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-2-adopt-the-supabaseserver-sdk", + "title": "Option 2: Adopt the @supabase/server SDK" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-1-read-the-new-keys-from-the-environment", + "title": "Option 1: Read the new keys from the environment" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-5-verify-nothing-uses-the-legacy-keys", + "title": "Step 5: Verify nothing uses the legacy keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-6-deactivate-the-legacy-keys", + "title": "Step 6: Deactivate the legacy keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#known-limitations", + "title": "Known limitations" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#next-steps", + "title": "Next steps" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#before-you-start", + "title": "Before you start" } ], - "resultChars": 6745 + "resultChars": 99025 } ] }, @@ -8191,36 +8237,7 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase local psql database query cli worker node supabase-js examples\", limit: 5) { edges { node { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on CLICommandReference { title href content } ... on TroubleshootingGuide { title href content } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" - }, - { - "url": "https://supabase.com/docs/guides/local-development", - "title": "Local Development & CLI" - }, - { - "url": "https://supabase.com/docs/guides/database/inspect", - "title": "Debugging and monitoring" - } - ], - "resultChars": 36819 - } - ] + "calls": [] }, "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", @@ -8281,50 +8298,35 @@ }, "docs": { "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,140p'\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 7789 - }, { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase db start --from-backup logical backup pg_restore dump\", limit: 5) { nodes { title href content } } }", + "query": "query {\n searchDocs(query: \"local development restore pg_restore binary dump supabase cli\", limit: 5) {\n nodes {\n ... on Guide {\n title\n href\n content\n }\n ... on CLICommandReference {\n title\n href\n content\n }\n }\n }\n}", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/troubleshooting/download-logical-backups", - "title": "How to download logical backups in Supabase with physical backups enabled?" + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" }, { "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", "title": "Restore a Platform Project to Self-Hosted" }, { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/transferring-from-cloud-to-self-host-in-supabase-2oWNvW", - "title": "Transferring from platform to self-hosted Supabase" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" }, { - "url": "https://supabase.com/docs/guides/platform/backups", - "title": "Database Backups" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/heroku", + "title": "Migrate from Heroku to Supabase" } ], - "resultChars": 24731 + "resultChars": 37911 } ] }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini/build-database-001-migrate-postgres-to-supabase.json" }, { @@ -8384,55 +8386,153 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions verify_jwt auth.getUser Bearer token\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"Edge Functions verify_jwt getUser auth.uid Supabase\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, { "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#fetching-the-user", + "title": "Fetching the user" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#setting-up-auth-context", + "title": "Setting up auth context" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#example", + "title": "Example" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#combining-modes", + "title": "Combining modes" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#custom-error-responses", + "title": "Custom error responses" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#environment-variables", + "title": "Environment variables" }, { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" + "url": "https://supabase.com/docs/guides/functions/auth#authenticated-user-calls", + "title": "Authenticated user calls" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#service-to-service-calls", + "title": "Service-to-service calls" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#public-functions", + "title": "Public functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#external-webhooks", + "title": "External webhooks" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-mfa", + "title": "Multi-Factor Authentication" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-mfa#enforce-rules-for-mfa-logins", + "title": "Enforce rules for MFA logins" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-mfa#add-unenroll-flow", + "title": "Add unenroll flow" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-mfa#adding-to-your-app", + "title": "Adding to your app" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-mfa#overview", + "title": "Overview" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-mfa#apis", + "title": "APIs" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-mfa#server-side-rendering", + "title": "Server-Side Rendering" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-mfa#enforce-only-for-users-that-have-opted-in", + "title": "Enforce only for users that have opted-in" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-mfa#enforce-for-new-users-only", + "title": "Enforce for new users only" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-mfa#enforce-for-all-users-new-and-existing", + "title": "Enforce for all users (new and existing)" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-mfa#database", + "title": "Database" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-mfa#example-react", + "title": "Example: React" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-mfa#frequently-asked-questions", + "title": "Frequently asked questions" + }, + { + "url": "https://supabase.com/docs/guides/functions/storage-caching", + "title": "Integrating with Supabase Storage" + }, + { + "url": "https://supabase.com/docs/guides/functions/storage-caching#basic-file-operations", + "title": "Basic file operations" + }, + { + "url": "https://supabase.com/docs/guides/functions/storage-caching#cache-first-pattern", + "title": "Cache-first pattern" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-headers#understanding-authorization-headers", + "title": "Understanding authorization headers" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-headers#the-verify_jwt-platform-check", + "title": "The verify_jwt platform check" } ], - "resultChars": 32605 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,120p'\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 6745 + "resultChars": 77575 } ] }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini/build-functions-004-service-role-bypass.json" }, { @@ -8474,17 +8574,17 @@ { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"9c85b93d-750c-4545-ad08-2e86b35ad9a6\",\"metric\":\"steps_a_msj0xq5b\",\"value\":111}]}" + "notes": "status 200: [{\"user_id\":\"03a1dad4-8c1a-4dca-a84c-68ed328e87e1\",\"metric\":\"steps_a_mst1sldc\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"9c85b93d-750c-4545-ad08-2e86b35ad9a6\",\"metric\":\"steps_a_msj0xq5b\",\"value\":111}]}" + "notes": "status 200: [{\"user_id\":\"03a1dad4-8c1a-4dca-a84c-68ed328e87e1\",\"metric\":\"steps_a_mst1sldc\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"af4aed6d-bffa-426b-98f4-2f78b0fb1835\",\"metric\":\"steps_b_msj0xq5b\",\"value\":222}]}" + "notes": "status 200: [{\"user_id\":\"f3014ae6-adbb-4357-9541-470b8a801640\",\"metric\":\"steps_b_mst1sldc\",\"value\":222}]" }, { "name": "non-service key is not granted service access", @@ -8521,201 +8621,85 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions auth service_role key verify supabase access token request header user_id body\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", + "query": "query {\n searchDocs(query: \"edge functions verify jwt service_role apikey header supabase-js createClient getUser access token auth header\", limit: 5) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n ... on CLICommandReference { title href content }\n ... on ManagementApiReference { title href content }\n }\n }\n}", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", "title": "Integrating With Supabase Auth" }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#example", - "title": "Example" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#setting-up-auth-context", - "title": "Setting up auth context" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#fetching-the-user", - "title": "Fetching the user" - }, { "url": "https://supabase.com/docs/guides/getting-started/api-keys", "title": "Understanding API keys" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#security-considerations", - "title": "Security considerations" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#what-secret-keys-allow-access-to", - "title": "What secret keys allow access to" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#known-limitations-and-compatibility-differences", - "title": "Known limitations and compatibility differences" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#best-practices-for-handling-secret-keys", - "title": "Best practices for handling secret keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#what-to-do-if-a-secret-key-or-service_role-has-been-leaked-or-compromised", - "title": "What to do if a secret key or service_role has been leaked or compromised?" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#overview", - "title": "Overview" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#publishable-keys", - "title": "Publishable keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#interaction-with-supabase-auth", - "title": "Interaction with Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth#public-functions", - "title": "Public functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth#external-webhooks", - "title": "External webhooks" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth#combining-modes", - "title": "Combining modes" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth#custom-error-responses", - "title": "Custom error responses" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth#environment-variables", - "title": "Environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth#authenticated-user-calls", - "title": "Authenticated user calls" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth#service-to-service-calls", - "title": "Service-to-service calls" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { "url": "https://supabase.com/docs/guides/functions/auth-headers", "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers#understanding-authorization-headers", - "title": "Understanding authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers#the-verify_jwt-platform-check", - "title": "The verify_jwt platform check" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#next-steps", - "title": "Next steps" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#why-this-pattern-works", - "title": "Why this pattern works" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#browser-client", - "title": "Browser client" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#edge-function-websocket-proxy", - "title": "Edge Function (WebSocket proxy)" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#database-schema", - "title": "Database schema" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#architecture", - "title": "Architecture" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" } ], - "resultChars": 91747 + "resultChars": 46673 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase functions serve verify_jwt false config.toml edge function\", limit: 5) { nodes { title href content } } }", + "query": "query {\n searchDocs(query: \"@supabase/server userClaims id sub authMode supabaseAdmin createSupabaseContext withSupabase\", limit: 10) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n ... on CLICommandReference { title href content }\n ... on ManagementApiReference { title href content }\n }\n }\n}", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/functions/deploy", - "title": "Deploy to Production" + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", - "title": "Building an MCP Server with mcp-lite" - } - ], - "resultChars": 32331 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions service role key env var SUPABASE_SERVICE_ROLE_KEY secret key Deno.env\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", + "title": "Which package to use" + }, { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/database/postgres/roles", + "title": "Postgres Roles" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/auth/jwts", + "title": "JSON Web Token (JWT)" }, { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" + "url": "https://supabase.com/docs/guides/auth/third-party/auth0", + "title": "Auth0" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" } ], - "resultChars": 44193 + "resultChars": 129974 } ] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini/build-functions-005-dual-auth-user-secret.json" }, { @@ -8756,7 +8740,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019fdc8d-2d5c-7238-bdd6-e937eefa55fb/receipt-alpha.pdf, 019fdc8d-2d5c-7238-bdd6-e937eefa55fb/receipt-beta.pdf" + "notes": "saw: 01a000ad-ccf2-75c8-9ac1-33a7c1e79111/receipt-alpha.pdf, 01a000ad-ccf2-75c8-9ac1-33a7c1e79111/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -8777,7 +8761,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Creates private user-files bucket, owner-scoped SELECT and INSERT policies for authenticated role using first folder segment = auth.uid(), keeps RLS intact (no disabling), and provides createSignedUrl with a 15-minute expiry. Extra UPDATE/DELETE policies are acceptable." + "judgeNotes": "Configured a private user-files bucket, kept/enabled RLS on storage.objects, added authenticated owner/folder-scoped SELECT and INSERT policies, and provided supabase-js createSignedUrl code with an expiry for temporary sharing." } ], "skills": { @@ -8794,189 +8778,179 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"storage bucket policies objects path owner user id signed URL supabase\", limit: 5) { nodes { __typename title href ... on Guide { content } ... on TroubleshootingGuide { content } ... on ClientLibraryFunctionReference { content language methodName } } } }", + "query": "query { searchDocs(query: \"Supabase Storage RLS policies bucket_id owner path auth.uid() storage.objects\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-expo-react-native", + "title": "Build a User Management App with Expo React Native" }, { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/storage", - "title": "Storage" + "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", + "title": "Storage Helper Functions" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-swift", + "title": "Build a User Management App with Swift and SwiftUI" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", - "title": "Build a User Management App with SvelteKit" + "url": "https://supabase.com/docs/guides/storage/schema/design", + "title": "The Storage Schema" } ], - "resultChars": 119861 + "resultChars": 69824 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"storage.objects policy owner path prefix foldername user id signed URL createSignedUrl\", limit: 10) { nodes { __typename title href ... on Guide { content } ... on TroubleshootingGuide { content } ... on ClientLibraryFunctionReference { content language methodName } } } }", + "query": "query { searchDocs(query: \"storage policy create bucket user specific path upload download signed URLs Supabase\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" - }, - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" - }, - { - "url": "https://supabase.com/docs/guides/storage/cdn/smart-cdn", - "title": "Smart CDN" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-cant-i-uploadlistetc-my-public-bucket-Z6CmGt", - "title": "Why can't I upload/list/etc my public bucket?" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/reference/swift/storage-from-createsigneduploadurl", - "title": "from.createSignedUploadURL()" + "url": "https://supabase.com/docs/guides/storage", + "title": "Storage" }, { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" }, { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", + "title": "Build a User Management App with SvelteKit" }, { - "url": "https://supabase.com/docs/reference/swift/storage-from-createsignedurls", - "title": "from.createSignedUrls()" + "url": "https://supabase.com/docs/guides/storage/s3/compatibility", + "title": "S3 Compatibility" } ], - "resultChars": 17609 + "resultChars": 151961 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"storage access control policy examples foldername auth uid owner\", limit: 10) { nodes { __typename title href ... on Guide { content } ... on TroubleshootingGuide { content } } } }", + "query": "query { searchDocs(query: \"storage signed url private bucket download createSignedUrl supabase-js\", limit: 10) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" }, { - "url": "https://supabase.com/docs/guides/storage/security/ownership", - "title": "Ownership" + "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations", + "title": "Manage Storage Image Transformations usage" }, { - "url": "https://supabase.com/docs/guides/storage/quickstart", - "title": "Storage Quickstart" + "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", + "title": "Storage Image Transformations" }, { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" + "url": "https://supabase.com/docs/guides/storage/uploads/resumable-uploads", + "title": "Resumable Uploads" }, { - "url": "https://supabase.com/docs/guides/storage/management/copy-move-objects", - "title": "Copy Objects" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" }, { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" }, { "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", "title": "Storage Buckets" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/storage/s3/compatibility", + "title": "S3 Compatibility" + }, + { + "url": "https://supabase.com/docs/reference/swift/storage-from-createsigneduploadurl", + "title": "from.createSignedUploadURL()" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/supabase-storage-inefficient-folder-operations-and-hierarchical-rls-challenges-b05a4d", - "title": "Supabase Storage: Inefficient folder operations and hierarchical RLS challenges" + "url": "https://supabase.com/docs/guides/storage/management/copy-move-objects", + "title": "Copy Objects" } ], - "resultChars": 60503 + "resultChars": 91293 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"storage object ownership owner column path prefix foldername access control\", limit: 10) { nodes { __typename title href ... on Guide { content } ... on TroubleshootingGuide { content } } } }", + "query": "query { searchDocs(query: \"storage.objects owner policy authenticated select bucket_id path prefix auth.uid\", limit: 10) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/security/ownership", - "title": "Ownership" + "url": "https://supabase.com/docs/guides/storage/management/copy-move-objects", + "title": "Copy Objects" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/supabase-storage-inefficient-folder-operations-and-hierarchical-rls-challenges-b05a4d", - "title": "Supabase Storage: Inefficient folder operations and hierarchical RLS challenges" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/storage/management/copy-move-objects", - "title": "Copy Objects" + "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", + "title": "Custom Roles" }, { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" }, { - "url": "https://supabase.com/docs/guides/storage/schema/design", - "title": "The Storage Schema" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" }, { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" + "url": "https://supabase.com/docs/guides/storage/cdn/fundamentals", + "title": "Storage CDN" }, { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - } + "url": "https://supabase.com/docs/guides/storage/security/ownership", + "title": "Ownership" + }, + { + "url": "https://supabase.com/docs/guides/ai-tools/mcp", + "title": "Supabase MCP Server" + } ], - "resultChars": 32342 + "resultChars": 41901 }, { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js createSignedUrl storage from bucket expiresIn share file temporary link\", limit: 3) { nodes { __typename title href ... on ClientLibraryFunctionReference { content language methodName } } } }", - "hasContent": true, + "source": "web_search", + "query": "site:supabase.com changelog.md Supabase breaking changes storage createSignedUrl bucket policies August 2026", + "pages": [] + }, + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", "pages": [ { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - }, - { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations", - "title": "Manage Storage Image Transformations usage" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" + "url": "https://supabase.com/changelog.md" } - ], - "resultChars": 1476 + ] + }, + { + "source": "web_search", + "query": "Supabase changelog.md supabase.com changelog breaking-change storage signed urls", + "pages": [] + }, + { + "source": "web_search", + "query": "site:supabase.com/docs supabase storage private buckets signed URL createSignedUrl", + "pages": [] } ] }, "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini/build-storage-001-private-bucket-access.json" }, { @@ -9009,12 +8983,12 @@ { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "4 passed, 0 failed" + "notes": "10 passed, 0 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identified `posts` as the broken tenant isolation policy, grounded it in the pgTAP result (`cross-org posts` check failed), and did not blame `notes` or dismiss the tests. It also noted `notes` passed and treated the test results as authoritative." + "judgeNotes": "The agent correctly identified `posts` as having a tenant isolation flaw (`posts` ignores `org_id` / any authenticated member can read posts outside their org) and did not blame `notes` or dismiss pgTAP. It also reported pgTAP verification after the fix." } ], "skills": { @@ -9031,76 +9005,216 @@ "calls": [ { "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,160p'\"", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n \\\"breaking-change|supabase test|pgTAP|RLS|policy\\\" | sed -n '1,120p'\"", "hasContent": true, "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 1140 + "resultChars": 5372 }, { "source": "search_docs", - "query": "query {\n searchDocs(query: \"pgTAP tests RLS Supabase local database tests\", limit: 5) {\n nodes {\n ... on Guide {\n title\n href\n content\n }\n ... on TroubleshootingGuide {\n title\n href\n content\n }\n ... on CLICommandReference {\n title\n href\n content\n }\n }\n totalCount\n }\n}", + "query": "query { searchDocs(query: \"Supabase CLI test database pgTAP SQL tests local development\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/reference/cli/supabase-test-db", + "title": "Tests local database with pgTAP" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", + "title": "Testing and linting" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting#testing-your-edge-functions", + "title": "Testing your Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting#testing-auth-emails", + "title": "Testing Auth emails" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting#accessing-mailpit", + "title": "Accessing Mailpit" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting#going-into-production", + "title": "Going into production" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting#linting-your-database", + "title": "Linting your database" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting#testing-your-database", + "title": "Testing your database" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting#test-helpers", + "title": "Test helpers" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting#running-database-tests-in-ci", + "title": "Running database tests in CI" + }, { "url": "https://supabase.com/docs/guides/local-development/testing/overview", "title": "Testing Overview" }, { - "url": "https://supabase.com/docs/guides/database/testing", - "title": "Testing Your Database" + "url": "https://supabase.com/docs/guides/local-development/testing/overview#real-world-examples", + "title": "Real-World examples" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" + "url": "https://supabase.com/docs/guides/local-development/testing/overview#troubleshooting", + "title": "Troubleshooting" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/guides/local-development/testing/overview#additional-resources", + "title": "Additional resources" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/overview#continuous-integration-testing", + "title": "Continuous integration testing" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/overview#test-isolation-strategies", + "title": "Test isolation strategies" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/overview#application-level-testing", + "title": "Application-Level testing" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/overview#database-unit-testing-with-pgtap", + "title": "Database unit testing with pgTAP" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/overview#testing-approaches", + "title": "Testing approaches" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/overview#best-practices", + "title": "Best practices" }, { "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", "title": "Advanced pgTAP Testing" - } - ], - "resultChars": 68853 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"request.jwt.claims auth.uid set local pgtap Supabase\", limit: 5) {\n nodes {\n ... on Guide {\n title\n href\n content\n }\n }\n totalCount\n }\n}", - "hasContent": true, - "pages": [ + }, { - "url": "https://supabase.com/docs/guides/local-development/testing/overview", - "title": "Testing Overview" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#1-app-schema-definitions", + "title": "1. App schema definitions" }, { - "url": "https://supabase.com/docs/guides/realtime/authorization", - "title": "Realtime Authorization" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#using-databasedev", + "title": "Using database.dev" }, { - "url": "https://supabase.com/docs/guides/auth/third-party/firebase-auth", - "title": "Firebase Auth" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#setting-up-dbdev", + "title": "Setting up dbdev" }, { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#installing-test-helpers", + "title": "Installing test helpers" }, { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#test-helper-benefits", + "title": "Test helper benefits" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#schema-wide-row-level-security-testing", + "title": "Schema-wide Row Level Security testing" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#test-file-organization", + "title": "Test file organization" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#creating-a-pre-test-hook", + "title": "Creating a pre-test hook" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#benefits", + "title": "Benefits" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#example-advanced-rls-testing", + "title": "Example: Advanced RLS testing" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#not-another-todo-app-testing-complex-organizations", + "title": "Not another todo app: Testing complex organizations" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#system-overview", + "title": "System overview" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#what-makes-this-complex", + "title": "What makes this complex?" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#testing-focus-areas", + "title": "Testing focus areas" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#2-grant-role-privileges", + "title": "2. Grant role privileges" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#3-rls-policies-declaration", + "title": "3. RLS policies declaration" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#4-test-cases", + "title": "4. Test cases:" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#additional-resources", + "title": "Additional resources" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pgtap", + "title": "pgTAP: Unit Testing" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pgtap#resources", + "title": "Resources" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pgtap#testing-rls-policies", + "title": "Testing RLS policies" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pgtap#testing-columns", + "title": "Testing columns" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pgtap#testing-tables", + "title": "Testing tables" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pgtap#enable-the-extension", + "title": "Enable the extension" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pgtap#overview", + "title": "Overview" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pgtap#testing-functions", + "title": "Testing functions" } ], - "resultChars": 66662 + "resultChars": 146550 } ] }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini/build-tests-001-rls-tenant-isolation.json" }, { @@ -9124,12 +9238,38 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { - "name": "created auth sessions", - "passed": false, - "notes": "Internal server error" + "name": "document_sections.embedding is vector(384)", + "passed": true, + "notes": "vector(384)" + }, + { + "name": "HNSW index on the embedding column", + "passed": true, + "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" + }, + { + "name": "index operator class matches the search operator", + "passed": true, + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" + }, + { + "name": "user A search returns only own sections, best match first", + "passed": true + }, + { + "name": "user B search returns only own sections, best match first", + "passed": true + }, + { + "name": "user A reads only own sections through the API", + "passed": true + }, + { + "name": "user A reads only own documents through the API", + "passed": true } ], "skills": { @@ -9144,9 +9284,23 @@ }, "docs": { "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "web_search", + "query": "site:supabase.com changelog.md supabase", + "pages": [] + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"semantic search pgvector row level security documents ownership\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href methodName language } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"pgvector semantic search row level security ownership documents\", limit: 5) { nodes { title href content ... on Guide { subsections { totalCount } } } } }", "hasContent": true, "pages": [ { @@ -9158,47 +9312,47 @@ "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors", - "title": "Querying Vectors" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pgvector", - "title": "pgvector: Embeddings and vector similarity" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" } ], - "resultChars": 68480 + "resultChars": 98626 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"pgvector extension vector schema extensions match_documents function RLS authenticated ownership\", limit: 10) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href methodName language } } } }", + "query": "query { searchDocs(query: \"gte-small dimensions Supabase ai Session\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/functions/ai-models", + "title": "Running AI Models" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pgvector", - "title": "pgvector: Embeddings and vector similarity" + "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", + "title": "Choosing your Compute Add-on" }, { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" + "url": "https://supabase.com/docs/guides/ai-tools/plugins", + "title": "Supabase Plugin for AI Coding Agents" }, { - "url": "https://supabase.com/docs/guides/storage/vector/introduction", - "title": "Vector Buckets" + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" }, { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" + "url": "https://supabase.com/docs/guides/ai/examples/headless-vector-search", + "title": "Adding generative Q&A for your documentation" } ], - "resultChars": 49023 + "resultChars": 59363 } ] }, @@ -9234,12 +9388,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": false, - "judgeNotes": "prometheus.yml does not add a deployable Supabase scrape job. It relies on an entrypoint/env injection not shown, uses SUPABASE_SECRET_API_KEY instead of an HTTP Basic Auth password_file, and docker-compose.yml does not mount that password_file via a volume or Compose secret. The existing app job is preserved, but required Supabase scrape wiring is missing." + "judgeNotes": "Fails: Supabase scrape uses basic_auth.password with a placeholder/hardcoded secret instead of password_file, and docker-compose.yml does not mount/provide that password file via a volume or Compose secret." }, { "name": "documented live deployment and verification steps", "passed": false, - "judgeNotes": "README explains env-based setup and restart, but it does not require placing a matching secret file, and it lacks concrete verification steps via Prometheus targets, PromQL/Grafana, or equivalent." + "judgeNotes": "README explains endpoint, creating/copying a Secret API key, restart/reload, and Prometheus target verification, but it instructs users to paste the key directly into prometheus.yml. The rubric requires placing the matching secret file; the Compose/config setup has no secret file handling, so the secret setup is missing/mismatched." } ], "skills": { @@ -9256,23 +9410,9 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase metrics Prometheus project metrics\", limit: 5) { nodes { __typename title href content ... on Guide { subsections { nodes { title href content } } } ... on TroubleshootingGuide { } } totalCount } }", - "hasContent": true, - "pages": [] - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"metrics prometheus\", limit: 5) { nodes { title href content } totalCount } }", + "query": "query {\n searchDocs(query: \"Supabase metrics Prometheus project metrics endpoint observability Grafana\", limit: 5) {\n nodes {\n __typename\n ... on Guide { title href content }\n ... on TroubleshootingGuide { title href content }\n ... on ManagementApiReference { title href content }\n ... on CLICommandReference { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n }\n }\n}", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, { "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", "title": "Metrics API" @@ -9282,11 +9422,19 @@ "title": "Vendor-agnostic Metrics API setup" }, { - "url": "https://supabase.com/docs/guides/database/replication/manual-replication-monitoring", - "title": "Manual replication monitoring" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" + }, + { + "url": "https://supabase.com/docs/guides/database/connection-management", + "title": "Connection management" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" } ], - "resultChars": 23603 + "resultChars": 25917 } ] }, @@ -9328,7 +9476,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get('WEATHER_API_KEY')." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -9341,340 +9489,97 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions environment variables secrets deploy supabase secrets set function env var runtime\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", + "query": "query { searchDocs(query: \"Edge Functions environment variables secrets deploy supabase functions secrets set\", limit: 5) { nodes { title href ... on Guide { content } ... on CLICommandReference { content } ... on ClientLibraryFunctionReference { content methodName language } } totalCount } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", + "title": "Streaming Speech with ElevenLabs" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#x-forwarded-headers", - "title": "X-Forwarded headers" + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#forwarded-headers-and-cors", - "title": "Forwarded headers and CORS" + "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", + "title": "Inspecting edge function environment variables" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#opaque-key-translation", - "title": "Opaque key translation" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#api-key-enforcement-on-protected-routes", - "title": "API key enforcement on protected routes" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#dashboard-basic-auth", - "title": "Dashboard basic auth" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#authentication", - "title": "Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#routes", - "title": "Routes" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#how-the-configuration-is-rendered-at-startup", - "title": "How the configuration is rendered at startup" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#configuration-file-structure", - "title": "Configuration file structure" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#architecture", - "title": "Architecture" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#verify", - "title": "Verify" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#enabling-the-envoy-gateway", - "title": "Enabling the Envoy gateway" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#before-you-begin", - "title": "Before you begin" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#see-also", - "title": "See also" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#common-issues", - "title": "Common issues" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#logs", - "title": "Logs" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#admin-interface", - "title": "Admin interface" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#customizing-the-configuration", - "title": "Customizing the configuration" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#security-hardening", - "title": "Security hardening" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#cors", - "title": "CORS" - }, + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", + "title": "Transcription Telegram Bot" + } + ], + "resultChars": 39334 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase Edge Functions verify_jwt public function secrets set env-file WEATHER_API_KEY\", limit: 5) { nodes { title href ... on Guide { content } ... on CLICommandReference { content } } totalCount } }", + "hasContent": true, + "pages": [ { "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", "title": "Migrating to publishable and secret API keys" }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-2-adopt-the-supabaseserver-sdk", - "title": "Option 2: Adopt the @supabase/server SDK" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-5-verify-nothing-uses-the-legacy-keys", - "title": "Step 5: Verify nothing uses the legacy keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-6-deactivate-the-legacy-keys", - "title": "Step 6: Deactivate the legacy keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#known-limitations", - "title": "Known limitations" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#next-steps", - "title": "Next steps" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-1-read-the-new-keys-from-the-environment", - "title": "Option 1: Read the new keys from the environment" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#before-you-start", - "title": "Before you start" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-1-create-the-new-api-keys", - "title": "Step 1: Create the new API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-2-swap-the-publishable-key-in-client-code", - "title": "Step 2: Swap the publishable key in client code" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-3-swap-the-secret-key-in-backend-code", - "title": "Step 3: Swap the secret key in backend code" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#database-webhooks-and-pg_net", - "title": "Database Webhooks and pg_net" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-4-update-edge-functions", - "title": "Step 4: Update Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#run-locally", - "title": "Run locally" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#code-the-supabase-edge-function", - "title": "Code the Supabase Edge Function" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#dependencies", - "title": "Dependencies" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#set-up-the-environment-variables", - "title": "Set up the environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#create-a-supabase-edge-function-for-speech-generation", - "title": "Create a Supabase Edge Function for speech generation" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#configure-background-tasks-for-supabase-edge-functions", - "title": "Configure background tasks for Supabase Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#configure-the-storage-bucket", - "title": "Configure the storage bucket" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#create-a-supabase-project-locally", - "title": "Create a Supabase project locally" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#setup", - "title": "Setup" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#requirements", - "title": "Requirements" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#introduction", - "title": "Introduction" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#test-the-function", - "title": "Test the function" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#set-the-function-secrets", - "title": "Set the function secrets" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#deploy-to-supabase", - "title": "Deploy to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#try-it-out", - "title": "Try it out" - }, { "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", "title": "Transcription Telegram Bot" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#dependencies", - "title": "Dependencies" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#test-the-bot", - "title": "Test the bot" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#set-the-function-secrets", - "title": "Set the function secrets" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#set-up-the-webhook", - "title": "Set up the webhook" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#apply-the-database-migrations", - "title": "Apply the database migrations" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#deploy-to-supabase", - "title": "Deploy to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#code-the-telegram-bot", - "title": "Code the Telegram bot" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#set-up-the-environment-variables", - "title": "Set up the environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#create-a-supabase-edge-function-to-handle-telegram-webhook-requests", - "title": "Create a Supabase Edge Function to handle Telegram webhook requests" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#create-a-database-table-to-log-the-transcription-results", - "title": "Create a database table to log the transcription results" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#create-a-supabase-project-locally", - "title": "Create a Supabase project locally" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#register-a-telegram-bot", - "title": "Register a Telegram bot" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#setup", - "title": "Setup" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#requirements", - "title": "Requirements" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#introduction", - "title": "Introduction" + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", + "title": "Streaming Speech with ElevenLabs" }, { "url": "https://supabase.com/docs/guides/functions/secrets", "title": "Environment Variables" }, { - "url": "https://supabase.com/docs/guides/functions/secrets#default-secrets", - "title": "Default secrets" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets#production-secrets", - "title": "Production secrets" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets#local-secrets", - "title": "Local secrets" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets#accessing-environment-variables", - "title": "Accessing environment variables" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" } ], - "resultChars": 149695 + "resultChars": 46922 }, { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Edge Functions JWT verification no-verify-jwt public browser invoke\", limit: 5) { nodes { title href content } } }", - "hasContent": true, + "source": "web_search", + "query": "\"Project not specified\" \"sb-gateway-version\" Supabase functions", + "pages": [] + }, + { + "source": "web_search", + "query": "https://supabase.com/docs/guides/functions/quickstart", "pages": [ { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" + "url": "https://supabase.com/docs/guides/functions/quickstart" } - ], - "resultChars": 47787 + ] + }, + { + "source": "web_search", + "query": "\"sb-project-ref\" Supabase functions gateway", + "pages": [] + }, + { + "source": "web_search", + "query": "\"Project not specified\" \"sb-project-ref: null\" supabase edge gateway", + "pages": [] } ] }, "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini/deploy-functions-001-edge-function-secrets.json" }, { @@ -9730,24 +9635,52 @@ "docs": { "calls": [ { - "source": "web_search", - "query": "site:supabase.com/changelog.md Supabase breaking-change self-hosting docker compose", - "pages": [] + "source": "search_docs", + "query": "query {\n searchDocs(query: \"self-hosting docker compose environment variables supabase\", limit: 5) {\n edges {\n node {\n __typename\n ... on Guide {\n title\n href\n content\n }\n ... on TroubleshootingGuide {\n title\n href\n content\n }\n ... on CLICommandReference {\n title\n href\n content\n }\n }\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting", + "title": "Self-Hosting" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/remove-superuser-access", + "title": "Remove Superuser Access from Studio" + } + ], + "resultChars": 66993 }, { "source": "web_search", - "query": "https://supabase.com/docs/guides/self-hosting/docker", + "query": "https://supabase.com/changelog.md", "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/docker" + "url": "https://supabase.com/changelog.md" } ] + }, + { + "source": "web_search", + "query": "site:supabase.com changelog.md supabase", + "pages": [] } ] }, "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini/deploy-self-hosting-001-docker-compose.json" }, { @@ -9770,11 +9703,12 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "victim session active before delete-account", - "passed": true + "passed": false, + "notes": "permission denied for table users" }, { "name": "delete_account flow ran for the victim", @@ -9795,12 +9729,13 @@ }, { "name": "other users keep their sessions and access", - "passed": true + "passed": false, + "notes": "permission denied for table users" }, { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "Meets the rubric: identifies soft-delete-only bug, updates flow to delete auth user and lock down RLS, explains stale JWT caveat consistently with the RLS mitigation, and correctly distinguishes publishable/anon vs secret/service_role keys." + "judgeNotes": "Meets rubric: identifies soft-delete-only bug, implements auth user deletion with session/identity cascade, adds RLS live-user checks and explains resulting data-path vs stateless JWT window consistently, and correctly distinguishes publishable/anon from secret/service_role keys and RLS bypass." } ], "skills": { @@ -9817,45 +9752,68 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"delete user revoke sessions auth admin sign out user delete account\", limit: 5) { nodes { title href ... on Guide { content } ... on TroubleshootingGuide { content } ... on ClientLibraryFunctionReference { methodName language content href } } } }", + "query": "query { searchDocs(query: \"delete user sessions access tokens deleted user Supabase\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" }, { "url": "https://supabase.com/docs/guides/auth/managing-user-data", "title": "User Management" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", + "title": "OAuth 2.1 Flows" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-signout" + "url": "https://supabase.com/docs/guides/platform/sso", + "title": "Enable SSO for Your Organization" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" } ], - "resultChars": 20977 + "resultChars": 112526 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"publishable key secret key anon key service role frontend rls\", limit: 5) { nodes { title href ... on Guide { content } ... on ClientLibraryFunctionReference { methodName language content href } } } }", + "query": "query { searchDocs(query: \"auth.sessions session_id validate deleted user access tokens Supabase\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" }, { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/auth/jwts", + "title": "JSON Web Token (JWT)" + }, + { + "url": "https://supabase.com/docs/guides/platform/temporary-access", + "title": "Temporary access" + }, + { + "url": "https://supabase.com/docs/guides/auth/users", + "title": "Users" + } + ], + "resultChars": 51189 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"publishable keys secret keys anon service_role frontend RLS Supabase\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" }, { "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", @@ -9864,37 +9822,26 @@ { "url": "https://supabase.com/docs/guides/getting-started/api-keys", "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" } ], - "resultChars": 68789 - }, - { - "source": "web_search", - "query": "site:supabase.com/changelog.md Supabase changelog markdown", - "pages": [] - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [] + "resultChars": 74600 }, { "source": "web_search", - "query": "site:supabase.com/docs/guides/getting-started/api-keys Supabase publishable secret key anon service_role", + "query": "site:supabase.com/docs/guides/auth/managing-user-data deleting a user does not automatically sign out JWT will remain valid until expired", "pages": [] }, { "source": "web_search", - "query": "https://supabase.com/docs/guides/getting-started/api-keys", - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys" - } - ] - }, - { - "source": "web_search", - "query": "'Deleting users' in https://supabase.com/docs/guides/auth/managing-user-data", + "query": "site:supabase.com/docs new publishable secret keys legacy anon service_role equivalent frontend backend", "pages": [] } ] @@ -9950,7 +9897,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Identified the root cause as orders missing from supabase_realtime despite successful subscription, fixed by ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, and preserved courier_locations plus RLS/policies." + "judgeNotes": "The assistant correctly identified that orders was missing from the supabase_realtime publication despite subscription success, added public.orders with ALTER PUBLICATION supabase_realtime ADD TABLE, verified courier_locations remained, and did not weaken RLS or policies." } ], "skills": { @@ -9967,52 +9914,32 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Realtime postgres_changes publication supabase_realtime table not receiving events\", limit: 5) { nodes { __typename title href content ... on Guide { subsections { nodes { title href content } } } } } }", + "query": "query { searchDocs(query: \"postgres_changes INSERT realtime publication table orders Supabase\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", - "title": "Subscribing to Database Changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#broadcast-authorization", - "title": "Broadcast authorization" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#streaming-updates", - "title": "Streaming updates" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#streaming-inserts", - "title": "Streaming inserts" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#enable-postgres-changes", - "title": "Enable Postgres Changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#using-postgres-changes", - "title": "Using Postgres Changes" + "url": "https://supabase.com/docs/guides/realtime/benchmarks", + "title": "Benchmarks" }, { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#listening-on-client-side", - "title": "Listening on client side" + "url": "https://supabase.com/docs/guides/realtime/benchmarks#10kb-payload", + "title": "10KB payload" }, { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#create-a-trigger", - "title": "Create a trigger" + "url": "https://supabase.com/docs/guides/realtime/benchmarks#50kb-payload", + "title": "50KB payload" }, { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#create-a-trigger-function", - "title": "Create a trigger function" + "url": "https://supabase.com/docs/guides/realtime/benchmarks#broadcast-scalability-scenarios", + "title": "Broadcast: Scalability scenarios" }, { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#using-broadcast", - "title": "Using Broadcast" + "url": "https://supabase.com/docs/guides/realtime/benchmarks#realtime-auth", + "title": "Realtime Auth" }, { - "url": "https://supabase.com/docs/guides/realtime/benchmarks", - "title": "Benchmarks" + "url": "https://supabase.com/docs/guides/realtime/benchmarks#postgres-changes", + "title": "Postgres Changes" }, { "url": "https://supabase.com/docs/guides/realtime/benchmarks#methodology", @@ -10043,136 +9970,156 @@ "title": "1KB payload" }, { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#10kb-payload", - "title": "10KB payload" + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", + "title": "Subscribing to Database Changes" }, { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#50kb-payload", - "title": "50KB payload" + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#streaming-updates", + "title": "Streaming updates" }, { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#broadcast-scalability-scenarios", - "title": "Broadcast: Scalability scenarios" + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#streaming-inserts", + "title": "Streaming inserts" }, { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#realtime-auth", - "title": "Realtime Auth" + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#enable-postgres-changes", + "title": "Enable Postgres Changes" }, { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#postgres-changes", - "title": "Postgres Changes" + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#using-postgres-changes", + "title": "Using Postgres Changes" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes", - "title": "Postgres Changes" + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#listening-on-client-side", + "title": "Listening on client side" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#refreshed-tokens", - "title": "Refreshed tokens" + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#create-a-trigger", + "title": "Create a trigger" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#custom-tokens", - "title": "Custom tokens" + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#create-a-trigger-function", + "title": "Create a trigger function" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#private-schemas", - "title": "Private schemas" + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#broadcast-authorization", + "title": "Broadcast authorization" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#receiving-old-records", - "title": "Receiving old records" + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#using-broadcast", + "title": "Using Broadcast" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#quick-start", - "title": "Quick start" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes", + "title": "Postgres Changes" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#usage", - "title": "Usage" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes#not-equal-to-neq", + "title": "Not equal to (neq)" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-specific-schemas", - "title": "Listening to specific schemas" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes#equal-to-eq", + "title": "Equal to (eq)" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-insert-events", - "title": "Listening to INSERT events" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes#available-filters", + "title": "Available filters" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-update-events", - "title": "Listening to UPDATE events" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes#filtering-for-specific-changes", + "title": "Filtering for specific changes" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-delete-events", - "title": "Listening to DELETE events" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-multiple-changes", + "title": "Listening to multiple changes" + }, + { + "url": "https://supabase.com/docs/guides/realtime/postgres-changes#limitations", + "title": "Limitations" }, { "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-specific-tables", "title": "Listening to specific tables" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-multiple-changes", - "title": "Listening to multiple changes" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes#delete-events-are-not-filterable", + "title": "Delete events are not filterable" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#filtering-for-specific-changes", - "title": "Filtering for specific changes" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes#spaces-in-table-names", + "title": "Spaces in table names" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#available-filters", - "title": "Available filters" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes#database-instance-and-realtime-performance", + "title": "Database instance and realtime performance" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#equal-to-eq", - "title": "Equal to (eq)" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes#refreshed-tokens", + "title": "Refreshed tokens" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#not-equal-to-neq", - "title": "Not equal to (neq)" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes#custom-tokens", + "title": "Custom tokens" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#less-than-lt", - "title": "Less than (lt)" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes#private-schemas", + "title": "Private schemas" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#less-than-or-equal-to-lte", - "title": "Less than or equal to (lte)" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes#receiving-old-records", + "title": "Receiving old records" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#greater-than-gt", - "title": "Greater than (gt)" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes#contained-in-list-in", + "title": "Contained in list (in)" }, { "url": "https://supabase.com/docs/guides/realtime/postgres-changes#greater-than-or-equal-to-gte", "title": "Greater than or equal to (gte)" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#contained-in-list-in", - "title": "Contained in list (in)" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes#greater-than-gt", + "title": "Greater than (gt)" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#database-instance-and-realtime-performance", - "title": "Database instance and realtime performance" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes#less-than-or-equal-to-lte", + "title": "Less than or equal to (lte)" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#spaces-in-table-names", - "title": "Spaces in table names" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes#quick-start", + "title": "Quick start" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#delete-events-are-not-filterable", - "title": "Delete events are not filterable" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes#usage", + "title": "Usage" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#limitations", - "title": "Limitations" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-specific-schemas", + "title": "Listening to specific schemas" + }, + { + "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-insert-events", + "title": "Listening to INSERT events" + }, + { + "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-update-events", + "title": "Listening to UPDATE events" + }, + { + "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-delete-events", + "title": "Listening to DELETE events" + }, + { + "url": "https://supabase.com/docs/guides/realtime/postgres-changes#less-than-lt", + "title": "Less than (lt)" }, { "url": "https://supabase.com/docs/guides/realtime/protocol", "title": "Realtime Protocol" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#join-errors", - "title": "Join errors" + "url": "https://supabase.com/docs/guides/realtime/protocol#broadcast-text-frame", + "title": "broadcast (text frame)" }, { "url": "https://supabase.com/docs/guides/realtime/protocol#websocket-connection-setup", @@ -10230,10 +10177,6 @@ "url": "https://supabase.com/docs/guides/realtime/protocol#access_token", "title": "access_token" }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#broadcast-text-frame", - "title": "broadcast (text frame)" - }, { "url": "https://supabase.com/docs/guides/realtime/protocol#broadcast-binary-frame", "title": "broadcast (binary frame)" @@ -10286,6 +10229,10 @@ "url": "https://supabase.com/docs/guides/realtime/protocol#error-handling", "title": "Error handling" }, + { + "url": "https://supabase.com/docs/guides/realtime/protocol#join-errors", + "title": "Join errors" + }, { "url": "https://supabase.com/docs/guides/realtime/protocol#channel-level-system-errors", "title": "Channel-level system errors" @@ -10311,123 +10258,259 @@ "title": "Reconnection" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started", - "title": "Getting Started with Realtime" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#when-to-use-presence", - "title": "When to use Presence" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#when-to-use-logical-replication", + "title": "When to use logical replication" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#quick-start", - "title": "Quick start" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#source-postgres-prerequisites", + "title": "Source Postgres prerequisites" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#framework-examples", - "title": "Framework examples" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#access--privileges", + "title": "Access & privileges" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#integration-guides", - "title": "Integration guides" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#required-settings-for-logical-replication", + "title": "Required settings for logical replication" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#advanced-topics", - "title": "Advanced topics" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#replica-identity", + "title": "Replica identity" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#security--configuration", - "title": "Security & configuration" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#non-replicated-items", + "title": "Non-Replicated items" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#core-features", - "title": "Core features" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#step-1-configure-source-database", + "title": "Step 1: Configure source database" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#next-steps", - "title": "Next steps" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#postgresconf", + "title": "Postgres.conf" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#when-to-use-postgres-changes", - "title": "When to use Postgres Changes" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#pg_hbaconf", + "title": "pg_hba.conf" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#when-to-use-broadcast", - "title": "When to use Broadcast" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#step-2-verify-configuration", + "title": "Step 2: Verify configuration" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#choose-the-right-feature", - "title": "Choose the right feature" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#step-3-check-and-set-replica-identity", + "title": "Step 3: Check and set replica identity" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#clean-up-subscriptions", - "title": "Clean up subscriptions" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#set-connection-and-restore", + "title": "Set connection and restore" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#follow-naming-conventions", - "title": "Follow naming conventions" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#step-5-post-migration-tasks", + "title": "Step 5: Post-Migration tasks" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#use-private-channels", - "title": "Use private channels" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#update-statistics-important", + "title": "Update statistics (important)" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#essential-best-practices", - "title": "Essential best practices" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#verify-migration", + "title": "Verify migration" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#53-using-database-triggers", - "title": "5.3 using database triggers" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#re-enable-writes-on-source-if-keeping-it", + "title": "Re-enable writes on source (if keeping it)" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#52-using-httprest-api", - "title": "5.2 using HTTP/REST API" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#migration-time-estimates", + "title": "Migration time estimates" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#51-using-client-libraries", - "title": "5.1 using client libraries" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#important-notes", + "title": "Important notes" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#5-send-and-receive-messages", - "title": "5. Send and receive messages" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#method-3-logical-replication", + "title": "Method 3: Logical replication" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#4-set-up-authorization", - "title": "4. Set up authorization" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#step-8-synchronize-sequences", + "title": "Step 8: Synchronize sequences" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#3-create-your-first-channel", - "title": "3. Create your first Channel" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#step-7-monitor-replication-status", + "title": "Step 7: Monitor replication status" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#get-api-details", - "title": "Get API details" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#step-6-create-subscription-on-supabase", + "title": "Step 6: Create subscription on Supabase" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#2-initialize-the-client", - "title": "2. Initialize the client" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#step-5-create-publication-on-source", + "title": "Step 5: Create publication on source" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#1-install-the-client-library", - "title": "1. Install the client library" - } - ], - "resultChars": 389763 - } - ] - }, - "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", - "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/investigate-realtime-001-subscribed-no-events.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#step-4-export-and-restore-schema-only", + "title": "Step 4: Export and restore schema only" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#connection-modes", + "title": "Connection modes" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#method-1-google-colab-easiest", + "title": "Method 1: Google Colab (easiest)" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#method-2-manual-dumprestore", + "title": "Method 2: Manual dump/restore" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#prerequisites", + "title": "Prerequisites" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#source-postgres-requirements", + "title": "Source Postgres requirements" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#migration-environment", + "title": "Migration environment" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#pre-migration-checklist", + "title": "Pre-Migration checklist" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#check-available-extensions-in-supabase", + "title": "Check available extensions in Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#step-1-set-up-migration-vm", + "title": "Step 1: Set up migration VM" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#set-up-ubuntu-vm", + "title": "Set up Ubuntu VM" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#step-2-prepare-supabase-project", + "title": "Step 2: Prepare Supabase project" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#step-3-create-database-dump", + "title": "Step 3: Create database dump" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#set-source-database-to-read-only-mode-for-production-migration", + "title": "Set source database to read only mode for production migration" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#dump-the-database", + "title": "Dump the database" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#recommended-parallelization--j-values", + "title": "Recommended parallelization (-j values)" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#step-4-restore-to-supabase", + "title": "Step 4: Restore to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#getting-help", + "title": "Getting help" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#when-to-use-which-method", + "title": "When to use which method" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#important-limitations", + "title": "Important limitations" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#troubleshooting-logical-replication", + "title": "Troubleshooting logical replication" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#step-10-cleanup", + "title": "Step 10: Cleanup" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres#step-9-switch-to-supabase", + "title": "Step 9: Switch to Supabase" + } + ], + "resultChars": 319091 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"postgres_changes realtime published tables publication supabase\", limit: 10) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/realtime/architecture", + "title": "Realtime Architecture" + }, + { + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", + "title": "Subscribing to Database Changes" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/database/replication", + "title": "Database replication" + }, + { + "url": "https://supabase.com/docs/guides/database/replication/pipelines", + "title": "Set up Pipelines" + }, + { + "url": "https://supabase.com/docs/guides/realtime/authorization", + "title": "Realtime Authorization" + }, + { + "url": "https://supabase.com/docs/guides/realtime/getting_started", + "title": "Getting Started with Realtime" + }, + { + "url": "https://supabase.com/docs/guides/realtime/concepts", + "title": "Realtime Concepts" + }, + { + "url": "https://supabase.com/docs/guides/database/replication/pipelines-faq", + "title": "Pipelines FAQ" + }, + { + "url": "https://supabase.com/docs/guides/realtime/benchmarks", + "title": "Benchmarks" + } + ], + "resultChars": 125898 + } + ] + }, + "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", + "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini/investigate-realtime-001-subscribed-no-events.json" + }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" }, "eval": "investigate-reliability-003-edge-function-5xx-correlation", "stage": "investigate", @@ -10439,22 +10522,22 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform as the affected function and described repeated HTTP 503 gateway responses throughout the morning of 2026-04-28 from 07:00Z to 12:00Z, while distinguishing unrelated issues." + "judgeNotes": "Identified image-transform as affected and listed the recurring 503 pattern across the morning of 2026-04-28, covering all 8 gateway failures from 07:00Z to 12:00Z." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": true, - "judgeNotes": "Attributes recurring image-transform 503s to the edge/API gateway/platform layer rather than function code, and grounds this in API/gateway 503s alongside successful edge-function executions, plus distinguishes the avatar-upload 500 as a separate function-level error." + "passed": false, + "judgeNotes": "The assistant does not clearly attribute the recurring image-transform 503s to the gateway/Edge Functions platform layer. It frames them as likely in the upload function path, transient runtime/dependency issues, or possibly gateway/upstream, and recommends inspecting function code/dependencies, which fails the rubric." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended concrete next steps, including pulling fuller traces for failing request windows, checking Supabase edge/runtime incidents or rollouts, adding retry/backoff, and investigating a specific isolated 500 code path." + "judgeNotes": "Recommended concrete next steps including inspecting function dependencies, reviewing structured error logging, checking external services/storage operations, and adding retry handling." } ], "skills": { @@ -10463,7 +10546,8 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { @@ -10471,7 +10555,7 @@ }, "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini/investigate-reliability-003-edge-function-5xx-correlation.json" }, { @@ -10525,7 +10609,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies as deny-all for Data API, kept RLS enabled, created authenticated SELECT policy scoped to user_id = auth.uid(), and created authenticated INSERT policy with WITH CHECK enforcing user_id = auth.uid()." + "judgeNotes": "Diagnosed RLS deny-all due to no policies, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." } ], "skills": { @@ -10542,37 +10626,37 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Exposing a Table to the Data API RLS bookmarks anon authenticated GRANT\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on CLICommandReference { title href content } } } }", + "query": "query {\n searchDocs(query: \"RLS policy auth.uid() select insert own rows authenticated bookmarks\", limit: 5) {\n nodes {\n title\n href\n content\n ... on ClientLibraryFunctionReference {\n language\n methodName\n }\n }\n }\n}", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" }, { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", + "title": "Column Level Security" }, { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0017_foreign_table_in_api", - "title": "Database Advisor: Lint 0017_foreign_table_in_api" + "url": "https://supabase.com/docs/guides/realtime/getting_started", + "title": "Getting Started with Realtime" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/guides/realtime/authorization", + "title": "Realtime Authorization" }, { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0013_rls_disabled_in_public", - "title": "Database Advisor: Lint 0013_rls_disabled_in_public" + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" } ], - "resultChars": 58013 + "resultChars": 75067 } ] }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini/resolve-dataapi-001-empty-results.json" }, { @@ -10594,23 +10678,27 @@ ], "suite": "benchmark", "interface": "cli", - "passed": true, + "passed": false, "checks": [ { "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true + "passed": false, + "notes": "avatar_url not found on public.profiles" }, { "name": "migration 20240220000000 is recorded in the remote history", - "passed": true + "passed": false, + "notes": "remote history versions: [\"20240101000000\",\"20240115000000\"]" }, { "name": "remote migration history matches local migration files", - "passed": true + "passed": false, + "notes": "remote-only versions still unreconciled: [\"20240115000000\"] (remote: [\"20240101000000\",\"20240115000000\"], local: [\"20240101000000\",\"20240220000000\"])" }, { "name": "local migrations are a valid reconciled sequence", - "passed": true + "passed": false, + "notes": "expected 20240101000000_create_profiles → → 20240220000000_add_avatar_url, strictly ascending; got [20240101000000_create_profiles.sql, 20240220000000_add_avatar_url.sql]" }, { "name": "production profile data is intact (not reset)", @@ -10618,8 +10706,8 @@ }, { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": true, - "judgeNotes": "Applied avatar_url via `supabase db push --db-url \"$DB_URL\"` (#42), which showed `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` Reconciled orphan bio history by adding local migration `supabase/migrations/20240115000000_add_profile_bio.sql` (#36-37), after which `supabase migration list --db-url` showed local/remote aligned (#39, #45). Only read-only psql inspection observed; no prohibited workaround." + "passed": false, + "judgeNotes": "No successful remote `supabase db push` occurred; the only push attempts were dry-runs and failed with IPv6/DNS errors (#25, #28). No `supabase migration repair`, `supabase db pull`, or successful push reconciliation occurred. The only avatar_url application shown was direct local `psql` SQL against 127.0.0.1 (#50), not a Supabase CLI remote migration workflow." } ], "skills": { @@ -10637,7 +10725,7 @@ }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini/resolve-database-001-migration-history-mismatch.json" }, { @@ -10677,7 +10765,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -10695,32 +10783,11 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"composite index query optimization order by desc limit user_id created_at\", limit: 5) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n ... on ManagementApiReference { title href content }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" - }, - { - "url": "https://supabase.com/docs/guides/database/orioledb", - "title": "OrioleDB Overview" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/indexes", - "title": "Managing Indexes in Postgres" - } - ], - "resultChars": 18373 - } - ] + "calls": [] }, "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini/resolve-performance-001-slow-query-cpu-spike.json" }, { @@ -10797,37 +10864,37 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"row level security policy auth.uid select using with check memberships supabase\", limit: 5) { nodes { __typename title href content } } }", + "query": "query { searchDocs(query: \"Supabase row level security multi tenant select policy using auth.uid workspace_id\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/realtime/authorization", - "title": "Realtime Authorization" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" + "url": "https://supabase.com/docs/guides/auth/third-party/firebase-auth", + "title": "Firebase Auth" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/guides/auth/enterprise-sso/auth-sso-saml", + "title": "Single Sign-On with SAML 2.0 for Projects" }, { "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", "title": "RAG with Permissions" }, { - "url": "https://supabase.com/docs/guides/auth/third-party/firebase-auth", - "title": "Firebase Auth" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" } ], - "resultChars": 75855 + "resultChars": 89494 } ] }, "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini/resolve-security-002-rls-cross-tenant-leak.json" }, { @@ -10862,7 +10929,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user fbab494f-6a4b-4376-b26a-10e30c44a54d, signUp returned {\"userId\":\"fbab494f-6a4b-4376-b26a-10e30c44a54d\"}" + "notes": "db user 862c812c-0be5-4eb7-8c98-994e0c9b64d3, signUp returned {\"userId\":\"862c812c-0be5-4eb7-8c98-994e0c9b64d3\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -10877,7 +10944,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"fbab494f-6a4b-4376-b26a-10e30c44a54d\"}" + "notes": "{\"userId\":\"862c812c-0be5-4eb7-8c98-994e0c9b64d3\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -10900,7 +10967,18 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "web_search", + "query": "site:supabase.com/docs auth signUp raw_user_meta_data display_name options data signup REST supabase auth signup", + "pages": [] + }, + { + "source": "web_search", + "query": "site:supabase.com/docs /auth/v1/signup options data display_name REST", + "pages": [] + } + ] }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", @@ -10971,7 +11049,7 @@ }, "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-001-bootstrap-app.json" }, { @@ -11010,7 +11088,7 @@ }, { "name": "description column exists in the live database", - "passed": false + "passed": true } ], "skills": { @@ -11063,7 +11141,7 @@ { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 6) from the queue" + "notes": "function removed the seeded message (id 3) from the queue" } ], "skills": { @@ -11073,66 +11151,88 @@ "docs": { "calls": [ { - "source": "web_search", - "query": "Supabase pg_cron schedule cron.schedule syntax official docs", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs pgmq create queue function create queue", - "pages": [] - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "source": "search_docs", + "query": "query { searchDocs(query: \"pg_cron schedule unschedule idempotent Supabase cron.schedule job name\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } ... on ManagementApiReference { title href content } ... on ClientLibraryFunctionReference { title href content methodName language } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions" + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_cron", + "title": "pg_cron: Schedule Recurring Jobs with Cron Syntax in Postgres" } - ] - }, - { - "source": "web_search", - "query": "'pop(' in https://supabase.com/docs/guides/queues/pgmq", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs cron.schedule pg_cron supabase example", - "pages": [] - }, - { - "source": "web_search", - "query": "site:github.com supabase pg_cron cron.schedule supabase migration", - "pages": [] - }, - { - "source": "web_search", - "query": "'if not exists' in https://supabase.com/docs/guides/queues/pgmq", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs pg_cron unschedule cron.schedule idempotent", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs pgmq message_record msg_id message field", - "pages": [] + ], + "resultChars": 27114 }, { - "source": "web_search", - "query": "site:supabase.com/docs edge function config.toml verify_jwt false", - "pages": [] - } - ] - }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-003-pg-cron-queue-workflow.json" - }, + "source": "search_docs", + "query": "query { searchDocs(query: \"pgmq message_record msg_id delete pop read Supabase queue\", limit: 10) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" + }, + { + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" + }, + { + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", + "title": "Expose Queues for local and self-hosted Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/realtime/broadcast", + "title": "Broadcast" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/42501--permission-denied-for-table-httprequestqueue-KnozmQ", + "title": "42501 : permission denied for table http_request_queue" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase", + "title": "Migrating to Supabase" + } + ], + "resultChars": 102403 + } + ] + }, + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-003-pg-cron-queue-workflow.json" + }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", @@ -11306,7 +11406,7 @@ }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini-no-skills/build-database-001-migrate-postgres-to-supabase.json" }, { @@ -11352,12 +11452,12 @@ { "name": "user A cannot force-read user B note", "passed": true, - "notes": "status=403" + "notes": "status=200" }, { "name": "user B cannot force-read user A note", "passed": true, - "notes": "status=403" + "notes": "status=200" } ], "skills": { @@ -11365,40 +11465,11 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Edge Functions verify_jwt auth getUser service role private notes\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - } - ], - "resultChars": 39139 - } - ] + "calls": [] }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini-no-skills/build-functions-004-service-role-bypass.json" }, { @@ -11425,7 +11496,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { "name": "seed rows present", @@ -11440,17 +11511,17 @@ { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"a9708acd-ab70-4e53-8f37-66b0ed3c7434\",\"metric\":\"steps_a_msj11abm\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"008b373d-f012-4e05-a163-dc2d5327b9e8\",\"metric\":\"steps_a_mst1udcj\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: [{\"user_id\":\"a9708acd-ab70-4e53-8f37-66b0ed3c7434\",\"metric\":\"steps_a_msj11abm\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"008b373d-f012-4e05-a163-dc2d5327b9e8\",\"metric\":\"steps_a_mst1udcj\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", - "passed": false, - "notes": "status 401: {\"error\":\"Unauthorized\"}" + "passed": true, + "notes": "status 200: [{\"user_id\":\"0aeeddac-cf23-42ae-be0e-96bbee04045e\",\"metric\":\"steps_b_mst1udcj\",\"value\":222}]" }, { "name": "non-service key is not granted service access", @@ -11460,7 +11531,7 @@ { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"error\":\"Unauthorized\"}" + "notes": "status 401: {\"error\":\"Invalid or expired access token\"}" }, { "name": "a user token in the apikey slot is not treated as the service key", @@ -11469,8 +11540,8 @@ }, { "name": "implementation uses @supabase/server", - "passed": false, - "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" + "passed": true, + "notes": "imports @supabase/server / withSupabase" } ], "skills": { @@ -11481,31 +11552,59 @@ "calls": [ { "source": "search_docs", - "query": "query {\n searchDocs(query: \"edge function get user auth supabase service role apikey header SUPABASE_SERVICE_ROLE_KEY SUPABASE_ANON_KEY\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", + "query": "query { searchDocs(query: \"Supabase Edge Functions service role key Authorization apikey header access token auth uid req.json Deno.env.get\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on CLICommandReference { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" }, { "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" + } + ], + "resultChars": 48212 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions config.toml verify_jwt false functions config.toml local development\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on CLICommandReference { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/deploy", + "title": "Deploy to Production" + }, + { + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" + }, + { + "url": "https://supabase.com/docs/guides/functions/development-tips", + "title": "Development tips" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", + "title": "Building an MCP Server with mcp-lite" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" } ], - "resultChars": 56285 + "resultChars": 38079 } ] }, @@ -11552,7 +11651,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019fdc8c-c70c-75ca-a770-9cd6c0d41bf9/receipt-alpha.pdf, 019fdc8c-c70c-75ca-a770-9cd6c0d41bf9/receipt-beta.pdf" + "notes": "saw: 01a000ac-1dcd-7798-af43-63dc95149c6d/receipt-alpha.pdf, 01a000ac-1dcd-7798-af43-63dc95149c6d/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -11573,7 +11672,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Meets rubric: private user-files bucket, RLS remains enabled, authenticated SELECT and INSERT policies scoped to bucket and auth.uid() path prefix, and supabase-js uses createSignedUrl with expiry." + "judgeNotes": "Meets all requirements: private bucket, owner-scoped authenticated SELECT/INSERT policies with RLS kept enabled, and createSignedUrl with expiry for temporary sharing." } ], "skills": { @@ -11584,37 +11683,87 @@ "calls": [ { "source": "search_docs", - "query": "query {\n searchDocs(query: \"storage.foldername policy storage.objects auth.uid bucket private files\", limit: 5) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n ... on CLICommandReference { title href content }\n ... on ManagementApiReference { title href content }\n }\n }\n}", + "query": "query {\n searchDocs(query: \"storage.objects policy auth.uid path prefix RLS upload download signed url\", limit: 5) {\n nodes {\n __typename\n title\n href\n content\n }\n }\n}", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-cant-i-uploadlistetc-my-public-bucket-Z6CmGt", - "title": "Why can't I upload/list/etc my public bucket?" - }, - { - "url": "https://supabase.com/docs/guides/storage/quickstart", - "title": "Storage Quickstart" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", "title": "Storage Buckets" }, + { + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" + }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", "title": "Configure S3 Storage" + }, + { + "url": "https://supabase.com/docs/guides/storage/schema/design", + "title": "The Storage Schema" + } + ], + "resultChars": 21438 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"createSignedUrl supabase-js storage.from signed url example\", limit: 5) {\n nodes {\n __typename\n title\n href\n content\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" + }, + { + "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations", + "title": "Manage Storage Image Transformations usage" + }, + { + "url": "https://supabase.com/docs/guides/storage/uploads/resumable-uploads", + "title": "Resumable Uploads" + }, + { + "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", + "title": "Storage Image Transformations" + }, + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" + } + ], + "resultChars": 47418 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"storage.foldername helper function path array storage.objects\", limit: 5) {\n nodes {\n __typename\n title\n href\n content\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", + "title": "Storage Helper Functions" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-storage-ls", + "title": "List objects by path prefix" + }, + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-listv2" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-storage-rm", + "title": "Remove objects by file path" } ], - "resultChars": 25446 + "resultChars": 5999 } ] }, "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini-no-skills/build-storage-001-private-bucket-access.json" }, { @@ -11642,7 +11791,7 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" + "notes": "1 file(s): supabase/tests/tenant_isolation.sql" }, { "name": "pgTAP isolation tests ran and pass", @@ -11652,7 +11801,7 @@ { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as the table with the tenant isolation flaw, not `notes`, and treats pgTAP verification as the signal after adding isolation tests. It describes fixing `posts` RLS so authenticated users cannot read other organizations' posts." + "judgeNotes": "The agent correctly identifies `posts` as the tenant isolation flaw: its policy allows authenticated members to read posts from organizations they are not members of. It contrasts this with `notes`, grounds the conclusion in observed RLS/test behavior, and fixes/tests the `posts` policy rather than blaming tests or migration comments." } ], "skills": { @@ -11664,7 +11813,7 @@ }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini-no-skills/build-tests-001-rls-tenant-isolation.json" }, { @@ -11727,22 +11876,11 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "web_search", - "query": "site:supabase.com gte-small 384 Supabase AI Session dimension", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs pgvector Supabase vector search match function", - "pages": [] - } - ] + "calls": [] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini-no-skills/build-vectors-001-rag-with-permissions.json" }, { @@ -11771,13 +11909,13 @@ }, { "name": "configured the Supabase Metrics API scrape correctly", - "passed": false, - "judgeNotes": "Fails because Prometheus uses basic_auth.password with an environment variable instead of basic_auth.password_file, and docker-compose.yml does not mount the password file via a volume or Compose secret. App scrape and endpoint are otherwise preserved." + "passed": true, + "judgeNotes": "Meets requirements: HTTPS Supabase Metrics API scrape for project target, correct metrics path, Basic Auth with password_file, app scrape preserved, and docker-compose mounts the secrets directory containing the password_file." }, { "name": "documented live deployment and verification steps", "passed": false, - "judgeNotes": "README includes env setup, restart/recreate Compose, and Prometheus target verification, but it does not provide steps to create the Secret API key in Supabase or place a matching secret file. The setup uses .env rather than the required secret file, so the secret setup requirement is not met." + "judgeNotes": "README includes creating a Supabase Secret API key, placing it in observability/secrets/supabase_metrics_api_key, and restarting/reloading Prometheus/stack. However, it lacks concrete verification steps such as checking Prometheus Targets, running a PromQL query, or confirming data in Grafana." } ], "skills": { @@ -11787,36 +11925,28 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"metrics prometheus project metrics scrape endpoint\", limit: 10) {\n nodes {\n ... on Guide {\n title\n href\n content\n }\n ... on TroubleshootingGuide {\n title\n href\n content\n }\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n language\n methodName\n }\n }\n totalCount\n }\n}", - "hasContent": true, + "source": "web_search", + "query": "site:supabase.com docs metrics prometheus supabase project metrics endpoint", + "pages": [] + }, + { + "source": "web_search", + "query": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", "pages": [ { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/how-to-view-database-metrics-uqf2z_", - "title": "How to View Database Metrics" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted" } - ], - "resultChars": 29978 + ] + }, + { + "source": "web_search", + "query": "site:supabase.com/docs create project api key management api supabase", + "pages": [] + }, + { + "source": "web_search", + "query": "site:supabase.com/docs/reference/api get project api keys management api", + "pages": [] } ] }, @@ -11858,7 +11988,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get('WEATHER_API_KEY')." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -11873,7 +12003,7 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query:\"Edge Function environment variable deploy secrets WEATHER_API_KEY supabase functions secrets set\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", + "query": "query {\n searchDocs(query: \"Supabase Edge Function secrets deploy env file WEATHER_API_KEY functions deploy secrets set\", limit: 5) {\n edges {\n node {\n ... on Guide { title href content }\n ... on CLICommandReference { title href content }\n ... on ManagementApiReference { title href content }\n ... on TroubleshootingGuide { title href content }\n }\n }\n }\n}", "hasContent": true, "pages": [ { @@ -11881,476 +12011,549 @@ "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-1-create-the-new-api-keys", - "title": "Step 1: Create the new API keys" + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#before-you-start", - "title": "Before you start" + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", + "title": "Streaming Speech with ElevenLabs" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#next-steps", - "title": "Next steps" + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", + "title": "Transcription Telegram Bot" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#known-limitations", - "title": "Known limitations" + "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", + "title": "Supabase for Platforms" + } + ], + "resultChars": 59184 + }, + { + "source": "web_search", + "query": "Supabase Edge Function invocation URL project-ref functions.supabase.co functions/v1", + "pages": [] + }, + { + "source": "web_search", + "query": "site:supabase.com/docs functions/v1 invoke project ref supabase.co project not specified", + "pages": [] + }, + { + "source": "web_search", + "query": "site:supabase.com/docs Edge Function invoke apikey header Authorization Bearer publishable key", + "pages": [] + } + ] + }, + "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", + "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-functions-001-edge-function-secrets.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "deploy-self-hosting-001-docker-compose", + "stage": "deploy", + "product": [ + "database", + "auth", + "storage" + ], + "topic": [ + "self-hosting" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", + "passed": true + }, + { + "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", + "passed": true + }, + { + "name": "secrets rotated off the shipped defaults", + "passed": true + }, + { + "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"self-hosting docker compose Supabase environment variables config.toml gateway auth rest storage\", limit: 5) { nodes { __typename ... on Guide { title href content subsections { nodes { title href content } } } ... on TroubleshootingGuide { title href content } ... on CLICommandReference { title href content } ... on ManagementApiReference { title href content } } totalCount } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-6-deactivate-the-legacy-keys", - "title": "Step 6: Deactivate the legacy keys" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#before-you-begin", + "title": "Before you begin" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-5-verify-nothing-uses-the-legacy-keys", - "title": "Step 5: Verify nothing uses the legacy keys" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#step-1-get-your-platform-connection-string", + "title": "Step 1: Get your platform connection string" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-2-adopt-the-supabaseserver-sdk", - "title": "Option 2: Adopt the @supabase/server SDK" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#step-2-back-up-your-platform-database", + "title": "Step 2: Back up your platform database" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-1-read-the-new-keys-from-the-environment", - "title": "Option 1: Read the new keys from the environment" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#step-3-prepare-your-self-hosted-instance", + "title": "Step 3: Prepare your self-hosted instance" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-4-update-edge-functions", - "title": "Step 4: Update Edge Functions" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#step-4-restore-to-your-self-hosted-database", + "title": "Step 4: Restore to your self-hosted database" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-3-swap-the-secret-key-in-backend-code", - "title": "Step 3: Swap the secret key in backend code" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#step-5-verify-the-restore", + "title": "Step 5: Verify the restore" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#database-webhooks-and-pg_net", - "title": "Database Webhooks and pg_net" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#whats-included-in-the-restore-and-whats-not", + "title": "What's included in the restore and what's not" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-2-swap-the-publishable-key-in-client-code", - "title": "Step 2: Swap the publishable key in client code" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#auth-considerations", + "title": "Auth considerations" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", - "title": "Transcription Telegram Bot" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#postgres-version-compatibility", + "title": "Postgres version compatibility" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#deploy-to-supabase", - "title": "Deploy to Supabase" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#troubleshooting", + "title": "Troubleshooting" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#introduction", - "title": "Introduction" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#version-mismatches-between-platform-and-self-hosted", + "title": "Version mismatches between platform and self-hosted" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#requirements", - "title": "Requirements" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#extension-not-available", + "title": "Extension not available" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#setup", - "title": "Setup" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#connection-refused", + "title": "Connection refused" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#register-a-telegram-bot", - "title": "Register a Telegram bot" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#legacy-studio-configuration", + "title": "Legacy Studio configuration" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#create-a-supabase-project-locally", - "title": "Create a Supabase project locally" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#custom-roles-missing-passwords", + "title": "Custom roles missing passwords" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#create-a-database-table-to-log-the-transcription-results", - "title": "Create a database table to log the transcription results" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#additional-resources", + "title": "Additional resources" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#create-a-supabase-edge-function-to-handle-telegram-webhook-requests", - "title": "Create a Supabase Edge Function to handle Telegram webhook requests" + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#set-up-the-environment-variables", - "title": "Set up the environment variables" + "url": "https://supabase.com/docs/guides/self-hosting/docker#architecture", + "title": "Architecture" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#dependencies", - "title": "Dependencies" + "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-edge-functions", + "title": "Accessing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#code-the-telegram-bot", - "title": "Code the Telegram bot" + "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-apis", + "title": "Accessing APIs" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#apply-the-database-migrations", - "title": "Apply the database migrations" + "url": "https://supabase.com/docs/guides/self-hosting/docker#enabling-analytics", + "title": "Enabling analytics" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#set-up-the-webhook", - "title": "Set up the webhook" + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-https", + "title": "Configuring HTTPS" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#set-the-function-secrets", - "title": "Set the function secrets" + "url": "https://supabase.com/docs/guides/self-hosting/docker#managing-the-stack", + "title": "Managing the stack" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#test-the-bot", - "title": "Test the bot" + "url": "https://supabase.com/docs/guides/self-hosting/docker#updating", + "title": "Updating" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" + "url": "https://supabase.com/docs/guides/self-hosting/docker#uninstalling", + "title": "Uninstalling" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#deploy-to-supabase", - "title": "Deploy to Supabase" + "url": "https://supabase.com/docs/guides/self-hosting/docker#advanced-topics", + "title": "Advanced topics" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#try-it-out", - "title": "Try it out" + "url": "https://supabase.com/docs/guides/self-hosting/docker#setting-database-password", + "title": "Setting database password" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#run-locally", - "title": "Run locally" + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-and-securing-supabase", + "title": "Configuring and securing Supabase" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#code-the-supabase-edge-function", - "title": "Code the Supabase Edge Function" + "url": "https://supabase.com/docs/guides/self-hosting/docker#generate-keys-and-secrets", + "title": "Generate keys and secrets" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#dependencies", - "title": "Dependencies" + "url": "https://supabase.com/docs/guides/self-hosting/docker#configure-supabase-urls", + "title": "Configure Supabase URLs" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#set-up-the-environment-variables", - "title": "Set up the environment variables" + "url": "https://supabase.com/docs/guides/self-hosting/docker#where-to-find-your-credentials", + "title": "Where to find your credentials" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#create-a-supabase-edge-function-for-speech-generation", - "title": "Create a Supabase Edge Function for speech generation" + "url": "https://supabase.com/docs/guides/self-hosting/docker#studio-authentication", + "title": "Studio authentication" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#configure-background-tasks-for-supabase-edge-functions", - "title": "Configure background tasks for Supabase Edge Functions" + "url": "https://supabase.com/docs/guides/self-hosting/docker#starting-and-stopping", + "title": "Starting and stopping" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#configure-the-storage-bucket", - "title": "Configure the storage bucket" + "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-supabase-studio-dashboard", + "title": "Accessing Supabase Studio (Dashboard)" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#create-a-supabase-project-locally", - "title": "Create a Supabase project locally" + "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-postgres", + "title": "Accessing Postgres" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#setup", - "title": "Setup" + "url": "https://supabase.com/docs/guides/self-hosting/docker#demo", + "title": "Demo" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#introduction", - "title": "Introduction" + "url": "https://supabase.com/docs/guides/self-hosting/docker#managing-your-secrets", + "title": "Managing your secrets" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#requirements", - "title": "Requirements" + "url": "https://supabase.com/docs/guides/self-hosting/docker#setting-log_min_messages-in-postgres", + "title": "Setting log_min_messages in Postgres" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#test-the-function", - "title": "Test the function" + "url": "https://supabase.com/docs/guides/self-hosting/docker#exposing-your-postgres-database", + "title": "Exposing your Postgres database" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#set-the-function-secrets", - "title": "Set the function secrets" + "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-postgres-through-supavisor", + "title": "Accessing Postgres through Supavisor" }, { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-supabase-ai-assistant", + "title": "Configuring Supabase AI Assistant" }, { - "url": "https://supabase.com/docs/guides/functions/secrets#accessing-environment-variables", - "title": "Accessing environment variables" + "url": "https://supabase.com/docs/guides/self-hosting/docker#using-file-backend-in-storage-on-macos", + "title": "Using file backend in Storage on macOS" }, { - "url": "https://supabase.com/docs/guides/functions/secrets#local-secrets", - "title": "Local secrets" + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-s3-storage", + "title": "Configuring S3 Storage" }, { - "url": "https://supabase.com/docs/guides/functions/secrets#production-secrets", - "title": "Production secrets" + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-an-email-server", + "title": "Configuring an email server" }, { - "url": "https://supabase.com/docs/guides/functions/secrets#default-secrets", - "title": "Default secrets" + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-phone-login-sms-and-mfa", + "title": "Configuring phone login, SMS, and MFA" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-social-login-oauth-providers", + "title": "Configuring social login (OAuth) providers" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#changing-compute-sizes", - "title": "Changing compute sizes" + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-supabase-services", + "title": "Configuring Supabase services" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#recommended-api-keys", - "title": "Recommended API keys" + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-secrets", + "title": "Configuring secrets" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#nano-compute-instance", - "title": "Nano compute instance" + "url": "https://supabase.com/docs/guides/self-hosting/docker#changing-database-password", + "title": "Changing database password" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#launching-projects", - "title": "Launching projects" + "url": "https://supabase.com/docs/guides/self-hosting/docker#contents", + "title": "Contents" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#overview", - "title": "Overview" + "url": "https://supabase.com/docs/guides/self-hosting/docker#before-you-begin", + "title": "Before you begin" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#platform-kit", - "title": "Platform kit" + "url": "https://supabase.com/docs/guides/self-hosting/docker#system-requirements", + "title": "System requirements" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#claim-flow", - "title": "Claim flow" + "url": "https://supabase.com/docs/guides/self-hosting/docker#installing-supabase", + "title": "Installing Supabase" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#disaster-recovery-for-production", - "title": "Disaster recovery for production" + "url": "https://supabase.com/docs/guides/self-hosting/docker#quick-start-linux", + "title": "Quick start (Linux)" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#security-checks-for-production", - "title": "Security checks for production" + "url": "https://supabase.com/docs/guides/self-hosting/docker#manual-installation", + "title": "Manual installation" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#merge-all-changes", - "title": "Merge all changes" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#deploying-edge-functions", - "title": "Deploying Edge Functions" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#what-client-sdk-sends", + "title": "What client SDK sends" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#add-seed-data", - "title": "Add seed data" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#how-it-works", + "title": "How it works" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#reverting-changes", - "title": "Reverting changes" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#regenerating-asymmetric-key-pair", + "title": "Regenerating asymmetric key pair" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#create-a-restore-point", - "title": "Create a restore point" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#rotating-the-new-api-keys", + "title": "Rotating the new API keys" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#make-database-changes", - "title": "Make database changes" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#backward-compatibility", + "title": "Backward compatibility" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#creating-a-dev-branch", - "title": "Creating a DEV branch" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#differences-from-the-supabase-platform", + "title": "Differences from the Supabase platform" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#development-workflow", - "title": "Development workflow" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#environment-variables-configuration", + "title": "Environment variables configuration" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#configuration-changes", - "title": "Configuration changes" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#verifying-the-setup", + "title": "Verifying the setup" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#debugging-projects", - "title": "Debugging projects" - } - ], - "resultChars": 164975 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query:\"Edge Functions invoke URL functions/v1 project ref invoke runtime endpoint\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#new-api-keys-format", + "title": "New API keys format" + }, { - "url": "https://supabase.com/docs/guides/functions/recursive-functions", - "title": "Recursive / Nested Function Calls" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#adding-the-new-keys", + "title": "Adding the new keys" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#additional-resources", + "title": "Additional resources" }, { - "url": "https://supabase.com/docs/guides/functions/storage-caching", - "title": "Integrating with Supabase Storage" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#before-you-begin", + "title": "Before you begin" }, { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#authenticated-requests-user-session-jwt", + "title": "Authenticated requests (user session JWT)" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#unauthenticated-requests-api-key-only-no-user-session-jwt", + "title": "Unauthenticated requests (API key only, no user session JWT)" }, { - "url": "https://supabase.com/docs/guides/functions/ai-models", - "title": "Running AI Models" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#request-flows", + "title": "Request flows" }, { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#kong-api-gateway-routing", + "title": "Kong API gateway routing" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" }, { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#session-token", + "title": "Session token" }, { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" - } - ], - "resultChars": 93047 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query:\"Management API edge functions get function by slug project ref functions/{function_slug}\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#enable-the-s3-protocol-endpoint", + "title": "Enable the S3 protocol endpoint" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#test-with-the-aws-cli", + "title": "Test with the AWS CLI" + }, { - "url": "https://supabase.com/docs/reference/api/v1-get-a-function", - "title": "Retrieve a function" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#test-with-rclone", + "title": "Test with rclone" }, { - "url": "https://supabase.com/docs/reference/api/v1-delete-a-function", - "title": "Delete a function" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#how-to-configure-an-s3-backend", + "title": "How to configure an S3 backend" }, { - "url": "https://supabase.com/docs/reference/api/v1-update-a-function", - "title": "Update a function" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#using-rustfs", + "title": "Using RustFS" }, { - "url": "https://supabase.com/docs/reference/api/v1-list-all-functions", - "title": "List all functions" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#using-minio", + "title": "Using MinIO" }, { - "url": "https://supabase.com/docs/reference/api/v1-get-a-function-body", - "title": "Retrieve a function body" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#using-aws-s3", + "title": "Using AWS S3" }, { - "url": "https://supabase.com/docs/reference/api/v1-create-a-function", - "title": "Create a function" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#s3-compatible-providers", + "title": "S3-compatible providers" }, { - "url": "https://supabase.com/docs/reference/api/v1-deploy-a-function", - "title": "Deploy a function" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#verify", + "title": "Verify" }, { - "url": "https://supabase.com/docs/reference/api/v1-bulk-update-functions", - "title": "Bulk update functions" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#troubleshooting", + "title": "Troubleshooting" }, { - "url": "https://supabase.com/docs/reference/api/v1-get-project-function-combined-stats", - "title": "Gets a project's function combined statistics" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#signature-mismatch-errors", + "title": "Signature mismatch errors" }, { - "url": "https://supabase.com/docs/guides/ai-tools/mcp", - "title": "Supabase MCP Server" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#tus-upload-errors-on-cloudflare-r2", + "title": "TUS upload errors on Cloudflare R2" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#permission-denied-on-uploads", + "title": "Permission denied on uploads" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#upload-urls-point-to-localhost", + "title": "Upload URLs point to localhost" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#additional-resources", + "title": "Additional resources" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting", + "title": "Self-Hosting" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting#community-driven-projects", + "title": "Community-driven projects" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting#about-self-hosting", + "title": "About self-hosting" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting#how-self-hosted-supabase-differs", + "title": "How self-hosted Supabase differs" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting#your-responsibilities-when-self-hosting", + "title": "Your responsibilities when self-hosting" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting#telemetry", + "title": "Telemetry" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting#get-started", + "title": "Get started" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting#support-and-community", + "title": "Support and community" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting#enterprise-self-hosting", + "title": "Enterprise self-hosting" } ], - "resultChars": 18063 + "resultChars": 207476 }, { "source": "search_docs", - "query": "query { searchDocs(query:\"get_project_url project url supabase management api\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"docker-compose.yml .env.example self-hosted Supabase services kong auth postgrest realtime storage meta gotrue imgproxy studio vector supavisor\", limit: 10) { nodes { __typename ... on Guide { title href content } ... on CLICommandReference { title href content } ... on TroubleshootingGuide { title href content } } totalCount } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/analytics/connecting-to-analytics-bucket", - "title": "Iceberg Catalog" + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/ai-tools/mcp", - "title": "Supabase MCP Server" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" + "url": "https://supabase.com/docs/guides/getting-started/architecture", + "title": "Architecture" }, { - "url": "https://supabase.com/docs/guides/api/creating-routes", - "title": "Creating API Routes" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", + "title": "Configure Social Login (OAuth) Providers" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting", + "title": "Self-Hosting" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17", + "title": "Upgrade to Postgres 17" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/remove-superuser-access", + "title": "Remove Superuser Access from Studio" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https", + "title": "Configure Reverse Proxy and HTTPS" } ], - "resultChars": 69182 - } - ] - }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-functions-001-edge-function-secrets.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", - "product": [ - "database", - "auth", - "storage" - ], - "topic": [ - "self-hosting" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true - }, - { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true - }, - { - "name": "secrets rotated off the shipped defaults", - "passed": true - }, - { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "web_search", - "query": "Supabase self-hosting docker compose official secrets env file", - "pages": [] - }, - { - "source": "web_search", - "query": "site:github.com/supabase/supabase docker generate-keys.sh .env.example", - "pages": [] + "resultChars": 146772 } ] }, "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-self-hosting-001-docker-compose.json" }, { @@ -12385,16 +12588,18 @@ }, { "name": "delete-account revokes the user's sessions", - "passed": true, - "notes": "sessions left: 0" + "passed": false, + "notes": "sessions left: 1" }, { "name": "deleted user's refresh token is rejected", - "passed": true + "passed": false, + "notes": "refresh token still produces a session" }, { "name": "deleted user cannot sign back in", - "passed": true + "passed": false, + "notes": "deleted account can still sign in" }, { "name": "other users keep their sessions and access", @@ -12403,7 +12608,7 @@ { "name": "diagnosed and explained session revocation", "passed": false, - "judgeNotes": "The answer correctly diagnoses the soft-delete bug, deletes the auth user so sessions/refresh tokens cascade, adds RLS that blocks stale JWTs on the data path, and correctly explains publishable vs secret keys. However, it does not clearly state the required caveat that access tokens are stateless JWTs not recalled by deletion/revocation and that purely local validation such as getClaims/custom JWT middleware will continue accepting them until expiry." + "judgeNotes": "The assistant correctly identified the soft-delete/RLS issue and clarified publishable vs secret keys, and its access-window explanation is mostly consistent with the RLS-only fix. However, it did not fix the delete-account flow with real auth revocation: it left delete_account as a soft delete and did not delete the auth user, identities, sessions, or refresh tokens. The rubric requires real revocation, so this fails." } ], "skills": { @@ -12414,26 +12619,8 @@ "calls": [ { "source": "web_search", - "query": "site:supabase.com/docs publishable secret keys supabase frontend secret keys rls", + "query": "site:supabase.com/docs publishable key secret key Supabase frontend RLS", "pages": [] - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/getting-started/api-keys", - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys" - } - ] - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/auth/jwts", - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/jwts" - } - ] } ] }, @@ -12488,7 +12675,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "The assistant correctly identified that orders was missing from the supabase_realtime publication despite the channel subscribing, added only public.orders to the existing publication, and preserved courier_locations, RLS, and policies." + "judgeNotes": "The assistant correctly identified that orders was missing from the supabase_realtime publication despite the channel subscribing, added only public.orders via ALTER PUBLICATION, and verified courier_locations remained included. It did not disable RLS or weaken policies." } ], "skills": { @@ -12500,7 +12687,7 @@ }, "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-realtime-001-subscribed-no-events.json" }, { @@ -12527,17 +12714,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform as the affected function and described the recurring HTTP 503 pattern across the morning of 2026-04-28, covering all 8 gateway failures from 07:00Z to 12:00Z." + "judgeNotes": "The assistant correctly identified `image-transform` as the affected function and explicitly described the recurring 503 pattern across the morning of 2026-04-28, listing all 8 gateway failure timestamps from 07:00Z through 12:00Z." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": false, - "judgeNotes": "Although the assistant correctly attributes the 503s to the gateway/Edge invocation path and grounds this in valid observations, it recommends redeploying the functions as a remediation, which the rubric explicitly lists as a fail condition." + "judgeNotes": "Although it correctly attributes the recurring 503s to the gateway/platform layer and cites valid evidence, it also recommends redeploying the functions as a remediation, which the rubric marks as a fail condition." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended concrete next steps including redeploying Edge Functions, checking Supabase incidents/status, adding retry/backoff, and opening a support ticket with exact timestamps and deployment IDs." + "judgeNotes": "Recommended concrete next steps including opening a Supabase support/incident case with exact timestamps and request IDs, redeploying functions, adding retries, and monitoring specific function logs." } ], "skills": { @@ -12603,7 +12790,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies causing deny-all/empty Data API results, kept RLS enabled, and created authenticated SELECT and INSERT owner-scoped policies using auth.uid() = user_id, including WITH CHECK for INSERT. Extra update/delete owner policies do not violate the rubric." + "judgeNotes": "Diagnosed RLS enabled with no policies, kept RLS enabled, and added authenticated-only SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." } ], "skills": { @@ -12615,7 +12802,7 @@ }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-dataapi-001-empty-results.json" }, { @@ -12637,15 +12824,17 @@ ], "suite": "benchmark", "interface": "cli", - "passed": true, + "passed": false, "checks": [ { "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true + "passed": false, + "notes": "avatar_url not found on public.profiles" }, { "name": "migration 20240220000000 is recorded in the remote history", - "passed": true + "passed": false, + "notes": "remote history versions: [\"20240101000000\",\"20240115000000\"]" }, { "name": "remote migration history matches local migration files", @@ -12653,7 +12842,8 @@ }, { "name": "local migrations are a valid reconciled sequence", - "passed": true + "passed": false, + "notes": "expected 20240101000000_create_profiles → → 20240220000000_add_avatar_url, strictly ascending; got [20240101000000_create_profiles.sql, 20240115000000_add_profile_bio.sql]" }, { "name": "production profile data is intact (not reset)", @@ -12661,8 +12851,8 @@ }, { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": true, - "judgeNotes": "Avatar migration was applied through Supabase CLI with `supabase db push --db-url 'postgresql://postgres:postgres@172.17.0.1:41925/postgres?sslmode=disable'` (#34), which output `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration history was reconciled by adding the local migration file `supabase/migrations/20240115000000_add_profile_bio.sql` (#31), after which `supabase migration list` showed local and remote matched for `20240115000000` (#32), allowing the push to proceed. Only read-only `psql` inspections were used; no prohibited direct SQL mutation or prepared-statement workaround was seen." + "passed": false, + "judgeNotes": "No successful real `supabase db push` occurred; attempts #12, #16, #21, #29, and #31 all failed, so the avatar_url migration was not applied through the Supabase CLI. The orphan bio migration was only added locally in #46, but there was no successful CLI reconciliation to the remote via `supabase migration repair`, `supabase db pull`, or successful `supabase db push`. Management API database/query calls were used only for read-only inspection." } ], "skills": { @@ -12730,7 +12920,7 @@ }, "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-performance-001-slow-query-cpu-spike.json" }, { @@ -12802,7 +12992,7 @@ }, "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-security-002-rls-cross-tenant-leak.json" }, { @@ -12837,7 +13027,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 54178c78-bfc4-4a47-8331-65384278fdef, signUp returned {\"userId\":\"54178c78-bfc4-4a47-8331-65384278fdef\"}" + "notes": "db user 2ab9b66b-45c1-4658-9aa6-06b11227c4c8, signUp returned {\"userId\":\"2ab9b66b-45c1-4658-9aa6-06b11227c4c8\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -12852,7 +13042,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"54178c78-bfc4-4a47-8331-65384278fdef\"}" + "notes": "{\"userId\":\"2ab9b66b-45c1-4658-9aa6-06b11227c4c8\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -12882,111 +13072,61 @@ }, "docs": { "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js signUp email password options data user metadata signInWithPassword getUser select single profile\", limit: 5) { nodes { ... on ClientLibraryFunctionReference { title href content language methodName } ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - } - ], - "resultChars": 80822 - }, { "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- changelog auth mentions ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|auth|supabase-js' | sed -n '1,220p'; printf '%s\\\\n' '--- auth.mjs ---'; cat app/src/auth.mjs; printf '%s\\\\n' '--- app directory ---'; find app -maxdepth 3 -type f -o -type l | sort | xargs -r ls -l; printf '%s\\\\n' '--- service status ---'; supabase status -o env\"", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- AGENTS ---'; find .. -name AGENTS.md -print; printf '%s\\\\n' '--- FILES ---'; rg --files app supabase | sort; printf '%s\\\\n' '--- AUTH ---'; sed -n '1,240p' app/src/auth.mjs; printf '%s\\\\n' '--- PACKAGE ---'; cat app/package.json; printf '%s\\\\n' '--- CONFIG ---'; sed -n '1,240p' supabase/config.toml; printf '%s\\\\n' '--- SQL ---'; for f in supabase/migrations/*.sql; do echo \\\"### \"'$f\"; sed -n '\"'1,260p' \\\"\"'$f\"; done; printf '\"'%s\\\\n' '--- CHANGELOG AUTH HITS ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'auth|sign.?up|metadata|breaking-change' | head -n 120\"", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 1410 + "resultChars": 13992 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript signUp email password options data display_name\", limit: 10) { nodes { ... on ClientLibraryFunctionReference { title href content language methodName } ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"supabase-js createClient signUp email password options data user metadata signInWithPassword select single current user profile\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" + "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", + "title": "Customizing Emails by Language" }, { - "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", - "title": "Passwordless email logins" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" }, { - "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" }, { - "url": "https://supabase.com/docs/reference/csharp/auth-signinwithpassword", - "title": "SignIn(email, password)" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-verifyotp" + "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", + "title": "Passwordless email logins" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-passkey-list" + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-signinanonymously" + "url": "https://supabase.com/docs/guides/auth/passkeys", + "title": "Passkey authentication" }, { - "url": "https://supabase.com/docs/reference/dart/auth-signup" + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" } ], - "resultChars": 19311 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript auth getUser current user select maybeSingle profiles table\", limit: 8) { nodes { ... on ClientLibraryFunctionReference { title href content language methodName } ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-getuser" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-getuserbyid" - }, - { - "url": "https://supabase.com/docs/reference/javascript/select" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-listusers" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-createuser" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-updateuser" - }, - { - "url": "https://supabase.com/docs/reference/javascript/using-modifiers-maybesingle" - } - ], - "resultChars": 7456 + "resultChars": 135018 } ] }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.6/build-auth-001-email-password-flow.json" }, { @@ -13058,59 +13198,59 @@ "calls": [ { "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- CLAUDE SUPABASE ---'; sed -n '1,240p' .claude/skills/supabase/SKILL.md; printf '%s\\\\n' '--- CLAUDE POSTGRES ---'; sed -n '1,220p' .claude/skills/supabase-postgres-best-practices/SKILL.md; printf '%s\\\\n' '--- SECURITY REFS ---'; find .agents/skills/supabase-postgres-best-practices/references -maxdepth 1 -type f -printf '%f\\\\n' | sort | rg 'security|schema'; printf '%s\\\\n' '--- RLS REFERENCES ---'; cat .agents/skills/supabase-postgres-best-practices/references/security-rls-performance.md 2>/dev/null; cat .agents/skills/supabase-postgres-best-practices/references/security-rls.md 2>/dev/null; printf '%s\\\\n' '--- CLI VERSION ---'; supabase --version; printf '%s\\\\n' '--- GIT ---'; git status --short --branch 2>/dev/null || true; printf '%s\\\\n' '--- CHANGELOG MATCHES ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|row level|rls|local development|cli' | head -n 120\"", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- files ---'; find . -maxdepth 4 -type f -not -path './.git/*' | sort; printf '%s\\\\n' '--- postgres references ---'; find .agents/skills/supabase-postgres-best-practices/references -maxdepth 1 -type f | sort | sed -n '1,160p'; printf '%s\\\\n' '--- changelog breaking changes ---'; curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|breaking change' | head -40\"", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 8747 + "resultChars": 23196 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Data API expose table grant anon authenticated RLS select policy authenticated local development CLI migrations db reset\", limit: 8) { nodes { __typename title href content } } }", + "query": "query { searchDocs(query: \"Row Level Security policy authenticated role SELECT grant Data API tables not exposed automatically local development seed.sql auth signup\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" + "url": "https://supabase.com/docs/guides/auth/auth-mfa", + "title": "Multi-Factor Authentication" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" + "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration", + "title": "Build a Supabase Integration" }, { - "url": "https://supabase.com/docs/reference/cli/supabase-db-reset", - "title": "Resets the local database to current migrations" + "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", + "title": "Column Level Security" } ], - "resultChars": 124182 + "resultChars": 113086 } ] }, "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.6/build-cli-001-bootstrap-app.json" }, { @@ -13166,47 +13306,37 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase declarative database schemas generate migration db diff local\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"declarative database schemas schema_paths generate migration db diff local\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", "title": "Declarative database schemas" }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, { "url": "https://supabase.com/docs/guides/deployment/database-migrations", "title": "Database Migrations" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" + "url": "https://supabase.com/docs/reference/cli/supabase-db-diff", + "title": "Diffs the local database for schema changes" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" }, { "url": "https://supabase.com/docs/guides/deployment/managing-environments", "title": "Managing Environments" } ], - "resultChars": 82866 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|declarative|migration|db diff' | head -80; printf '\\\\n--- CONFIG ---\\\\n'; sed -n '1,220p' supabase/config.toml; printf '\\\\n--- SCHEMA ---\\\\n'; cat supabase/schemas/products.sql; printf '\\\\n--- MIGRATION ---\\\\n'; cat supabase/migrations/20260101000000_create_products.sql; printf '\\\\n--- STATUS ---\\\\n'; supabase status || true\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 8129 + "resultChars": 72112 } ] }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.6/build-cli-002-declarative-schema.json" }, { @@ -13262,2009 +13392,1130 @@ }, "docs": { "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- files ---'; rg --files -g 'AGENTS.md' -g 'supabase/**' -g '\"'!supabase/.temp/**'\"' | sort; printf '%s\\\\n' '--- root ---'; ls -la; printf '%s\\\\n' '--- supabase config ---'; test -f supabase/config.toml && sed -n '1,240p' supabase/config.toml || true; printf '%s\\\\n' '--- changelog relevant tags ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|queue|pgmq|cron' | head -n 160\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 10637 + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Queues pgmq create queue send read delete Edge Function cron schedule every minute\", limit: 8) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase Queues pgmq create queue send read pop delete messages SQL functions cron schedule every minute Edge Function local\", limit: 8) { nodes { __typename title href content ... on Guide { subsections { nodes { title href content } } } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" }, { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" + "url": "https://supabase.com/docs/guides/queues#resources", + "title": "Resources" + }, + { + "url": "https://supabase.com/docs/guides/queues#features", + "title": "Features" }, { "url": "https://supabase.com/docs/guides/cron", "title": "Cron" }, { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" + "url": "https://supabase.com/docs/guides/cron#how-does-cron-work", + "title": "How does Cron work?" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" + "url": "https://supabase.com/docs/guides/cron#resources", + "title": "Resources" }, { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/schedule-functions#examples", + "title": "Examples" + }, + { + "url": "https://supabase.com/docs/guides/functions/schedule-functions#invoke-an-edge-function-every-minute", + "title": "Invoke an Edge Function every minute" + }, + { + "url": "https://supabase.com/docs/guides/functions/schedule-functions#resources", + "title": "Resources" }, { "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", "title": "Consuming Supabase Queue Messages with Edge Functions" }, { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - } - ], - "resultChars": 44996 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Cron schedule database function cron.schedule pgmq send queue\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#consuming-messages-in-an-edge-function", + "title": "Consuming messages in an Edge Function" + }, + { + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#concepts", + "title": "Concepts" + }, { "url": "https://supabase.com/docs/guides/database/extensions/pg_net", "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#analyzing-responses", + "title": "Analyzing responses" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#debugging-requests", + "title": "Debugging requests" }, { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_get", + "title": "http_get" }, { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-signature", + "title": "Signature " }, { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" - } - ], - "resultChars": 35015 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- CHANGELOG MATCHES ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|queue|cron|edge function' | head -n 160 || true; printf '%s\\\\n' '--- CONFIG RELEVANT ---'; rg -n '\"'^'\"\\\\[|enabled|verify_jwt|additional_search_path|port' supabase/config.toml; printf '%s\\\\n' '--- MIGRATION HEAD ---'; sed -n '1,100p' supabase/migrations/20260101000000_enable_cron_and_queues.sql; printf '%s\\\\n' '--- MIGRATION TAIL ---'; tail -n 80 supabase/migrations/20260101000000_enable_cron_and_queues.sql; printf '%s\\\\n' '--- MIGRATION SYMBOLS ---'; rg -n 'create extension|pgmq.create|cron.schedule|pgmq_public|create schema' supabase/migrations/20260101000000_enable_cron_and_queues.sql\"", - "pages": [ + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-usage", + "title": "Usage " + }, { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 19533 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions SUPABASE_URL SUPABASE_SERVICE_ROLE_KEY environment variables local serve createClient\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_post", + "title": "http_post" + }, { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-signature", + "title": "Signature " }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-usage", + "title": "Usage " }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_delete", + "title": "http_delete" }, { - "url": "https://supabase.com/docs/guides/functions/unit-test", - "title": "Testing your Edge Functions" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-signature-1", + "title": "Signature " }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - } - ], - "resultChars": 40557 - } - ] - }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-cli-003-pg-cron-queue-workflow.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-dataapi-001-relational-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "report numbers match the database (per customer, sorted)", - "passed": true, - "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table customers" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/report.mjs" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase javascript select nested foreign tables service role secret key createClient Node\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#inspecting-request-data", + "title": "Inspecting request data" + }, { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#inspecting-failed-requests", + "title": "Inspecting failed requests" }, { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#configuration", + "title": "Configuration" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-current-settings", + "title": "Get current settings" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/performing-administration-tasks-on-the-server-side-with-the-servicerole-secret-BYM4Fa", - "title": "Performing administration tasks on the server side with a secret key" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#alter-settings", + "title": "Alter settings" }, { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-updateclient" - } - ], - "resultChars": 22127 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- changelog relevant scan ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -n -m 20 'breaking-change|supabase-js|secret key|postgrest|data api' || true; printf '%s\\\\n' '--- root listing ---'; ls -la; printf '%s\\\\n' '--- app listing ---'; find app -maxdepth 2 -type f -print -exec sed -n '1,80p' {} \\\\;; printf '%s\\\\n' '--- node/npm ---'; node --version; npm --version; npm root -g; printf '%s\\\\n' '--- installed candidates ---'; find . -maxdepth 3 -type d \\\\( -name '@supabase' -o -name 'node_modules' \\\\) -print; printf '%s\\\\n' '--- supabase status env ---'; supabase status -o env\"", - "pages": [ + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#examples", + "title": "Examples" + }, { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 1554 - } - ] - }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.6/build-dataapi-001-relational-report.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-dataapi-002-restock-alert-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "alerts match the database (below threshold, sorted)", - "passed": true, - "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table inventory" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js select nested foreign tables createClient secret key Node backend order results\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#invoke-a-supabase-edge-function", + "title": "Invoke a Supabase Edge Function" + }, { - "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", - "title": "Engineering for Scale" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#call-an-endpoint-every-minute-with-pg_cron", + "title": "Call an endpoint every minute with pg_cron" }, { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#execute-pg_net-in-a-trigger", + "title": "Execute pg_net in a trigger" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/performing-administration-tasks-on-the-server-side-with-the-servicerole-secret-BYM4Fa", - "title": "Performing administration tasks on the server side with a secret key" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#send-multiple-table-rows-in-one-request", + "title": "Send multiple table rows in one request" }, { - "url": "https://supabase.com/docs/guides/api/creating-routes", - "title": "Creating API Routes" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#limitations", + "title": "Limitations" }, { - "url": "https://supabase.com/docs/guides/api/quickstart", - "title": "Build an API route in less than 2 minutes." - } - ], - "resultChars": 27813 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- app/package.json ---'; cat app/package.json; printf '%s\\\\n' '--- git status ---'; git status --short; printf '%s\\\\n' '--- supabase status ---'; supabase status -o env; printf '%s\\\\n' '--- relevant changelog scan ---'; curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|secret key|supabase-js|postgrest|data api' | head -80\"", - "pages": [ + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#resources", + "title": "Resources" + }, { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 15363 - } - ] - }, - "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", - "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.6/build-dataapi-002-restock-alert-report.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-database-001-migrate-postgres-to-supabase", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true - }, - { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true - }, - { - "name": "foreign key constraints survived the restore", - "passed": true - }, - { - "name": "tasks_team_status_idx index survived the restore", - "passed": true - }, - { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"restore pg_restore local Supabase binary dump roles ownership extensions\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#enable-the-extension", + "title": "Enable the extension" + }, { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#delete-usage", + "title": "Usage " }, { - "url": "https://supabase.com/docs/guides/self-hosting/custom-postgres-extensions", - "title": "Custom Postgres Extensions" + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/transferring-from-cloud-to-self-host-in-supabase-2oWNvW", - "title": "Transferring from platform to self-hosted Supabase" + "url": "https://supabase.com/docs/guides/cron/quickstart#edit-a-job", + "title": "Edit a job" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", - "title": "Migrate from Postgres to Supabase" + "url": "https://supabase.com/docs/guides/cron/quickstart#call-a-database-stored-procedure", + "title": "Call a database stored procedure" }, { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - } - ], - "resultChars": 50510 - } - ] - }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-database-001-migrate-postgres-to-supabase.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-functions-004-service-role-bypass", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "rls", - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "rejects missing auth", - "passed": true, - "notes": "status=401" - }, - { - "name": "user A reads own note", - "passed": true, - "notes": "status=200" - }, - { - "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" - }, - { - "name": "user A cannot force-read user B note", - "passed": true, - "notes": "status=200" - }, - { - "name": "user B cannot force-read user A note", - "passed": true, - "notes": "status=200" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions authenticate user Authorization header createClient service role getUser RLS\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/cron/quickstart#call-a-database-function-every-5-minutes", + "title": "Call a database function every 5 minutes" + }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" + "url": "https://supabase.com/docs/guides/cron/quickstart#run-a-vacuum-every-day", + "title": "Run a vacuum every day" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/cron/quickstart#delete-data-every-week", + "title": "Delete data every week" }, { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" + "url": "https://supabase.com/docs/guides/cron/quickstart#examples", + "title": "Examples" }, { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - } - ], - "resultChars": 73053 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions auth getUser Authorization header SUPABASE_ANON_KEY createClient\", limit: 10) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/cron/quickstart#inspecting-job-runs", + "title": "Inspecting job runs" + }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/cron/quickstart#unschedule-a-job", + "title": "Unschedule a job" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/cron/quickstart#activatedeactivate-a-job", + "title": "Activate/Deactivate a job" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/cron/quickstart#schedule-a-job", + "title": "Schedule a job" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/guides/cron/quickstart#caution-scheduling-system-maintenance", + "title": "Caution: Scheduling system maintenance" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/cron/quickstart#invoke-supabase-edge-function-every-30-seconds", + "title": "Invoke Supabase Edge Function every 30 seconds" }, { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicdeletequeue_name-message_id", + "title": "pgmq_public.delete(queue_name, message_id)" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicpopqueue_name", + "title": "pgmq_public.pop(queue_name)" }, { - "url": "https://supabase.com/docs/guides/functions/storage-caching", - "title": "Integrating with Supabase Storage" - } - ], - "resultChars": 96751 - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [ + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicsendqueue_name-message-sleep_seconds", + "title": "pgmq_public.send(queue_name, message, sleep_seconds)" + }, { - "url": "https://supabase.com/changelog.md" - } - ] - }, - { - "source": "web_search", - "query": "site:supabase.com/changelog.md Supabase changelog breaking change Edge Functions auth API keys", - "pages": [] - } - ] - }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-functions-004-service-role-bypass.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-functions-005-dual-auth-user-secret", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "sdk", - "rls", - "security" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"c2a94e60-d0fd-41fa-9393-3d88c36ccd1e\",\"metric\":\"steps_a_msj16f6m\",\"value\":111}]" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: [{\"user_id\":\"c2a94e60-d0fd-41fa-9393-3d88c36ccd1e\",\"metric\":\"steps_a_msj16f6m\",\"value\":111}]" - }, - { - "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"29d6a26d-c31f-496e-a486-a4bd4a0adeee\",\"metric\":\"steps_b_msj16f6m\",\"value\":222}]" - }, - { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "a user token in the apikey slot is not treated as the service key", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "implementation uses @supabase/server", - "passed": true, - "notes": "imports @supabase/server / withSupabase" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions JWT verification authorization header service role apikey header Deno serve createClient auth getUser\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicsend_batchqueue_name-messages-sleep_seconds", + "title": "pgmq_public.send_batch(queue_name, messages, sleep_seconds)" + }, { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicarchivequeue_name-message_id", + "title": "pgmq_public.archive(queue_name, message_id)" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicreadqueue_name-sleep_seconds-n", + "title": "pgmq_public.read(queue_name, sleep_seconds, n)" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" + "url": "https://supabase.com/docs/guides/queues/quickstart#concepts", + "title": "Concepts" }, { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" + "url": "https://supabase.com/docs/guides/queues/quickstart#enqueueing-and-dequeueing-messages", + "title": "Enqueueing and dequeueing messages" }, { - "url": "https://supabase.com/docs/guides/functions/http-methods", - "title": "Routing" + "url": "https://supabase.com/docs/guides/queues/quickstart#grant-permissions-to-pgmq_public-database-functions", + "title": "Grant permissions to pgmq_public database functions" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/queues/quickstart#enable-rls-on-your-tables-in-pgmq-schema", + "title": "Enable RLS on your tables in pgmq schema" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - } - ], - "resultChars": 60939 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- CHANGELOG BREAKING TAGS ---'; curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|edge function|secret key|service.role' | head -80; printf '%s\\\\n' '--- PROJECT FILES ---'; find .. -name AGENTS.md -print; rg --files -g '\"'!node_modules'\"' -g '\"'!vendor'\"' | sort | sed -n '1,240p'; printf '%s\\\\n' '--- SUPABASE CONFIG ---'; test -f supabase/config.toml && sed -n '1,240p' supabase/config.toml || true; printf '%s\\\\n' '--- MIGRATION MATCHES ---'; rg -n \\\"user_stats|create policy|enable row level security\\\" supabase . --glob '*.sql' --glob '\"'!node_modules'\"' || true\"", - "pages": [ + "url": "https://supabase.com/docs/guides/queues/quickstart#expose-queues-to-client-side-consumers", + "title": "Expose Queues to client-side consumers" + }, { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/queues/quickstart#what-happens-when-you-create-a-queue", + "title": "What happens when you create a queue?" + }, + { + "url": "https://supabase.com/docs/guides/queues/quickstart#create-queues", + "title": "Create Queues" + }, + { + "url": "https://supabase.com/docs/guides/queues/quickstart#queue-types", + "title": "Queue types" + }, + { + "url": "https://supabase.com/docs/guides/queues/quickstart#message", + "title": "Message" + }, + { + "url": "https://supabase.com/docs/guides/queues/quickstart#pull-based-queue", + "title": "Pull-Based Queue" } ], - "resultChars": 24755 + "resultChars": 111443 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Securing Edge Functions verify_jwt false custom JWT validation apikey service role key Authorization header\", limit: 10) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Queues API Reference pgmq_public read delete send queue_name sleep_seconds n\", limit: 10) { nodes { __typename title href content ... on Guide { subsections { nodes { title href content } } } ... on ClientLibraryFunctionReference { language methodName } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicdeletequeue_name-message_id", + "title": "pgmq_public.delete(queue_name, message_id)" }, { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicreadqueue_name-sleep_seconds-n", + "title": "pgmq_public.read(queue_name, sleep_seconds, n)" }, { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicpopqueue_name", + "title": "pgmq_public.pop(queue_name)" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicsendqueue_name-message-sleep_seconds", + "title": "pgmq_public.send(queue_name, message, sleep_seconds)" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicsend_batchqueue_name-messages-sleep_seconds", + "title": "pgmq_public.send_batch(queue_name, messages, sleep_seconds)" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicarchivequeue_name-message_id", + "title": "pgmq_public.archive(queue_name, message_id)" }, { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" + "url": "https://supabase.com/docs/guides/queues/pgmq#list_queues", + "title": "list_queues" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - } - ], - "resultChars": 121781 - } - ] - }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.6/build-functions-005-dual-auth-user-secret.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-storage-001-private-bucket-access", - "stage": "build", - "product": [ - "storage", - "database" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true - }, - { - "name": "RLS still enabled on storage.objects", - "passed": true - }, - { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 019fdc8e-c286-72ec-86c2-693b611d4fc9/receipt-alpha.pdf, 019fdc8e-c286-72ec-86c2-693b611d4fc9/receipt-beta.pdf" - }, - { - "name": "user B cannot read user A files", - "passed": true - }, - { - "name": "anon reads no files", - "passed": true - }, - { - "name": "user A can upload into own folder", - "passed": true - }, - { - "name": "user B cannot upload into user A folder", - "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "The answer creates a private user-files bucket, keeps storage.objects RLS enabled, adds authenticated SELECT and INSERT policies scoped to bucket and user-owned path via auth.uid(), and provides supabase-js createSignedUrl with a 15-minute expiry. No fail conditions present." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Storage access control RLS policies foldername auth.uid createSignedUrl private bucket upload download\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/queues/pgmq#metrics", + "title": "metrics" + }, { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" + "url": "https://supabase.com/docs/guides/queues/pgmq#metrics_all", + "title": "metrics_all" }, { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" + "url": "https://supabase.com/docs/guides/queues/pgmq#types", + "title": "Types" }, { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/queues/pgmq#message_record", + "title": "message_record" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/queues/pgmq#resources", + "title": "Resources" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started", - "title": "Getting Started with Realtime" + "url": "https://supabase.com/docs/guides/queues/pgmq#features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", - "title": "Custom Roles" + "url": "https://supabase.com/docs/guides/queues/pgmq#enable-the-extension", + "title": "Enable the extension" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/why-cant-i-uploadlistetc-my-public-bucket-Z6CmGt", - "title": "Why can't I upload/list/etc my public bucket?" + "url": "https://supabase.com/docs/guides/queues/pgmq#get-usage", + "title": "Usage " }, { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - } - ], - "resultChars": 46298 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|storage|signed url|rls' | head -80\"", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/queues/pgmq#queue-management", + "title": "Queue management" + }, { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 14020 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript createSignedUrl storage expiresIn private bucket signed URL\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/queues/pgmq#create", + "title": "create" + }, { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", - "title": "Storage Image Transformations" + "url": "https://supabase.com/docs/guides/queues/pgmq#create_unlogged", + "title": "create_unlogged" }, { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" + "url": "https://supabase.com/docs/guides/queues/pgmq#detach_archive", + "title": "detach_archive" }, { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" + "url": "https://supabase.com/docs/guides/queues/pgmq#drop_queue", + "title": "drop_queue" }, { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" + "url": "https://supabase.com/docs/guides/queues/pgmq#sending-messages", + "title": "Sending messages" }, { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - } - ], - "resultChars": 25918 - } - ] - }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-storage-001-private-bucket-access.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-tests-001-rls-tenant-isolation", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "tests", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" - }, - { - "name": "pgTAP isolation tests ran and pass", - "passed": true, - "notes": "36 passed, 0 failed" - }, - { - "name": "agent correctly identifies the posts isolation bug from test results", - "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as having a tenant isolation flaw: the policy omits `org_id`, allowing tenant A to see posts from tenant B. It grounds this in reproduced query results and adds pgTAP coverage. It does not blame `notes` instead or dismiss test results." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"row level security testing policies auth.uid tenant isolation pgTAP\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/queues/pgmq#send", + "title": "send" + }, { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" + "url": "https://supabase.com/docs/guides/queues/pgmq#send_batch", + "title": "send_batch" }, { - "url": "https://supabase.com/docs/guides/local-development/testing/overview", - "title": "Testing Overview" + "url": "https://supabase.com/docs/guides/queues/pgmq#reading-messages", + "title": "Reading messages" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pgtap", - "title": "pgTAP: Unit Testing" + "url": "https://supabase.com/docs/guides/queues/pgmq#read", + "title": "read" }, { - "url": "https://supabase.com/docs/guides/database/testing", - "title": "Testing Your Database" + "url": "https://supabase.com/docs/guides/queues/pgmq#read_with_poll", + "title": "read_with_poll" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - } - ], - "resultChars": 56902 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- skill relevant sections ---'; rg -n \\\"RLS|policy|test|docs|changelog\\\" .claude/skills/supabase/SKILL.md; printf '%s\\\\n' '--- security references ---'; find .claude/skills/supabase-postgres-best-practices/references -maxdepth 1 -type f | sort | rg 'security|index'; printf '%s\\\\n' '--- read security rules ---'; cat .claude/skills/supabase-postgres-best-practices/references/security-rls.md 2>/dev/null || true; cat .claude/skills/supabase-postgres-best-practices/references/security-rls-performance.md 2>/dev/null || true; printf '%s\\\\n' '--- changelog relevant ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|row.level|rls|pgtap' | head -n 120; printf '%s\\\\n' '--- workspace ---'; find . -name AGENTS.md -print; ls -la; printf '%s\\\\n' '--- repo files ---'; rg --files -g '\"'!node_modules'\"' -g '\"'!vendor'\"' | sed -n '1,240p'\"", - "pages": [ + "url": "https://supabase.com/docs/guides/queues/pgmq#pop", + "title": "pop" + }, { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 17714 - } - ] - }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-tests-001-rls-tenant-isolation.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-vectors-001-rag-with-permissions", - "stage": "build", - "product": [ - "database", - "vectors" - ], - "topic": [ - "sql", - "rls" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "document_sections.embedding is vector(384)", - "passed": true, - "notes": "vector(384)" - }, - { - "name": "HNSW index on the embedding column", - "passed": true, - "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" - }, - { - "name": "index operator class matches the search operator", - "passed": true, - "notes": "function operators: <#>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" - }, - { - "name": "user A search returns only own sections, best match first", - "passed": true - }, - { - "name": "user B search returns only own sections, best match first", - "passed": true - }, - { - "name": "user A reads only own sections through the API", - "passed": true - }, - { - "name": "user A reads only own documents through the API", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"pgvector semantic search match documents RLS HNSW vector extension Edge Functions\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/queues/pgmq#deletingarchiving-messages", + "title": "Deleting/Archiving messages" + }, { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" + "url": "https://supabase.com/docs/guides/queues/pgmq#delete-single", + "title": "delete (single)" }, { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" + "url": "https://supabase.com/docs/guides/queues/pgmq#delete-batch", + "title": "delete (batch)" }, { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/queues/pgmq#purge_queue", + "title": "purge_queue" }, { - "url": "https://supabase.com/docs/guides/ai/vector-indexes", - "title": "Vector indexes" + "url": "https://supabase.com/docs/guides/queues/pgmq#archive-single", + "title": "archive (single)" }, { - "url": "https://supabase.com/docs/guides/ai/going-to-prod", - "title": "Going to Production" - } - ], - "resultChars": 52858 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Edge Functions gte-small vector dimensions 384 match_document_sections\", limit: 8) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/queues/pgmq#archive-batch", + "title": "archive (batch)" + }, { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" + "url": "https://supabase.com/docs/guides/queues/pgmq#utilities", + "title": "Utilities" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" + "url": "https://supabase.com/docs/guides/queues/pgmq#set_vt", + "title": "set_vt" }, { - "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", - "title": "Choosing your Compute Add-on" + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" }, { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" + "url": "https://supabase.com/docs/guides/queues#resources", + "title": "Resources" }, { - "url": "https://supabase.com/docs/guides/functions/recursive-functions", - "title": "Recursive / Nested Function Calls" + "url": "https://supabase.com/docs/guides/queues#features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/ai/langchain", - "title": "LangChain" + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" }, { - "url": "https://supabase.com/docs/guides/functions/architecture", - "title": "Edge Functions Architecture" + "url": "https://supabase.com/docs/guides/queues/quickstart#enable-rls-on-your-tables-in-pgmq-schema", + "title": "Enable RLS on your tables in pgmq schema" }, { - "url": "https://supabase.com/docs/guides/storage/vector/working-with-indexes", - "title": "Working with Vector Indexes" - } - ], - "resultChars": 110370 - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [ + "url": "https://supabase.com/docs/guides/queues/quickstart#grant-permissions-to-pgmq_public-database-functions", + "title": "Grant permissions to pgmq_public database functions" + }, { - "url": "https://supabase.com/changelog.md" - } - ] - }, - { - "source": "web_search", - "query": "site:supabase.com/changelog.md Supabase changelog pgvector Edge Functions semantic search", - "pages": [] - } - ] - }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-vectors-001-rag-with-permissions.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "deploy-database-001-prometheus-metrics", - "stage": "deploy", - "product": [ - "database" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "passed": true, - "checks": [ - { - "name": "preserved existing app scrape job", - "passed": true - }, - { - "name": "configured the Supabase Metrics API scrape correctly", - "passed": true, - "judgeNotes": "Prometheus preserves the app job and adds a deployable Supabase scrape over HTTPS to /customer/v1/privileged/metrics for evalshostedprojectxy.supabase.co using basic_auth with password_file. docker-compose wires the matching password file via a Compose secret." - }, - { - "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README includes creating a Supabase Secret API key, storing it in the Compose secret file path, recreating the stack, and verifying via Prometheus targets plus a direct metrics API curl check." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Prometheus metrics endpoint customer v1 privileged metrics service_role basic auth hosted project\", limit: 5) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" + "url": "https://supabase.com/docs/guides/queues/quickstart#enqueueing-and-dequeueing-messages", + "title": "Enqueueing and dequeueing messages" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" + "url": "https://supabase.com/docs/guides/queues/quickstart#concepts", + "title": "Concepts" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" + "url": "https://supabase.com/docs/guides/queues/quickstart#pull-based-queue", + "title": "Pull-Based Queue" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - } - ], - "resultChars": 20081 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i 'metrics|prometheus|breaking-change' | head -80\"", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/queues/quickstart#message", + "title": "Message" + }, { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 3574 - } - ] - }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", - "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/deploy-database-001-prometheus-metrics.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "deploy-functions-001-edge-function-secrets", - "stage": "deploy", - "product": [ - "edge-functions" - ], - "topic": [ - "security" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "WEATHER_API_KEY is set as a Function secret on the project", - "passed": true - }, - { - "name": "the weather function is deployed to the project", - "passed": true, - "notes": "status ACTIVE" - }, - { - "name": "the weather function reads WEATHER_API_KEY from the environment", - "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." - }, - { - "name": "WEATHER_API_KEY value is not committed to the repo", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions secrets environment variables WEATHER_API_KEY deploy CLI invoke CORS\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/queues/quickstart#queue-types", + "title": "Queue types" + }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/queues/quickstart#create-queues", + "title": "Create Queues" }, { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/queues/quickstart#what-happens-when-you-create-a-queue", + "title": "What happens when you create a queue?" }, { - "url": "https://supabase.com/docs/guides/functions/development-environment", - "title": "Development Environment" + "url": "https://supabase.com/docs/guides/queues/quickstart#expose-queues-to-client-side-consumers", + "title": "Expose Queues to client-side consumers" }, { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting", - "title": "Troubleshooting" + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#concepts", + "title": "Concepts" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#consuming-messages-in-an-edge-function", + "title": "Consuming messages in an Edge Function" } ], - "resultChars": 64693 + "resultChars": 141154 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Edge Functions deploy secrets set env-file Deno.env.get invoke no verify jwt browser CORS\", limit: 10) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase Cron schedule SQL cron.schedule job name every minute pgmq.send queue message\", limit: 8) { nodes { __typename title href content ... on Guide { subsections { nodes { title href content } } } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" }, { - "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", - "title": "Getting Started with Edge Functions (Dashboard)" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#why-not-generate-all-embeddings-in-a-single-edge-function-request", + "title": "Why not generate all embeddings in a single Edge Function request?" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#why-not-one-request-per-row", + "title": "Why not one request per row?" }, { - "url": "https://supabase.com/docs/guides/ai/examples/openai", - "title": "Generating OpenAI GPT3 completions" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#why-queue-requests-instead-of-processing-them-immediately", + "title": "Why queue requests instead of processing them immediately?" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#is-10-seconds-a-good-interval-for-processing", + "title": "Is 10 seconds a good interval for processing?" }, { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#how-do-we-handle-retries", + "title": "How do we handle retries?" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#how-do-visibility-timeouts-work", + "title": "How do visibility timeouts work?" }, { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#understanding-the-challenge", + "title": "Understanding the challenge" }, { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#understanding-the-architecture", + "title": "Understanding the architecture" }, { - "url": "https://supabase.com/docs/guides/functions/quickstart", - "title": "Getting Started with Edge Functions" - } - ], - "resultChars": 82037 - } - ] - }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/deploy-functions-001-edge-function-secrets.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", - "product": [ - "database", - "auth", - "storage" - ], - "topic": [ - "self-hosting" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true - }, - { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true - }, - { - "name": "secrets rotated off the shipped defaults", - "passed": true - }, - { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"self-hosting Docker docker-compose .env secrets JWT keys Vault logflare pooler official\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#implementation", + "title": "Implementation" + }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#step-1-enable-extensions", + "title": "Step 1: Enable extensions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", - "title": "Configure Social Login (OAuth) Providers" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#step-3-create-queue-and-triggers", + "title": "Step 3: Create queue and triggers" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#step-2-create-utility-functions", + "title": "Step 2: Create utility functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#see-also", + "title": "See also" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-phone-mfa", - "title": "Configure Phone Login & MFA" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#conclusion", + "title": "Conclusion" }, { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates", - "title": "Custom Email Templates" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#troubleshooting", + "title": "Troubleshooting" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#3-insert-and-update-documents", + "title": "3. Insert and update documents" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - } - ], - "resultChars": 131466 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"set -e\nprintf '%s\\\\n' '--- changelog tags relevant to self-hosting ---'\ncurl -fsSL https://supabase.com/changelog.md | rg -i -n -m 30 'breaking-change|self-host|docker|auth key|api key' || true\nrm -rf /tmp/supabase-upstream\ngit clone --depth 1 --filter=blob:none --sparse https://github.com/supabase/supabase.git /tmp/supabase-upstream >/tmp/supabase-clone.log 2>&1\ngit -C /tmp/supabase-upstream sparse-checkout set docker\nprintf '%s\\\\n' '--- upstream revision ---'\ngit -C /tmp/supabase-upstream rev-parse HEAD\ngit -C /tmp/supabase-upstream log -1 --format='%cI %s'\nprintf '%s\\\\n' '--- docker files ---'\nfind /tmp/supabase-upstream/docker -maxdepth 3 -type f | sed 's#/tmp/supabase-upstream/docker/##' | sort | sed -n '1,240p'\nprintf '%s\\\\n' '--- docker changelog head ---'\nsed -n '1,220p' /tmp/supabase-upstream/docker/CHANGELOG.md\nprintf '%s\\\\n' '--- env variables ---'\nsed -n '1,260p' /tmp/supabase-upstream/docker/.env.example\"", - "pages": [ + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#optional-clearing-embeddings-on-update", + "title": "(Optional) Clearing embeddings on update" + }, { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 29108 - } - ] - }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/deploy-self-hosting-001-docker-compose.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", - "product": [ - "auth" - ], - "topic": [ - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "victim session active before delete-account", - "passed": true - }, - { - "name": "delete_account flow ran for the victim", - "passed": true - }, - { - "name": "delete-account revokes the user's sessions", - "passed": true, - "notes": "sessions left: 0" - }, - { - "name": "deleted user's refresh token is rejected", - "passed": true - }, - { - "name": "deleted user cannot sign back in", - "passed": true - }, - { - "name": "other users keep their sessions and access", - "passed": true - }, - { - "name": "diagnosed and explained session revocation", - "passed": true, - "judgeNotes": "The answer identifies the soft-delete-only root cause, implements deletion of auth.users and RLS checks to close the data path for stale JWTs, consistently explains that JWTs remain cryptographically valid until exp while data access is blocked by the implemented RLS, and correctly distinguishes publishable/frontend/RLS keys from secret/backend/bypass-RLS keys." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"delete user existing access token session revoke refresh token admin deleteUser signOut scope global\", limit: 8) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#2-create-triggers-to-enqueue-embedding-jobs", + "title": "2. Create triggers to enqueue embedding jobs" + }, { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#1-create-table-to-store-documents-with-embeddings", + "title": "1. Create table to store documents with embeddings" }, { - "url": "https://supabase.com/docs/guides/auth/signout", - "title": "Signing out" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#usage", + "title": "Usage" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#step-4-create-the-edge-function", + "title": "Step 4: Create the Edge Function" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-signout" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#inspecting-request-data", + "title": "Inspecting request data" }, { - "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#delete-usage", + "title": "Usage " }, { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#analyzing-responses", + "title": "Analyzing responses" }, { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - } - ], - "resultChars": 68746 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"cat .agents/skills/supabase-postgres-best-practices/references/security-privileges.md; printf '\\\\n---RLS---\\\\n'; cat .agents/skills/supabase-postgres-best-practices/references/security-rls-basics.md; printf '\\\\n---CHANGELOG MATCHES---\\\\n'; curl -fsSL https://supabase.com/changelog.md | rg -i -n 'breaking-change|auth|jwt|api key|publishable|secret' | head -80\"", - "pages": [ + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#debugging-requests", + "title": "Debugging requests" + }, { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 14841 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"publishable key secret key RLS bypass service_role anon new API keys sb_publishable sb_secret\", limit: 10) { nodes { ... on Guide { title href content } ... on ManagementApiReference { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#enable-the-extension", + "title": "Enable the extension" + }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_get", + "title": "http_get" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-signature", + "title": "Signature " }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-usage", + "title": "Usage " }, { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_post", + "title": "http_post" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-signature", + "title": "Signature " }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-usage", + "title": "Usage " }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_delete", + "title": "http_delete" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-signature-1", + "title": "Signature " }, { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - } - ], - "resultChars": 137700 - } - ] - }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/investigate-auth-001-deleted-user-access.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "investigate-realtime-001-subscribed-no-events", - "stage": "investigate", - "product": [ - "realtime", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "orders table added to supabase_realtime publication", - "passed": true - }, - { - "name": "courier_locations still in supabase_realtime publication", - "passed": true - }, - { - "name": "publication still publishes INSERT events", - "passed": true - }, - { - "name": "RLS still enabled on orders", - "passed": true - }, - { - "name": "staff can still read orders through RLS", - "passed": true, - "notes": "authenticated sees 2 of 2 orders" - }, - { - "name": "diagnosed missing publication membership", - "passed": true, - "judgeNotes": "Identified orders missing from supabase_realtime despite SUBSCRIBED channel, fixed with ALTER PUBLICATION ADD TABLE public.orders, preserved courier_locations and RLS/policies." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- files ---'; rg --files -g '\"'!node_modules'\"' -g '\"'!dist'\"' | sed -n '1,240p'; printf '%s\\\\n' '--- realtime/order refs ---'; rg -n \\\"postgres_changes|orders|courier|supabase_realtime|publication\\\" -g '\"'!node_modules'\"' -g '\"'!dist'\"' . || true; printf '%s\\\\n' '--- changelog realtime refs ---'; curl -fsSL https://supabase.com/changelog.md | rg -n -i \\\"realtime|postgres_changes|publication\\\" | head -40 || true\"", - "pages": [ + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#inspecting-failed-requests", + "title": "Inspecting failed requests" + }, { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 28635 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Postgres Changes enable table supabase_realtime publication add table postgres_changes RLS\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#configuration", + "title": "Configuration" + }, { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", - "title": "Subscribing to Database Changes" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-current-settings", + "title": "Get current settings" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes", - "title": "Postgres Changes" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#alter-settings", + "title": "Alter settings" }, { - "url": "https://supabase.com/docs/guides/realtime/authorization", - "title": "Realtime Authorization" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#examples", + "title": "Examples" }, { - "url": "https://supabase.com/docs/guides/realtime/benchmarks", - "title": "Benchmarks" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#invoke-a-supabase-edge-function", + "title": "Invoke a Supabase Edge Function" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#call-an-endpoint-every-minute-with-pg_cron", + "title": "Call an endpoint every minute with pg_cron" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#execute-pg_net-in-a-trigger", + "title": "Execute pg_net in a trigger" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#send-multiple-table-rows-in-one-request", + "title": "Send multiple table rows in one request" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#limitations", + "title": "Limitations" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#resources", + "title": "Resources" + }, + { + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" + }, + { + "url": "https://supabase.com/docs/guides/cron#resources", + "title": "Resources" + }, + { + "url": "https://supabase.com/docs/guides/cron#how-does-cron-work", + "title": "How does Cron work?" + }, + { + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" + }, + { + "url": "https://supabase.com/docs/guides/queues#features", "title": "Features" - } - ], - "resultChars": 92624 - } - ] - }, - "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", - "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/investigate-realtime-001-subscribed-no-events.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "investigate-reliability-003-edge-function-5xx-correlation", - "stage": "investigate", - "product": [ - "edge-functions" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "identified image-transform and the recurring 503 pattern", - "passed": true, - "judgeNotes": "The assistant explicitly identified `image-transform` as affected and described eight recurring 503 responses across 07:00–12:00 UTC on 2026-04-28, matching the required pattern." - }, - { - "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": true, - "judgeNotes": "The assistant explicitly attributes the recurring image-transform 503s to the API Gateway/Edge Runtime/platform layer before invocation, not function code. This is grounded in valid observations: the 503s reached the gateway but had no Edge Function execution records, nearby invocations succeeded, and it distinguishes the unrelated avatar-upload 500 as an application-level error. It also advises not to redeploy or modify function/storage/database config." - }, - { - "name": "recommended a concrete next step", - "passed": true, - "judgeNotes": "The assistant recommended concrete next steps, including opening a Supabase support ticket with the project, UTC window, and request IDs, plus capturing specific failure metadata and adding retries." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [ + }, { - "url": "https://supabase.com/changelog.md" - } - ] - }, - { - "source": "web_search", - "query": "site:supabase.com/changelog.md Supabase changelog storage April 2026", - "pages": [] - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions 503 gateway before invocation logs troubleshooting retry transient errors\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/queues#resources", + "title": "Resources" + }, { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-503-response", - "title": "Edge Function 503 error response" + "url": "https://supabase.com/docs/guides/cron/install#uninstall", + "title": "Uninstall" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-504-error-response", - "title": "Edge Function 504 error response" + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" }, { - "url": "https://supabase.com/docs/guides/functions/recursive-functions", - "title": "Recursive / Nested Function Calls" + "url": "https://supabase.com/docs/guides/functions/schedule-functions#resources", + "title": "Resources" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - } - ], - "resultChars": 43053 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"function_edge_logs execution_id function_id internal_failure 503\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/functions/schedule-functions#examples", + "title": "Examples" + }, { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-503-response", - "title": "Edge Function 503 error response" + "url": "https://supabase.com/docs/guides/functions/schedule-functions#invoke-an-edge-function-every-minute", + "title": "Invoke an Edge Function every minute" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-504-error-response", - "title": "Edge Function 504 error response" + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-500-error-response", - "title": "Edge Function 500 error response" - } - ], - "resultChars": 32009 - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/troubleshooting/edge-function-503-response", - "pages": [ + "url": "https://supabase.com/docs/guides/cron/quickstart#inspecting-job-runs", + "title": "Inspecting job runs" + }, { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-503-response" - } - ] - } - ] - }, - "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", - "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.6/investigate-reliability-003-edge-function-5xx-correlation.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", - "product": [ - "data-api", - "database", - "auth" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS still enabled on bookmarks", - "passed": true - }, - { - "name": "user A reads own bookmarks", - "passed": true - }, - { - "name": "user B cannot read user A bookmarks", - "passed": true - }, - { - "name": "anon reads no bookmarks", - "passed": true - }, - { - "name": "user A can save a new bookmark", - "passed": true - }, - { - "name": "user B cannot insert a bookmark as user A", - "passed": true - }, - { - "name": "diagnosed RLS and added owner-scoped policies", - "passed": true, - "judgeNotes": "Diagnosed RLS enabled with no policies causing deny-all Data API behavior, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts. No permissive/public/anon policies or RLS disabling." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"cat .agents/skills/supabase-postgres-best-practices/references/security-rls-basics.md; printf '\\\\n---PERF---\\\\n'; cat .agents/skills/supabase-postgres-best-practices/references/security-rls-performance.md; printf '\\\\n---PRIVILEGES---\\\\n'; cat .agents/skills/supabase-postgres-best-practices/references/security-privileges.md; printf '\\\\n---CHANGELOG MATCHES---\\\\n'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|data api|row level|rls' | head -120\"", - "pages": [ + "url": "https://supabase.com/docs/guides/cron/quickstart#unschedule-a-job", + "title": "Unschedule a job" + }, { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/cron/quickstart#activatedeactivate-a-job", + "title": "Activate/Deactivate a job" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#edit-a-job", + "title": "Edit a job" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#schedule-a-job", + "title": "Schedule a job" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#caution-scheduling-system-maintenance", + "title": "Caution: Scheduling system maintenance" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#invoke-supabase-edge-function-every-30-seconds", + "title": "Invoke Supabase Edge Function every 30 seconds" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#call-a-database-stored-procedure", + "title": "Call a database stored procedure" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#call-a-database-function-every-5-minutes", + "title": "Call a database function every 5 minutes" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#run-a-vacuum-every-day", + "title": "Run a vacuum every day" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#delete-data-every-week", + "title": "Delete data every week" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#examples", + "title": "Examples" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#metrics", + "title": "metrics" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#list_queues", + "title": "list_queues" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#set_vt", + "title": "set_vt" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#utilities", + "title": "Utilities" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#archive-batch", + "title": "archive (batch)" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#archive-single", + "title": "archive (single)" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#purge_queue", + "title": "purge_queue" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#delete-batch", + "title": "delete (batch)" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#delete-single", + "title": "delete (single)" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#deletingarchiving-messages", + "title": "Deleting/Archiving messages" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#pop", + "title": "pop" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#read_with_poll", + "title": "read_with_poll" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#read", + "title": "read" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#reading-messages", + "title": "Reading messages" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#send_batch", + "title": "send_batch" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#send", + "title": "send" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#sending-messages", + "title": "Sending messages" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#drop_queue", + "title": "drop_queue" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#detach_archive", + "title": "detach_archive" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#create_unlogged", + "title": "create_unlogged" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#create", + "title": "create" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#queue-management", + "title": "Queue management" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#get-usage", + "title": "Usage " + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#enable-the-extension", + "title": "Enable the extension" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#resources", + "title": "Resources" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#message_record", + "title": "message_record" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#types", + "title": "Types" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#metrics_all", + "title": "metrics_all" } ], - "resultChars": 8535 + "resultChars": 216504 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Data API table grants authenticated RLS select insert policy auth.uid bookmarks\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"cron.schedule job_name schedule command select cron.schedule('job-name', '* * * * *'\", limit: 6) { nodes { __typename title href content ... on Guide { subsections { nodes { title href content } } } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" }, { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/cron/quickstart#caution-scheduling-system-maintenance", + "title": "Caution: Scheduling system maintenance" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" + "url": "https://supabase.com/docs/guides/cron/quickstart#call-a-database-function-every-5-minutes", + "title": "Call a database function every 5 minutes" }, { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" + "url": "https://supabase.com/docs/guides/cron/quickstart#call-a-database-stored-procedure", + "title": "Call a database stored procedure" }, { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - } - ], - "resultChars": 51520 + "url": "https://supabase.com/docs/guides/cron/quickstart#invoke-supabase-edge-function-every-30-seconds", + "title": "Invoke Supabase Edge Function every 30 seconds" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#schedule-a-job", + "title": "Schedule a job" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#edit-a-job", + "title": "Edit a job" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#activatedeactivate-a-job", + "title": "Activate/Deactivate a job" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#unschedule-a-job", + "title": "Unschedule a job" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#inspecting-job-runs", + "title": "Inspecting job runs" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#examples", + "title": "Examples" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#delete-data-every-week", + "title": "Delete data every week" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#run-a-vacuum-every-day", + "title": "Run a vacuum every day" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#enable-the-extension", + "title": "Enable the extension" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_get", + "title": "http_get" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-signature", + "title": "Signature " + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-usage", + "title": "Usage " + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_post", + "title": "http_post" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-signature", + "title": "Signature " + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-usage", + "title": "Usage " + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_delete", + "title": "http_delete" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-signature-1", + "title": "Signature " + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#inspecting-request-data", + "title": "Inspecting request data" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#inspecting-failed-requests", + "title": "Inspecting failed requests" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#configuration", + "title": "Configuration" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-current-settings", + "title": "Get current settings" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#alter-settings", + "title": "Alter settings" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#examples", + "title": "Examples" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#invoke-a-supabase-edge-function", + "title": "Invoke a Supabase Edge Function" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#call-an-endpoint-every-minute-with-pg_cron", + "title": "Call an endpoint every minute with pg_cron" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#execute-pg_net-in-a-trigger", + "title": "Execute pg_net in a trigger" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#send-multiple-table-rows-in-one-request", + "title": "Send multiple table rows in one request" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#limitations", + "title": "Limitations" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#resources", + "title": "Resources" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#debugging-requests", + "title": "Debugging requests" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#analyzing-responses", + "title": "Analyzing responses" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#delete-usage", + "title": "Usage " + }, + { + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" + }, + { + "url": "https://supabase.com/docs/guides/cron/install#uninstall", + "title": "Uninstall" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_cron", + "title": "pg_cron: Schedule Recurring Jobs with Cron Syntax in Postgres" + }, + { + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" + }, + { + "url": "https://supabase.com/docs/guides/cron#how-does-cron-work", + "title": "How does Cron work?" + }, + { + "url": "https://supabase.com/docs/guides/cron#resources", + "title": "Resources" + } + ], + "resultChars": 122750 } ] }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/resolve-dataapi-001-empty-results.json" + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/build-cli-003-pg-cron-queue-workflow.json" }, { "experiment": "codex-gpt-5.6", @@ -15275,42 +14526,44 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "resolve-database-001-migration-history-mismatch", - "stage": "resolve", + "eval": "build-dataapi-001-relational-report", + "stage": "build", "product": [ + "data-api", "database" ], "topic": [ - "migrations" + "sdk" ], "suite": "benchmark", "interface": "cli", - "passed": true, + "cliVersion": "2.109.1", + "passed": false, "checks": [ { - "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true - }, - { - "name": "migration 20240220000000 is recorded in the remote history", - "passed": true + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" }, { - "name": "remote migration history matches local migration files", - "passed": true + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" }, { - "name": "local migrations are a valid reconciled sequence", - "passed": true + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" }, { - "name": "production profile data is intact (not reset)", - "passed": true + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" }, { - "name": "the avatar migration and history reconciliation were done via the Supabase CLI", + "name": "report queries via the Data API, not raw SQL", "passed": true, - "judgeNotes": "PASS. The agent added the missing local bio migration file `supabase/migrations/20240115000000_add_profile_bio.sql`, which reconciled the orphan remote history entry for CLI pushes. The pending avatar migration was then applied through a real `supabase db push` in command #10, whose output includes `Finished supabase db push` (and #11 confirms remote history contains `20240220000000 | add_avatar_url`). No prohibited direct-SQL mutation or prepared-statement workaround was seen." + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -15327,88 +14580,48 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI db push migration history repair linked project deploy migrations\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"supabase javascript select foreign table relationships nested query service role backend\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" + "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", + "title": "Engineering for Scale" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" }, { - "url": "https://supabase.com/docs/reference/cli/supabase-migration-repair", - "title": "Repair the migration history table" + "url": "https://supabase.com/docs/guides/api/sql-to-api", + "title": "Converting SQL to JavaScript API" }, { - "url": "https://supabase.com/docs/reference/cli/supabase-db-push", - "title": "Push new migrations to the remote database" + "url": "https://supabase.com/docs/guides/database/joins-and-nesting", + "title": "Querying Joins and Nested tables" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" } ], - "resultChars": 61657 + "resultChars": 40663 }, { "source": "shell_fetch", - "query": "/bin/bash -lc \"set -o pipefail\nprintf '%s\\\\n' '--- CHANGELOG BREAKING CHANGES ---'\ncurl -fsSL https://supabase.com/changelog.md | rg -n -i -C 2 'breaking-change|migration|db push' | head -120 || true\nprintf '%s\\\\n' '--- GIT STATUS ---'\ngit status --short --branch 2>&1 || true\nprintf '%s\\\\n' '--- SUPABASE FILES ---'\nfind supabase -maxdepth 4 -type f -print | sort\nprintf '%s\\\\n' '--- CONFIG ---'\nsed -n '1,260p' supabase/config.toml 2>/dev/null || true\nprintf '%s\\\\n' '--- MIGRATIONS ---'\nfor f in supabase/migrations/*.sql; do echo \\\"===== \"'$f\"; sed -n '\"'1,260p' \\\"\"'$f\"; done\nprintf '\"'%s\\\\n' '--- LOCAL LINK METADATA ---'\nfind supabase/.temp -maxdepth 2 -type f -print -exec sh -c 'echo \\\"--- \"'$1\"; cat \"$1\"'\"' _ {} \\\\; 2>/dev/null || true\nprintf '%s\\\\n' '--- CLI / ENV NAMES ---'\nsupabase --version\nprintf 'SUPABASE vars: '; env | cut -d= -f1 | rg '\"'^(SUPABASE|POSTGRES|PG)'\"' | sort | tr '\\\\n' ' '; echo\nprintf '%s\\\\n' '--- PROFILE ---'\ncat .supabase-eval-profile.yaml\"", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- changelog relevant scan ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -m 20 'breaking|postgrest|supabase-js|javascript|select|relationship' || true; printf '%s\\\\n' '--- report ---'; sed -n '1,240p' app/report.mjs; printf '%s\\\\n' '--- package ---'; cat app/package.json; printf '%s\\\\n' '--- migration ---'; sed -n '1,280p' supabase/migrations/0000_orders_schema.sql; printf '%s\\\\n' '--- config ---'; sed -n '1,220p' supabase/config.toml; printf '%s\\\\n' '--- status ---'; supabase status\"", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 478 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase migration history remote migration versions not found local migration repair db pull migration fetch\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/new-branch-doesnt-copy-database", - "title": "Why are my Supabase branches empty?" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-migration-repair", - "title": "Repair the migration history table" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/branch-in-migrations-failed-status", - "title": "Troubleshooting MIGRATIONS_FAILED: missing tables or an incomplete schema on your branch" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-push", - "title": "Push new migrations to the remote database" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-migration-fetch", - "title": "Fetch migration files from history table" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-pull", - "title": "Pull schema from the remote database" - } - ], - "resultChars": 58381 + "resultChars": 1507 } ] }, - "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", - "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/resolve-database-001-migration-history-mismatch.json" + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/build-dataapi-001-relational-report.json" }, { "experiment": "codex-gpt-5.6", @@ -15419,39 +14632,44 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "resolve-performance-001-slow-query-cpu-spike", - "stage": "resolve", + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", "product": [ + "data-api", "database" ], "topic": [ - "observability", - "sql" + "sdk" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "inspected pg_stat_statements for query performance", - "passed": true + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" }, { - "name": "ran EXPLAIN on the expensive query", - "passed": true + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" }, { - "name": "created index covering user_id and created_at", - "passed": true + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" }, { - "name": "query plan uses an index and avoids sequential scan", + "name": "implementation uses @supabase/supabase-js", "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "imports found in: app/restock.mjs" }, { - "name": "inserts still work", - "passed": true + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -15468,99 +14686,111 @@ "calls": [ { "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | rg -n -i \"breaking-change|postgres|index|database\" | head -40'", - "hasContent": true, + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- app/restock.mjs ---' && sed -n '1,240p' app/restock.mjs && printf '%s\\\\n' '--- app/package.json ---' && cat app/package.json && printf '%s\\\\n' '--- migration ---' && sed -n '1,320p' supabase/migrations/0000_inventory_schema.sql && printf '%s\\\\n' '--- config ---' && sed -n '1,220p' supabase/config.toml && printf '%s\\\\n' '--- changelog relevant scan ---' && curl -fsSL https://supabase.com/changelog.md | rg -i -n -m 20 'breaking-change|postgrest|supabase-js|javascript'\"", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 7495 + "resultChars": 10796 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Postgres query optimization composite index ORDER BY WHERE EXPLAIN\", limit: 3) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"supabase-js select foreign tables nested relationships service_role secret key Node createClient\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" }, { - "url": "https://supabase.com/docs/guides/database/orioledb", - "title": "OrioleDB Overview" + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/performing-administration-tasks-on-the-server-side-with-the-servicerole-secret-BYM4Fa", + "title": "Performing administration tasks on the server side with a secret key" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/realtime/reports", - "title": "Realtime Reports" + "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", + "title": "Engineering for Scale" } ], - "resultChars": 63315 - } - ] - }, - "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", - "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/resolve-performance-001-slow-query-cpu-spike.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", + "resultChars": 27050 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"JavaScript joins nested tables select foreign key relationships supabase-js\", limit: 4) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/joins-and-nesting", + "title": "Querying Joins and Nested tables" + }, + { + "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", + "title": "Engineering for Scale" + }, + { + "url": "https://supabase.com/docs/guides/api/automatic-retries-in-supabase-js", + "title": "How to do automatic retries with `supabase-js`" + }, + { + "url": "https://supabase.com/docs/guides/database/tables", + "title": "Tables and Data" + } + ], + "resultChars": 61892 + } + ] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/build-dataapi-002-restock-alert-report.json" + }, + { + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "resolve-security-002-rls-cross-tenant-leak", - "stage": "resolve", + "eval": "build-database-001-migrate-postgres-to-supabase", + "stage": "build", "product": [ - "database", - "auth" + "database" ], "topic": [ - "rls", - "security" + "migrations" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "RLS enabled on notes", - "passed": true - }, - { - "name": "tenant A sees only org A notes", - "passed": true - }, - { - "name": "tenant B cannot read org A notes", - "passed": true - }, - { - "name": "tenant A author can update own note", - "passed": true - }, - { - "name": "tenant B cannot update org A note", + "name": "all 3 tables exist (teams, members, tasks)", "passed": true }, { - "name": "tenant B author can delete own note", + "name": "row counts match (teams=5, members=10, tasks=13)", "passed": true }, { - "name": "tenant B cannot delete org A note", + "name": "foreign key constraints survived the restore", "passed": true }, { - "name": "tenant A can insert note in own org", + "name": "tasks_team_status_idx index survived the restore", "passed": true }, { - "name": "tenant B cannot insert into org A", + "name": "sequences synced (next insert won't conflict with existing IDs)", "passed": true } ], @@ -15576,2651 +14806,3519 @@ }, "docs": { "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- files ---'; rg --files -g 'AGENTS.md' -g 'supabase/**' -g 'package.json' -g '*.sql' | sed -n '1,240p'; printf '%s\\\\n' '--- changelog security/rls mentions ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -n -m 20 'breaking-change|row level|rls|policy|security'\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 4349 - }, { "source": "search_docs", - "query": "query { searchDocs(query: \"row level security multi tenant workspace membership policies auth uid index policy performance\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"restore pg_restore custom format dump local Supabase existing Postgres database migrate\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" }, { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" }, { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" }, { - "url": "https://supabase.com/docs/guides/database/database-advisors", - "title": "Performance and Security Advisors" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", + "title": "Migrate from Vercel Postgres to Supabase" }, { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/neon", + "title": "Migrate from Neon to Supabase" + } + ], + "resultChars": 42818 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"set -o pipefail\nprintf '%s\\\\n' '--- workspace ---'\nfind . -maxdepth 2 -type f -printf '%p %s bytes\\\\n' | sort\nprintf '%s\\\\n' '--- versions ---'\nsupabase --version\npg_restore --version\npsql --version\ndocker --version\nprintf '%s\\\\n' '--- dump metadata ---'\nfile source.dump\npg_restore --list source.dump | sed -n '1,80p'\nprintf '%s\\\\n' '--- relevant changelog entries ---'\ncurl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|restore|migration|postgres' | head -n 100 || true\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 48168 + "resultChars": 6870 } ] }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/resolve-security-002-rls-cross-tenant-leak.json" + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/build-database-001-migrate-postgres-to-supabase.json" }, { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "build-auth-001-email-password-flow", + "eval": "build-functions-004-service-role-bypass", "stage": "build", "product": [ + "edge-functions", "auth", "database" ], "topic": [ - "sdk", - "rls" + "rls", + "security", + "sdk" ], "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", + "interface": "mcp", "passed": true, "checks": [ { - "name": "auth module loads and the driver completes", - "passed": true, - "notes": "driver produced a result" - }, - { - "name": "signUp creates the account and returns its user id", - "passed": true, - "notes": "db user 14e42a99-dd08-453c-a361-a8b7871b40cc, signUp returned {\"userId\":\"14e42a99-dd08-453c-a361-a8b7871b40cc\"}" - }, - { - "name": "signup metadata reaches the profile (display name)", - "passed": true, - "notes": "profiles.display_name = \"Alex Doe\"" - }, - { - "name": "wrong password is rejected gracefully (no throw, no session)", + "name": "rejects missing auth", "passed": true, - "notes": "{\"error\":\"Invalid login credentials\"}" + "notes": "status=401" }, { - "name": "signIn with the right password returns the user id", + "name": "user A reads own note", "passed": true, - "notes": "{\"userId\":\"14e42a99-dd08-453c-a361-a8b7871b40cc\"}" + "notes": "status=200" }, { - "name": "getMyProfile returns the signed-in user's profile", + "name": "reads only with the caller's JWT", "passed": true, - "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + "notes": "bearer_tokens=2, all_match=true" }, { - "name": "app code does not use the secret / service-role key", + "name": "user A cannot force-read user B note", "passed": true, - "notes": "no secret-key references found" + "notes": "status=200" }, { - "name": "implementation uses @supabase/supabase-js", + "name": "user B cannot force-read user A note", "passed": true, - "notes": "imports found in: app/src/auth.mjs" + "notes": "status=200" } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js createClient signUp user metadata signInWithPassword getSession select single profiles\", limit: 8) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Edge Functions auth Authorization header getUser service role row level security\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-linkedin", - "title": "Login with LinkedIn" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", - "title": "Login with Figma" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-google", - "title": "Login with Google" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-gitlab", - "title": "Login with GitLab" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-slack", - "title": "Login with Slack" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" } ], - "resultChars": 166581 + "resultChars": 79214 + }, + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "web_search", + "query": "site:supabase.com/changelog Edge Functions breaking change JWT authorization", + "pages": [] } ] }, - "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", - "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-auth-001-email-password-flow.json" + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/build-functions-004-service-role-bypass.json" }, { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "build-cli-001-bootstrap-app", + "eval": "build-functions-005-dual-auth-user-secret", "stage": "build", "product": [ - "database", - "data-api" + "edge-functions", + "auth", + "database" ], "topic": [ - "migrations", - "rls" + "sdk", + "rls", + "security" ], "suite": "benchmark", "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "supabase project initialised (supabase/config.toml exists)", - "passed": true - }, - { - "name": "todos table is created by a migration file", - "passed": true + "name": "seed rows present", + "passed": true, + "notes": "found 2/2 seeded rows" }, { - "name": "todos table exists with at least 2 seeded rows", + "name": "rejects request with no credentials", "passed": true, - "notes": "found 2 rows" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { - "name": "row level security is enabled on todos", - "passed": true + "name": "user with JWT reads only their own rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"5e380a62-99d7-414f-9dbc-8602a96e061b\",\"metric\":\"steps_a_mst1rz2v\",\"value\":111}]" }, { - "name": "a SELECT policy targets the authenticated role", - "passed": true + "name": "user cannot read another user's rows by passing user_id", + "passed": true, + "notes": "status 200: [{\"user_id\":\"5e380a62-99d7-414f-9dbc-8602a96e061b\",\"metric\":\"steps_a_mst1rz2v\",\"value\":111}]" }, { - "name": "REST API returns no todos to anonymous requests", + "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "0 rows" + "notes": "status 200: [{\"user_id\":\"78cb9408-339f-43d9-9800-90040688560a\",\"metric\":\"steps_b_mst1rz2v\",\"value\":222}]" }, { - "name": "REST API returns the todos to authenticated requests", + "name": "non-service key is not granted service access", "passed": true, - "notes": "2 rows" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "rejects an unverified (forged) user token", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "a user token in the apikey slot is not treated as the service key", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "implementation uses @supabase/server", + "passed": true, + "notes": "imports @supabase/server / withSupabase" } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Row Level Security authenticated users SELECT policy anon no rows migrations seed local development\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Edge Functions JWT verification service role apikey Authorization header Deno serve createClient getUser\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/functions/websockets", + "title": "Handling WebSockets" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", + "title": "Resumable WebSockets with Edge Functions" } ], - "resultChars": 95821 - } - ] - }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-cli-001-bootstrap-app.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-cli-002-declarative-schema", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "declarative-schema", - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase db diff used to generate the migration", - "passed": true - }, - { - "name": "schema file updated to include description column", - "passed": true - }, - { - "name": "a new migration was generated for the change", - "passed": true - }, - { - "name": "description column exists in the live database", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ + "resultChars": 39423 + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI create migration alter table add column local database\", limit: 3) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"@supabase/server Edge Functions verifyCredentials createAdminClient createContextClient apikey service role user auth\", limit: 10) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" + "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", + "title": "Resumable WebSockets with Edge Functions" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" + "url": "https://supabase.com/docs/guides/auth/passkeys", + "title": "Passkey authentication" + }, + { + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-createuser" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", + "title": "Which package to use" } ], - "resultChars": 54403 - } - ] - }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-cli-002-declarative-schema.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", - "product": [ - "database", - "edge-functions", - "cron", - "queues" - ], - "topic": [ - "sql", - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" - }, - { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 0 -> 1" - }, - { - "name": "process-tasks function drains the queue", - "passed": true, - "notes": "function removed the seeded message (id 6) from the queue" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ + "resultChars": 77282 + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Queues pgmq create queue send messages Edge Function read delete pgmq_public cron schedule every minute\", limit: 8) { nodes { __typename title href content } } }", + "query": "query { searchDocs(query: \"@supabase/server reference withSupabase auth user service secret apikey auth options either\", limit: 10) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", + "title": "Which package to use" }, { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" + "url": "https://supabase.com/docs/guides/auth/passkeys", + "title": "Passkey authentication" }, { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" + "url": "https://supabase.com/docs/guides/auth/general-configuration", + "title": "General configuration" }, { - "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", - "title": "Expose Queues for local and self-hosted Supabase" + "url": "https://supabase.com/docs/guides/troubleshooting/performing-administration-tasks-on-the-server-side-with-the-servicerole-secret-BYM4Fa", + "title": "Performing administration tasks on the server side with a secret key" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" + }, + { + "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", + "title": "Creating a Supabase client for SSR" } ], - "resultChars": 39939 + "resultChars": 162418 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Queues Quickstart pgmq.create cron.schedule SQL schedule database jobs\", limit: 6) { nodes { __typename title href content } } }", + "query": "query { searchDocs(query: \"Deno.serve(withSupabase Edge Functions @supabase/server\", limit: 10) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" }, { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" + "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", + "title": "Getting Started with Edge Functions (Dashboard)" }, { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", + "title": "Testing and linting" }, { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - } + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/sentry-monitoring", + "title": "Monitoring with Sentry" + }, + { + "url": "https://supabase.com/docs/guides/functions/unit-test", + "title": "Testing your Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart", + "title": "Getting Started with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", + "title": "Which package to use" + } + ], + "resultChars": 77435 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"config.toml functions entrypoint verify_jwt import_map Supabase Edge Function\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" + }, + { + "url": "https://supabase.com/docs/guides/functions/development-tips", + "title": "Development tips" + }, + { + "url": "https://supabase.com/docs/guides/functions/deploy", + "title": "Deploy to Production" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/cloudflare-turnstile", + "title": "CAPTCHA support with Cloudflare Turnstile" + } ], - "resultChars": 42534 + "resultChars": 26101 } ] }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-cli-003-pg-cron-queue-workflow.json" + "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", + "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/build-functions-005-dual-auth-user-secret.json" }, { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "build-dataapi-001-relational-report", + "eval": "build-storage-001-private-bucket-access", "stage": "build", "product": [ - "data-api", + "storage", "database" ], "topic": [ + "rls", "sdk" ], "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, + "interface": "mcp", + "passed": true, "checks": [ { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "report numbers match the database (per customer, sorted)", - "passed": true, - "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + "name": "bucket user-files exists", + "passed": true }, { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table customers" + "name": "bucket user-files is private", + "passed": true }, { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "name": "RLS still enabled on storage.objects", + "passed": true }, { - "name": "report queries via the Data API, not raw SQL", + "name": "user A lists only own files", "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.6-no-skills/build-dataapi-001-relational-report.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-dataapi-002-restock-alert-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ + "notes": "saw: 01a000ac-e224-703c-804f-9bd513c91412/receipt-alpha.pdf, 01a000ac-e224-703c-804f-9bd513c91412/receipt-beta.pdf" + }, { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" + "name": "user B cannot read user A files", + "passed": true }, { - "name": "alerts match the database (below threshold, sorted)", - "passed": true, - "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + "name": "anon reads no files", + "passed": true }, { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table inventory" + "name": "user A can upload into own folder", + "passed": true }, { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "name": "user B cannot upload into user A folder", + "passed": true }, { - "name": "report queries via the Data API, not raw SQL", + "name": "configured private per-user storage access", "passed": true, - "notes": "no psql / raw Postgres driver usage found" + "judgeNotes": "Creates private user-files bucket, adds authenticated SELECT and INSERT policies scoped to bucket and first path segment matching auth.uid(), does not disable RLS or use public URLs/service role, and provides createSignedUrl code with expiry." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript supabase-js select foreign tables nested relationships service role Node backend\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Storage access control policy storage.foldername signed URL createSignedUrl private bucket supabase-js\", limit: 8) { nodes { title href content ... on ClientLibraryFunctionReference { language methodName } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", - "title": "Engineering for Scale" + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" }, { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/firebase-storage", + "title": "Migrated from Firebase Storage to Supabase" }, { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/storage", + "title": "Storage" }, { - "url": "https://supabase.com/docs/guides/database/joins-and-nesting", - "title": "Querying Joins and Nested tables" + "url": "https://supabase.com/docs/guides/functions/examples/amazon-bedrock-image-generator", + "title": "Generate Images with Amazon Bedrock" }, { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" + }, + { + "url": "https://supabase.com/docs/guides/storage/s3/compatibility", + "title": "S3 Compatibility" + }, + { + "url": "https://supabase.com/docs/guides/storage/buckets/creating-buckets", + "title": "Creating Buckets" + }, + { + "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", + "title": "Storage Image Transformations" + } + ], + "resultChars": 80226 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|storage|signed url|signed-url' | head -160\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 11408 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase javascript createSignedUrl download option expiresIn storage\", limit: 5) { nodes { title href content ... on ClientLibraryFunctionReference { language methodName } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", + "title": "Storage Image Transformations" + }, + { + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" + }, + { + "url": "https://supabase.com/docs/guides/storage/uploads/resumable-uploads", + "title": "Resumable Uploads" + }, + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" + }, + { + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" } ], - "resultChars": 45250 + "resultChars": 53197 } ] }, - "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", - "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", + "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", "attempts": 2, - "sourcePath": "codex-gpt-5.6-no-skills/build-dataapi-002-restock-alert-report.json" + "sourcePath": "codex-gpt-5.6/build-storage-001-private-bucket-access.json" }, { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "build-database-001-migrate-postgres-to-supabase", + "eval": "build-tests-001-rls-tenant-isolation", "stage": "build", "product": [ "database" ], "topic": [ - "migrations" + "tests", + "rls" ], "suite": "benchmark", "interface": "cli", "passed": true, "checks": [ { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true - }, - { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true - }, - { - "name": "foreign key constraints survived the restore", - "passed": true + "name": "pgTAP test file(s) written under supabase/tests/", + "passed": true, + "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" }, { - "name": "tasks_team_status_idx index survived the restore", - "passed": true + "name": "pgTAP isolation tests ran and pass", + "passed": true, + "notes": "15 passed, 0 failed" }, { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true + "name": "agent correctly identifies the posts isolation bug from test results", + "passed": true, + "judgeNotes": "The agent correctly identified `posts` as having the tenant isolation flaw, described authenticated org members being able to read other tenants’ posts, and grounded the conclusion in reproduction/test results. It did not blame `notes` or dismiss pgTAP results." } ], "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI restore pg_dump custom format local database migrate existing Postgres\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Row Level Security tenant isolation testing pgTAP auth.uid set_config request.jwt.claims\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", - "title": "Migrate from Postgres to Supabase" + "url": "https://supabase.com/docs/guides/local-development/testing/overview", + "title": "Testing Overview" }, { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" + "url": "https://supabase.com/docs/guides/auth/auth-mfa", + "title": "Multi-Factor Authentication" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", - "title": "Migrate from Vercel Postgres to Supabase" + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/neon", - "title": "Migrate from Neon to Supabase" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" } ], - "resultChars": 42818 + "resultChars": 68172 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- changelog security/rls mentions ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -n -m 20 'breaking-change|row level|rls|pgtap|test db' || true; printf '%s\\\\n' '--- workspace files ---'; rg --files -g 'AGENTS.md' -g '\"'!node_modules'\"' -g '\"'!vendor'\"'; rg --files -g 'supabase/**' -g '*.sql' -g 'package.json' -g 'README*' | sort | sed -n '1,240p'; printf '%s\\\\n' '--- skill security refs ---'; find .claude/skills/supabase-postgres-best-practices/references -maxdepth 1 -type f | sort | rg 'security|rls'\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 4921 } ] }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-database-001-migrate-postgres-to-supabase.json" + "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", + "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/build-tests-001-rls-tenant-isolation.json" }, { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "build-functions-004-service-role-bypass", + "eval": "build-vectors-001-rag-with-permissions", "stage": "build", "product": [ - "edge-functions", - "auth", - "database" + "database", + "vectors" ], "topic": [ - "rls", - "security", - "sdk" + "sql", + "rls" ], "suite": "benchmark", "interface": "mcp", "passed": true, "checks": [ { - "name": "rejects missing auth", + "name": "document_sections.embedding is vector(384)", "passed": true, - "notes": "status=401" + "notes": "vector(384)" }, { - "name": "user A reads own note", + "name": "HNSW index on the embedding column", "passed": true, - "notes": "status=200" + "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { - "name": "reads only with the caller's JWT", + "name": "index operator class matches the search operator", "passed": true, - "notes": "bearer_tokens=2, all_match=true" + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { - "name": "user A cannot force-read user B note", - "passed": true, - "notes": "status=200" + "name": "user A search returns only own sections, best match first", + "passed": true }, { - "name": "user B cannot force-read user A note", - "passed": true, - "notes": "status=200" + "name": "user B search returns only own sections, best match first", + "passed": true + }, + { + "name": "user A reads only own sections through the API", + "passed": true + }, + { + "name": "user A reads only own documents through the API", + "passed": true } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"cat .agents/skills/supabase-postgres-best-practices/references/security-rls-basics.md; printf '\\\\n--- RLS PERF ---\\\\n'; cat .agents/skills/supabase-postgres-best-practices/references/security-rls-performance.md; printf '\\\\n--- INDEX TYPES ---\\\\n'; cat .agents/skills/supabase-postgres-best-practices/references/query-index-types.md; printf '\\\\n--- CHANGELOG VECTOR ITEMS ---\\\\n'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 '(vector|pgvector|breaking-change)' | head -160\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 15395 + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions authenticate user Authorization header getUser createClient SUPABASE_ANON_KEY RLS\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"pgvector semantic search match_documents HNSW vector cosine RLS security invoker function\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/ai/hybrid-search", + "title": "Hybrid search" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/ai/vector-indexes", + "title": "Vector indexes" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/ai/vector-indexes/ivf-indexes", + "title": "IVFFlat indexes" }, { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" + }, + { + "url": "https://supabase.com/docs/guides/ai/vector-columns", + "title": "Vector columns" + }, + { + "url": "https://supabase.com/docs/guides/ai/going-to-prod", + "title": "Going to Production" + }, + { + "url": "https://supabase.com/docs/guides/ai/vector-indexes/hnsw-indexes", + "title": "HNSW indexes" + } + ], + "resultChars": 73600 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase AI gte-small embedding dimensions 384 vector\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", + "title": "Choosing your Compute Add-on" + }, + { + "url": "https://supabase.com/docs/guides/ai/concepts", + "title": "Concepts" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" + }, + { + "url": "https://supabase.com/docs/guides/ai", + "title": "AI & Vectors" + }, + { + "url": "https://supabase.com/docs/guides/ai/vector-columns", + "title": "Vector columns" } ], - "resultChars": 32789 + "resultChars": 53612 } ] }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-functions-004-service-role-bypass.json" + "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", + "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/build-vectors-001-rag-with-permissions.json" }, { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "build-functions-005-dual-auth-user-secret", - "stage": "build", + "eval": "deploy-database-001-prometheus-metrics", + "stage": "deploy", "product": [ - "edge-functions", - "auth", "database" ], "topic": [ - "sdk", - "rls", - "security" + "observability" ], "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"7771b8ed-5e55-4cd4-af22-e43905cae0fe\",\"metric\":\"steps_a_msj0yta6\",\"value\":111}]" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: [{\"user_id\":\"7771b8ed-5e55-4cd4-af22-e43905cae0fe\",\"metric\":\"steps_a_msj0yta6\",\"value\":111}]" - }, - { - "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"7bc5ace7-c5be-42dc-ba44-d5537f012dc1\",\"metric\":\"steps_b_msj0yta6\",\"value\":222}]" - }, - { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "name": "preserved existing app scrape job", + "passed": true }, { - "name": "a user token in the apikey slot is not treated as the service key", + "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "judgeNotes": "Meets rubric: app scrape preserved; Supabase scrape uses HTTPS, /customer/v1/privileged/metrics, Basic Auth with password_file, a supabase.co project target placeholder, and docker-compose wires the same password_file via a Compose secret." }, { - "name": "implementation uses @supabase/server", + "name": "documented live deployment and verification steps", "passed": true, - "notes": "imports @supabase/server / withSupabase" + "judgeNotes": "README includes steps to create a Supabase Secret API key, write it to the expected Compose secret file, set the project ref, recreate the Compose stack, and verify via Prometheus targets or PromQL up{job=\"supabase\"}." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions authenticate user JWT Authorization header service role secret key apikey header verify_jwt false getClaims\", limit: 8) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase project metrics Prometheus endpoint Grafana authentication observability\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", + "title": "Metrics API" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-getclaims" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" }, { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" + "url": "https://supabase.com/docs/guides/database/connection-management", + "title": "Connection management" }, { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/platform/read-replicas", + "title": "Read Replicas" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-getclaims", - "title": "getClaims()" + "url": "https://supabase.com/docs/guides/troubleshooting/how-to-view-database-metrics-uqf2z_", + "title": "How to View Database Metrics" } ], - "resultChars": 46978 + "resultChars": 35772 }, { - "source": "search_docs", - "query": "query { searchDocs(query: \"API keys publishable secret key apikey header Edge Functions secret key service_role authorization\", limit: 10) { nodes { title href content } } }", - "hasContent": true, + "source": "shell_fetch", + "query": "/bin/bash -lc \"set -o pipefail\nprintf '%s\\\\n' '--- CHANGELOG RELEVANCE ---'\ncurl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'metrics|prometheus|breaking-change' | head -120 || true\nprintf '%s\\\\n' '--- AGENTS ---'\nfind .. -name AGENTS.md -print\nprintf '%s\\\\n' '--- ROOT ---'\nls -la\nprintf '%s\\\\n' '--- FILES ---'\nrg --files -g '\"'!node_modules'\"' -g '\"'!vendor'\"' | sed -n '1,240p'\nprintf '%s\\\\n' '--- CLI ---'\nsupabase --version\nsupabase --help | sed -n '1,100p'\"", "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", - "title": "Getting Started with Edge Functions (Dashboard)" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 177206 + "resultChars": 3774 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"@supabase/server Edge Functions API key environment SUPABASE_SECRET_KEYS authenticate request\", limit: 10) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Vendor-agnostic Metrics API Prometheus configuration basic_auth password_file scrape interval 60 seconds\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { - "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration", - "title": "Build a Supabase Integration" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", + "title": "Metrics API" } ], - "resultChars": 90248 + "resultChars": 19981 } ] }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-functions-005-dual-auth-user-secret.json" + "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", + "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/deploy-database-001-prometheus-metrics.json" }, { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "build-storage-001-private-bucket-access", - "stage": "build", + "eval": "deploy-functions-001-edge-function-secrets", + "stage": "deploy", "product": [ - "storage", - "database" + "edge-functions" ], "topic": [ - "rls", - "sdk" + "security" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true - }, - { - "name": "RLS still enabled on storage.objects", + "name": "WEATHER_API_KEY is set as a Function secret on the project", "passed": true }, { - "name": "user A lists only own files", + "name": "the weather function is deployed to the project", "passed": true, - "notes": "saw: 019fdc8d-04dc-730f-a0ac-4ed1bc5a28ce/receipt-alpha.pdf, 019fdc8d-04dc-730f-a0ac-4ed1bc5a28ce/receipt-beta.pdf" - }, - { - "name": "user B cannot read user A files", - "passed": true - }, - { - "name": "anon reads no files", - "passed": true + "notes": "status ACTIVE" }, { - "name": "user A can upload into own folder", - "passed": true + "name": "the weather function reads WEATHER_API_KEY from the environment", + "passed": true, + "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { - "name": "user B cannot upload into user A folder", + "name": "WEATHER_API_KEY value is not committed to the repo", "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Meets rubric: private user-files bucket (public=false), authenticated SELECT and INSERT policies scoped to bucket and auth.uid() path prefix with WITH CHECK for upload, no RLS disabling/permissive public policies, and supabase-js createSignedUrl with expiry for temporary sharing." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Storage access control RLS policies storage.objects foldername auth.uid signed URL createSignedUrl upload download private bucket\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "query": "query { searchDocs(query: \"Edge Functions environment variables secrets supabase secrets set deploy no verify jwt CORS\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" }, { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, { - "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", - "title": "Custom Roles" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" + "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", + "title": "Inspecting edge function environment variables" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" } ], - "resultChars": 22016 + "resultChars": 61098 } ] }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-storage-001-private-bucket-access.json" + "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", + "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/deploy-functions-001-edge-function-secrets.json" }, { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "build-tests-001-rls-tenant-isolation", - "stage": "build", + "eval": "deploy-self-hosting-001-docker-compose", + "stage": "deploy", "product": [ - "database" + "database", + "auth", + "storage" ], "topic": [ - "tests", - "rls" + "self-hosting" ], "suite": "benchmark", "interface": "cli", "passed": true, "checks": [ { - "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" + "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", + "passed": true }, { - "name": "pgTAP isolation tests ran and pass", - "passed": true, - "notes": "5 passed, 2 failed" + "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", + "passed": true }, { - "name": "agent correctly identifies the posts isolation bug from test results", - "passed": true, - "judgeNotes": "Correctly identifies `posts` as leaking cross-tenant rows for authenticated users, distinguishes `notes` as correctly isolated, and grounds the conclusion in failing pgTAP results." + "name": "secrets rotated off the shipped defaults", + "passed": true + }, + { + "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", + "passed": true } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"database testing pgTAP row level security auth.uid tests\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"self-hosting Docker compose production secrets JWT_SECRET ANON_KEY SERVICE_ROLE_KEY dashboard username password\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/testing/overview", - "title": "Testing Overview" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pgtap", - "title": "pgTAP: Unit Testing" + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", + "title": "Configure Social Login (OAuth) Providers" }, { - "url": "https://supabase.com/docs/guides/database/testing", - "title": "Testing Your Database" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" } ], - "resultChars": 70562 + "resultChars": 200148 } ] }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-tests-001-rls-tenant-isolation.json" - }, + "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", + "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/deploy-self-hosting-001-docker-compose.json" + }, { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "build-vectors-001-rag-with-permissions", - "stage": "build", + "eval": "investigate-auth-001-deleted-user-access", + "stage": "investigate", "product": [ - "database", - "vectors" + "auth" ], "topic": [ - "sql", - "rls" + "security", + "sdk" ], "suite": "benchmark", "interface": "mcp", "passed": true, "checks": [ { - "name": "document_sections.embedding is vector(384)", - "passed": true, - "notes": "vector(384)" + "name": "victim session active before delete-account", + "passed": true }, { - "name": "HNSW index on the embedding column", - "passed": true, - "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" + "name": "delete_account flow ran for the victim", + "passed": true }, { - "name": "index operator class matches the search operator", + "name": "delete-account revokes the user's sessions", "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" + "notes": "sessions left: 0" }, { - "name": "user A search returns only own sections, best match first", + "name": "deleted user's refresh token is rejected", "passed": true }, { - "name": "user B search returns only own sections, best match first", + "name": "deleted user cannot sign back in", "passed": true }, { - "name": "user A reads only own sections through the API", + "name": "other users keep their sessions and access", "passed": true }, { - "name": "user A reads only own documents through the API", - "passed": true + "name": "diagnosed and explained session revocation", + "passed": true, + "judgeNotes": "The answer identifies the soft-delete-only defect, implements hard deletion of auth.users with cascading session/refresh-token removal plus RLS active-session checks, explains the JWT caveat consistently with that fix (data path closed by RLS, token still locally valid until exp), and correctly distinguishes publishable vs secret keys, including that secret bypasses RLS and is server-only." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- relevant best-practice files ---'; rg --files .agents/skills/supabase-postgres-best-practices/references | rg 'security|rls|function|index'; printf '%s\\\\n' '--- changelog auth/key mentions ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|publishable|secret key|sign.?out|delete.*user|session' | head -n 160\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 8421 + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase AI gte-small embedding dimensions pgvector semantic search match_documents RLS security invoker\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"delete user access token remains valid session revoke sign out JWT session_id RLS\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/ai", - "title": "AI & Vectors" + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" }, { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" + "url": "https://supabase.com/docs/reference/swift/auth-admin-signout", + "title": "signOut()" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pgvector", - "title": "pgvector: Embeddings and vector similarity" + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" }, { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" + "url": "https://supabase.com/docs/reference/javascript/auth-signout" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" + "url": "https://supabase.com/docs/guides/storage/security/ownership", + "title": "Ownership" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" + }, + { + "url": "https://supabase.com/docs/guides/auth/third-party/clerk", + "title": "Clerk" + }, + { + "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", + "title": "Multiple SSO Providers" } ], - "resultChars": 63278 + "resultChars": 64888 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"gte-small 384 dimensions Supabase.ai.Session\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "query": "query { searchDocs(query: \"publishable key secret key frontend RLS anon service_role apikey authorization\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", - "title": "Choosing your Compute Add-on" + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" }, { - "url": "https://supabase.com/docs/guides/functions/ai-models", - "title": "Running AI Models" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" }, { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" + }, + { + "url": "https://supabase.com/docs/guides/realtime/getting_started", + "title": "Getting Started with Realtime" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/storage/vector/working-with-indexes", - "title": "Working with Vector Indexes" + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" } ], - "resultChars": 76269 - } - ] - }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-vectors-001-rag-with-permissions.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "deploy-database-001-prometheus-metrics", - "stage": "deploy", - "product": [ - "database" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "passed": true, - "checks": [ - { - "name": "preserved existing app scrape job", - "passed": true - }, - { - "name": "configured the Supabase Metrics API scrape correctly", - "passed": true, - "judgeNotes": "Meets all requirements: app scrape preserved, Supabase HTTPS metrics endpoint configured with Basic Auth password_file, project target present, and Docker Compose wires the secret to /run/secrets." - }, - { - "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README includes creating a Supabase secret API key, writing it to the matching Docker secret file, recreating the Compose Prometheus service, and verifying via Prometheus targets or an up{job=\"supabase\"} query." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ + "resultChars": 102497 + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Prometheus project metrics endpoint customer v1 privileged metrics service_role basic auth observability\", limit: 8) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"sb_publishable sb_secret secret key bypass RLS Authorization user JWT behavior\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + } + ], + "resultChars": 74569 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"delete user access token valid until expires sign out scope global admin deleteUser\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-signout" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports", - "title": "Reports" + "url": "https://supabase.com/docs/reference/swift/auth-admin-signout", + "title": "signOut()" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pgaudit", - "title": "PGAudit: Postgres Auditing" + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + }, + { + "url": "https://supabase.com/docs/guides/auth/signout", + "title": "Signing out" + }, + { + "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" + }, + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + } + ], + "resultChars": 13335 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Understanding API keys publishable key secret key format privileges role anon service_role\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" } ], - "resultChars": 82042 + "resultChars": 59611 } ] }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", - "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/deploy-database-001-prometheus-metrics.json" + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/investigate-auth-001-deleted-user-access.json" }, { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "deploy-functions-001-edge-function-secrets", - "stage": "deploy", + "eval": "investigate-realtime-001-subscribed-no-events", + "stage": "investigate", "product": [ - "edge-functions" + "realtime", + "database" ], "topic": [ - "security" + "sdk" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "WEATHER_API_KEY is set as a Function secret on the project", + "name": "orders table added to supabase_realtime publication", "passed": true }, { - "name": "the weather function is deployed to the project", - "passed": true, - "notes": "status ACTIVE" + "name": "courier_locations still in supabase_realtime publication", + "passed": true }, { - "name": "the weather function reads WEATHER_API_KEY from the environment", - "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "name": "publication still publishes INSERT events", + "passed": true }, { - "name": "WEATHER_API_KEY value is not committed to the repo", + "name": "RLS still enabled on orders", "passed": true + }, + { + "name": "staff can still read orders through RLS", + "passed": true, + "notes": "authenticated sees 2 of 2 orders" + }, + { + "name": "diagnosed missing publication membership", + "passed": true, + "judgeNotes": "The assistant correctly identified that the channel can reach SUBSCRIBED while INSERT events are silent because public.orders was missing from the supabase_realtime publication. It fixed exactly that with ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, preserved courier_locations in the publication, and did not disable RLS or weaken policies." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Edge Functions secrets environment variables Deno.env deploy functions invoke no verify jwt\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", + "query": "query { searchDocs(query: \"Postgres Changes realtime publication tables supabase_realtime INSERT events\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", + "title": "Subscribing to Database Changes" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes", + "title": "Postgres Changes" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/realtime/benchmarks", + "title": "Benchmarks" }, { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" + "url": "https://supabase.com/docs/guides/realtime/architecture", + "title": "Realtime Architecture" + }, + { + "url": "https://supabase.com/docs/guides/realtime/getting_started", + "title": "Getting Started with Realtime" } ], - "resultChars": 37288 + "resultChars": 91663 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- files ---'; rg --files -g '\"'!node_modules'\"' | sed -n '1,160p'; printf '%s\\\\n' '--- realtime references ---'; rg -n \\\"postgres_changes|courier|orders|supabase_realtime|publication\\\" -g '\"'!node_modules'\"' . | sed -n '1,240p'; printf '%s\\\\n' '--- changelog realtime mentions ---'; curl -Ls https://supabase.com/changelog.md | rg -n -i \\\"realtime|postgres changes|breaking-change\\\" | head -80\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 32513 } ] }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/deploy-functions-001-edge-function-secrets.json" + "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", + "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/investigate-realtime-001-subscribed-no-events.json" }, { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", + "eval": "investigate-reliability-003-edge-function-5xx-correlation", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "identified image-transform and the recurring 503 pattern", + "passed": true, + "judgeNotes": "The assistant identified `image-transform` as the affected function and described the recurring HTTP 503 pattern on 2026-04-28, including 8 gateway failures clustered throughout the morning/06:00–12:00 UTC with half-hour timing. This satisfies the rubric." + }, + { + "name": "attributed recurring 503s to gateway/platform layer, not function code", + "passed": true, + "judgeNotes": "The assistant attributes the 503s to the API gateway/platform layer, noting they appeared only in gateway logs and never reached the image-transform runtime, while successful requests used the same deployment and stayed fast. It also distinguishes the separate avatar-upload 500 as a function-level error." + }, + { + "name": "recommended a concrete next step", + "passed": true, + "judgeNotes": "The assistant recommended specific actionable next steps, including checking the scheduler/batch process, adding jitter/backoff, reviewing Edge Function concurrency metrics, capturing gateway request IDs, and opening a Supabase support case with specific gateway events." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", + "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/investigate-reliability-003-edge-function-5xx-correlation.json" + }, + { + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "resolve-dataapi-001-empty-results", + "stage": "resolve", "product": [ + "data-api", "database", - "auth", - "storage" + "auth" ], "topic": [ - "self-hosting" + "rls", + "sdk" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", + "name": "RLS still enabled on bookmarks", "passed": true }, { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", + "name": "user A reads own bookmarks", "passed": true }, { - "name": "secrets rotated off the shipped defaults", + "name": "user B cannot read user A bookmarks", "passed": true }, { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", + "name": "anon reads no bookmarks", + "passed": true + }, + { + "name": "user A can save a new bookmark", + "passed": true + }, + { + "name": "user B cannot insert a bookmark as user A", "passed": true + }, + { + "name": "diagnosed RLS and added owner-scoped policies", + "passed": true, + "judgeNotes": "Diagnosed RLS deny-all due to enabled RLS with no policies, kept RLS enabled, created authenticated SELECT and INSERT owner-scoped policies using auth.uid() with WITH CHECK for inserts, and verified isolation." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"self-hosting Docker compose install docker .env secrets JWT_SECRET ANON_KEY SERVICE_ROLE_KEY POSTGRES_PASSWORD SECRET_KEY_BASE VAULT_ENC_KEY\", limit: 8) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"Data API RLS authenticated grant select insert policy auth.uid() bookmarks\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#environment-variables-configuration", - "title": "Environment variables configuration" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#verifying-the-setup", - "title": "Verifying the setup" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#new-api-keys-format", - "title": "New API keys format" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#adding-the-new-keys", - "title": "Adding the new keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#additional-resources", - "title": "Additional resources" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#before-you-begin", - "title": "Before you begin" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#differences-from-the-supabase-platform", - "title": "Differences from the Supabase platform" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#backward-compatibility", - "title": "Backward compatibility" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#rotating-the-new-api-keys", - "title": "Rotating the new API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#regenerating-asymmetric-key-pair", - "title": "Regenerating asymmetric key pair" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#how-it-works", - "title": "How it works" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#what-client-sdk-sends", - "title": "What client SDK sends" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#kong-api-gateway-routing", - "title": "Kong API gateway routing" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#request-flows", - "title": "Request flows" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#unauthenticated-requests-api-key-only-no-user-session-jwt", - "title": "Unauthenticated requests (API key only, no user session JWT)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#authenticated-requests-user-session-jwt", - "title": "Authenticated requests (user session JWT)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#before-you-begin", - "title": "Before you begin" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#enabling-the-envoy-gateway", - "title": "Enabling the Envoy gateway" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#verify", - "title": "Verify" + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#architecture", - "title": "Architecture" + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#configuration-file-structure", - "title": "Configuration file structure" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#how-the-configuration-is-rendered-at-startup", - "title": "How the configuration is rendered at startup" + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#routes", - "title": "Routes" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#authentication", - "title": "Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#dashboard-basic-auth", - "title": "Dashboard basic auth" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#api-key-enforcement-on-protected-routes", - "title": "API key enforcement on protected routes" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#opaque-key-translation", - "title": "Opaque key translation" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#forwarded-headers-and-cors", - "title": "Forwarded headers and CORS" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#x-forwarded-headers", - "title": "X-Forwarded headers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#cors", - "title": "CORS" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#security-hardening", - "title": "Security hardening" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#customizing-the-configuration", - "title": "Customizing the configuration" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#admin-interface", - "title": "Admin interface" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#logs", - "title": "Logs" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#common-issues", - "title": "Common issues" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#see-also", - "title": "See also" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", - "title": "Configure Social Login (OAuth) Providers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#additional-resources", - "title": "Additional resources" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-5-verify-the-configuration", - "title": "Step 5: Verify the configuration" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-4-restart-the-auth-service", - "title": "Step 4: Restart the auth service" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#test-the-login-flow", - "title": "Test the login flow" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-3-enable-the-matching-lines-in-docker-compose-configuration", - "title": "Step 3: Enable the matching lines in Docker Compose configuration" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-2-configure-environment-variables", - "title": "Step 2: Configure environment variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-1-register-your-app-with-the-provider", - "title": "Step 1: Register your app with the provider" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-by-step-configuration", - "title": "Step-by-step configuration" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#auth-environment-variables", - "title": "Auth environment variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#oauth-request-flow", - "title": "OAuth request flow" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#before-you-begin", - "title": "Before you begin" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#site-url-or-redirect-url-errors-after-login", - "title": "Site URL or redirect URL errors after login" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#nonce-check-failure-on-mobile-google-sign-in", - "title": "Nonce check failure on mobile (Google Sign In)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#auth-service-fails-to-start", - "title": "Auth service fails to start" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#environment-variable-reference", - "title": "Environment variable reference" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#other-supported-providers", - "title": "Other supported providers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#variables-added-to-the-environment-but-provider-still-not-working", - "title": "Variables added to the environment but provider still not working" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#provider-specific-setup", - "title": "Provider-specific setup" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#generate-keys-and-secrets", - "title": "Generate keys and secrets" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-and-securing-supabase", - "title": "Configuring and securing Supabase" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#manual-installation", - "title": "Manual installation" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#quick-start-linux", - "title": "Quick start (Linux)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#installing-supabase", - "title": "Installing Supabase" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#system-requirements", - "title": "System requirements" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#before-you-begin", - "title": "Before you begin" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#contents", - "title": "Contents" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#exposing-your-postgres-database", - "title": "Exposing your Postgres database" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#setting-log_min_messages-in-postgres", - "title": "Setting log_min_messages in Postgres" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#using-file-backend-in-storage-on-macos", - "title": "Using file backend in Storage on macOS" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-supabase-ai-assistant", - "title": "Configuring Supabase AI Assistant" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-postgres-through-supavisor", - "title": "Accessing Postgres through Supavisor" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-s3-storage", - "title": "Configuring S3 Storage" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-an-email-server", - "title": "Configuring an email server" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-phone-login-sms-and-mfa", - "title": "Configuring phone login, SMS, and MFA" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-social-login-oauth-providers", - "title": "Configuring social login (OAuth) providers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-supabase-services", - "title": "Configuring Supabase services" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-secrets", - "title": "Configuring secrets" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#changing-database-password", - "title": "Changing database password" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#setting-database-password", - "title": "Setting database password" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#architecture", - "title": "Architecture" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#advanced-topics", - "title": "Advanced topics" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#uninstalling", - "title": "Uninstalling" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#updating", - "title": "Updating" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#managing-the-stack", - "title": "Managing the stack" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-https", - "title": "Configuring HTTPS" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#enabling-analytics", - "title": "Enabling analytics" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-apis", - "title": "Accessing APIs" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-edge-functions", - "title": "Accessing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-postgres", - "title": "Accessing Postgres" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-supabase-studio-dashboard", - "title": "Accessing Supabase Studio (Dashboard)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#demo", - "title": "Demo" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#managing-your-secrets", - "title": "Managing your secrets" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#starting-and-stopping", - "title": "Starting and stopping" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#studio-authentication", - "title": "Studio authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#where-to-find-your-credentials", - "title": "Where to find your credentials" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configure-supabase-urls", - "title": "Configure Supabase URLs" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#memory-or-timeout-errors", - "title": "Memory or timeout errors" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#custom-env-vars-not-available-in-functions", - "title": "Custom env vars not available in functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#changes-to-function-code-not-reflected-after-editing", - "title": "Changes to function code not reflected after editing" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#500-error-on-invocation", - "title": "500 error on invocation" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#copying-functions-from-supabase-platform", - "title": "Copying functions from Supabase platform" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#deploying-functions-to-a-remote-server", - "title": "Deploying functions to a remote server" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#managing-functions-via-dashboard", - "title": "Managing functions via dashboard" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#internal-vs-external-urls", - "title": "Internal vs external URLs" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#calling-supabase-services-from-functions", - "title": "Calling Supabase services from functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#accessing-variables-in-functions", - "title": "Accessing variables in functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#using-inline-environment-variables", - "title": "Using inline environment variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#using-an-env-file-recommended", - "title": "Using an env file (recommended)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#custom-environment-variables", - "title": "Custom environment variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-3-invoke-your-function", - "title": "Step 3: Invoke your function" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-2-restart-the-functions-service-to-pick-up-the-new-function", - "title": "Step 2: Restart the functions service to pick up the new function" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-1-add-a-new-function-directory-and-the-function-code", - "title": "Step 1: Add a new function directory and the function code" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#create-a-new-function", - "title": "Create a new function" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#invoke-the-default-function", - "title": "Invoke the default function" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates", - "title": "Custom Email Templates" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#step-1-create-a-templates-directory", - "title": "Step 1: Create a templates directory" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#overview", - "title": "Overview" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#authentication-email-templates", - "title": "Authentication email templates" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#example", - "title": "Example" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#step-2-update-docker-composeyml", - "title": "Step 2: Update docker-compose.yml" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#what-this-configuration-does", - "title": "What this configuration does" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#step-3-restart-containers", - "title": "Step 3: Restart containers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#notification-email-templates", - "title": "Notification email templates" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#example-1", - "title": "Example" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#step-1-create-the-templates-directory", - "title": "Step 1: Create the templates directory" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#step-2-update-docker-composeyml-1", - "title": "Step 2: Update docker-compose.yml" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#step-3-restart-containers-1", - "title": "Step 3: Restart containers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#signature-mismatch-errors", - "title": "Signature mismatch errors" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#test-with-the-aws-cli", - "title": "Test with the AWS CLI" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#test-with-rclone", - "title": "Test with rclone" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#how-to-configure-an-s3-backend", - "title": "How to configure an S3 backend" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#using-rustfs", - "title": "Using RustFS" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#using-minio", - "title": "Using MinIO" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#using-aws-s3", - "title": "Using AWS S3" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#s3-compatible-providers", - "title": "S3-compatible providers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#verify", - "title": "Verify" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#session-token", - "title": "Session token" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#tus-upload-errors-on-cloudflare-r2", - "title": "TUS upload errors on Cloudflare R2" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#permission-denied-on-uploads", - "title": "Permission denied on uploads" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#upload-urls-point-to-localhost", - "title": "Upload URLs point to localhost" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#additional-resources", - "title": "Additional resources" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#enable-the-s3-protocol-endpoint", - "title": "Enable the S3 protocol endpoint" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17", - "title": "Upgrade to Postgres 17" - }, + "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", + "title": "Storage Helper Functions" + } + ], + "resultChars": 52907 + } + ] + }, + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/resolve-dataapi-001-empty-results.json" + }, + { + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "resolve-database-001-migration-history-mismatch", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "the avatar_url column is applied on the hosted profiles table", + "passed": true + }, + { + "name": "migration 20240220000000 is recorded in the remote history", + "passed": true + }, + { + "name": "remote migration history matches local migration files", + "passed": true + }, + { + "name": "local migrations are a valid reconciled sequence", + "passed": true + }, + { + "name": "production profile data is intact (not reset)", + "passed": true + }, + { + "name": "the avatar migration and history reconciliation were done via the Supabase CLI", + "passed": true, + "judgeNotes": "Avatar migration was applied through `supabase db push --db-url \"$DB_URL\" --yes` in action #13, with subsequent remote inspection showing `avatar_url` and migration history containing `20240220000000 add_avatar_url`. History was reconciled by adding the missing local file `supabase/migrations/20240115000000_add_profile_bio.sql` in action #11, after which CLI dry-run no longer reported the orphan remote migration. No disallowed direct SQL mutation or prepared-statement workaround was seen." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI deploy database migrations db push migration repair linked project\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#new-deployment-with-postgres-17", - "title": "New deployment with Postgres 17" + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#before-you-begin", - "title": "Before you begin" + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#run-the-upgrade", - "title": "Run the upgrade" + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#restoring-from-a-manual-backup", - "title": "Restoring from a manual backup" + "url": "https://supabase.com/docs/reference/cli/supabase-db-push", + "title": "Push new migrations to the remote database" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#postgres-17-fails-to-start-with-a-leftover-db-config-volume", - "title": "Postgres 17 fails to start with a leftover db-config volume" - }, + "url": "https://supabase.com/docs/guides/deployment/managing-environments", + "title": "Managing Environments" + } + ], + "resultChars": 70851 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"set -o pipefail\nprintf '%s\\\\n' '--- CHANGELOG BREAKING CHANGES ---'\ncurl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|migration|db push|postgres' | head -80 || true\nprintf '%s\\\\n' '--- PROJECT FILES ---'\nfind supabase -maxdepth 4 -type f -print | sort\nprintf '%s\\\\n' '--- GIT STATUS ---'\ngit status --short --branch 2>&1 || true\nprintf '%s\\\\n' '--- EVAL PROFILE ---'\ncat .supabase-eval-profile.yaml\nprintf '%s\\\\n' '--- CONFIG ---'\nsed -n '1,260p' supabase/config.toml 2>/dev/null || true\nprintf '%s\\\\n' '--- MIGRATIONS ---'\nfor f in supabase/migrations/*.sql; do echo \\\"### \"'$f\"; sed -n '\"'1,260p' \\\"\"'$f\"; done\nprintf '\"'%s\\\\n' '--- CLI ---'\nsupabase --version\nprintf '%s\\\\n' '--- ENV KEYS ---'\nenv | cut -d= -f1 | rg 'SUPABASE|POSTGRES|DATABASE|DB_' | sort || true\nprintf '%s\\\\n' '--- LINK STATE ---'\nfind supabase/.temp -maxdepth 2 -type f -print -exec sh -c 'echo --- \"'$1; cat \"$1\"'\"' _ {} \\\\; 2>/dev/null || true\"", + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#disk-space-issues-during-upgrade", - "title": "Disk space issues during upgrade" - }, + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 907 + } + ] + }, + "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", + "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/resolve-database-001-migration-history-mismatch.json" + }, + { + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "resolve-performance-001-slow-query-cpu-spike", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "observability", + "sql" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "inspected pg_stat_statements for query performance", + "passed": true + }, + { + "name": "ran EXPLAIN on the expensive query", + "passed": true + }, + { + "name": "created index covering user_id and created_at", + "passed": true + }, + { + "name": "query plan uses an index and avoids sequential scan", + "passed": true, + "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + }, + { + "name": "inserts still work", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#services-fail-to-connect-after-upgrade", - "title": "Services fail to connect after upgrade" - }, + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "web_search", + "query": "site:supabase.com/changelog.md Supabase changelog breaking change database indexes pg_stat_statements", + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"query performance pg_stat_statements EXPLAIN indexes order by limit recent rows\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#pgsodium--supabase-vault-errors", - "title": "pgsodium / Supabase Vault errors" + "url": "https://supabase.com/docs/guides/database/inspect", + "title": "Database debugging and monitoring" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#pg_upgrade-fails-with-replication-slot-errors", - "title": "pg_upgrade fails with replication slot errors" + "url": "https://supabase.com/docs/guides/database/orioledb", + "title": "OrioleDB Overview" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#troubleshooting", - "title": "Troubleshooting" + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#upgrade-process-details", - "title": "Upgrade process details" - }, + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + } + ], + "resultChars": 31393 + } + ] + }, + "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", + "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/resolve-performance-001-slow-query-cpu-spike.json" + }, + { + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "resolve-security-002-rls-cross-tenant-leak", + "stage": "resolve", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS enabled on notes", + "passed": true + }, + { + "name": "tenant A sees only org A notes", + "passed": true + }, + { + "name": "tenant B cannot read org A notes", + "passed": true + }, + { + "name": "tenant A author can update own note", + "passed": true + }, + { + "name": "tenant B cannot update org A note", + "passed": true + }, + { + "name": "tenant B author can delete own note", + "passed": true + }, + { + "name": "tenant B cannot delete org A note", + "passed": true + }, + { + "name": "tenant A can insert note in own org", + "passed": true + }, + { + "name": "tenant B cannot insert into org A", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#custom-postgres-configuration", - "title": "Custom Postgres configuration" - }, + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "web_search", + "query": "site:supabase.com/changelog.md Supabase changelog breaking change RLS", + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"row level security multi tenant workspace organization membership auth.uid policy exists performance\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#rollback", - "title": "Rollback" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#after-the-upgrade", - "title": "After the upgrade" + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#extensions-removed-in-postgres-17", - "title": "Extensions removed in Postgres 17" + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#requirements", - "title": "Requirements" + "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", + "title": "Column Level Security" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#create-a-backup", - "title": "Create a backup" - }, + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0003_auth_rls_initplan", + "title": "Database Advisor: Lint 0003_auth_rls_initplan" + } + ], + "resultChars": 44967 + } + ] + }, + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/resolve-security-002-rls-cross-tenant-leak.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user 4fe0f864-0fca-4065-bf08-25ee74682908, signUp returned {\"userId\":\"4fe0f864-0fca-4065-bf08-25ee74682908\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"4fe0f864-0fca-4065-bf08-25ee74682908\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"JavaScript signUp signInWithPassword auth user metadata profiles select maybeSingle\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#what-the-upgrade-does", - "title": "What the upgrade does" + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#upgrade-an-existing-postgres-15-deployment", - "title": "Upgrade an existing Postgres 15 deployment" + "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", + "title": "signInWithPassword()" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#provider-not-enabled-or-provider-seen-as-false-in-settings" + "url": "https://supabase.com/docs/reference/javascript/using-modifiers-maybesingle" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#401-invalid-jwt" + "url": "https://supabase.com/docs/reference/dart/using-modifiers-maybesingle" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#400-missing-function-name-in-request" + "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#permission-denied-on-the-data-directory" + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithsso" } ], - "resultChars": 450330 + "resultChars": 5222 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Docker self-hosting quick start Linux git clone supabase docker copy .env.example generate-keys.sh\", limit: 5) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", + "query": "query { searchDocs(query: \"JavaScript createClient auth signUp user metadata data display_name getSession select single\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-supabase-ai-assistant", - "title": "Configuring Supabase AI Assistant" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#changing-database-password", - "title": "Changing database password" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-secrets", - "title": "Configuring secrets" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-supabase-services", - "title": "Configuring Supabase services" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-social-login-oauth-providers", - "title": "Configuring social login (OAuth) providers" + "url": "https://supabase.com/docs/reference/javascript/auth-signup" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-phone-login-sms-and-mfa", - "title": "Configuring phone login, SMS, and MFA" + "url": "https://supabase.com/docs/reference/dart/auth-signup" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-an-email-server", - "title": "Configuring an email server" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-s3-storage", - "title": "Configuring S3 Storage" + "url": "https://supabase.com/docs/reference/javascript/auth-admin-createuser" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#using-file-backend-in-storage-on-macos", - "title": "Using file backend in Storage on macOS" + "url": "https://supabase.com/docs/reference/javascript/auth-signinanonymously" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-postgres-through-supavisor", - "title": "Accessing Postgres through Supavisor" + "url": "https://supabase.com/docs/reference/javascript/auth-getuser" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#exposing-your-postgres-database", - "title": "Exposing your Postgres database" + "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", + "title": "Customizing Emails by Language" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#setting-log_min_messages-in-postgres", - "title": "Setting log_min_messages in Postgres" - }, + "url": "https://supabase.com/docs/reference/javascript/auth-getsession" + } + ], + "resultChars": 15042 + } + ] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6-no-skills/build-auth-001-email-password-flow.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "build-cli-001-bootstrap-app", + "stage": "build", + "product": [ + "database", + "data-api" + ], + "topic": [ + "migrations", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "supabase project initialised (supabase/config.toml exists)", + "passed": true + }, + { + "name": "todos table is created by a migration file", + "passed": true + }, + { + "name": "todos table exists with at least 2 seeded rows", + "passed": true, + "notes": "found 2 rows" + }, + { + "name": "row level security is enabled on todos", + "passed": true + }, + { + "name": "a SELECT policy targets the authenticated role", + "passed": true + }, + { + "name": "REST API returns no todos to anonymous requests", + "passed": true, + "notes": "0 rows" + }, + { + "name": "REST API returns the todos to authenticated requests", + "passed": true, + "notes": "2 rows" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Row Level Security authenticated users select policy anon no rows seed data migrations Supabase\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/docker#managing-your-secrets", - "title": "Managing your secrets" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#demo", - "title": "Demo" + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#contents", - "title": "Contents" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#before-you-begin", - "title": "Before you begin" + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#system-requirements", - "title": "System requirements" - }, + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + } + ], + "resultChars": 64781 + } + ] + }, + "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", + "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6-no-skills/build-cli-001-bootstrap-app.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "build-cli-002-declarative-schema", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "declarative-schema", + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": false, + "checks": [ + { + "name": "supabase db diff used to generate the migration", + "passed": false + }, + { + "name": "schema file updated to include description column", + "passed": true + }, + { + "name": "a new migration was generated for the change", + "passed": true + }, + { + "name": "description column exists in the live database", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI create local database migration alter table add column\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/docker#installing-supabase", - "title": "Installing Supabase" + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#quick-start-linux", - "title": "Quick start (Linux)" + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#manual-installation", - "title": "Manual installation" - }, + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + } + ], + "resultChars": 54403 + } + ] + }, + "prompt": "Add a description text column to the `products` table in my local Supabase stack", + "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6-no-skills/build-cli-002-declarative-schema.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "build-cli-003-pg-cron-queue-workflow", + "stage": "build", + "product": [ + "database", + "edge-functions", + "cron", + "queues" + ], + "topic": [ + "sql", + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", + "passed": true, + "notes": "schedule='* * * * *', active=true" + }, + { + "name": "cron command enqueues to the 'tasks' queue", + "passed": true, + "notes": "queue depth 0 -> 1" + }, + { + "name": "process-tasks function drains the queue", + "passed": true, + "notes": "function removed the seeded message (id 4) from the queue" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase Queues pgmq cron schedule enqueue messages Edge Function read delete archive local\", limit: 8) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-and-securing-supabase", - "title": "Configuring and securing Supabase" + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#generate-keys-and-secrets", - "title": "Generate keys and secrets" + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configure-supabase-urls", - "title": "Configure Supabase URLs" + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#where-to-find-your-credentials", - "title": "Where to find your credentials" + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#studio-authentication", - "title": "Studio authentication" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#starting-and-stopping", - "title": "Starting and stopping" + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-supabase-studio-dashboard", - "title": "Accessing Supabase Studio (Dashboard)" + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-postgres", - "title": "Accessing Postgres" - }, + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", + "title": "Testing and linting" + } + ], + "resultChars": 58800 + } + ] + }, + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6-no-skills/build-cli-003-pg-cron-queue-workflow.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"JavaScript select nested relationships foreign key joins service role secret key createClient\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-edge-functions", - "title": "Accessing Edge Functions" + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-apis", - "title": "Accessing APIs" + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-updateclient" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#enabling-analytics", - "title": "Enabling analytics" + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-deleteclient" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-https", - "title": "Configuring HTTPS" + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-getclient" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#managing-the-stack", - "title": "Managing the stack" - }, + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-regenerateclientsecret" + } + ], + "resultChars": 2052 + } + ] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6-no-skills/build-dataapi-001-relational-report.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase javascript client select foreign tables nested relationships service role local URL\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/docker#updating", - "title": "Updating" + "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", + "title": "Engineering for Scale" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#uninstalling", - "title": "Uninstalling" + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#advanced-topics", - "title": "Advanced topics" + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#architecture", - "title": "Architecture" + "url": "https://supabase.com/docs/guides/api/sql-to-api", + "title": "Converting SQL to JavaScript API" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#setting-database-password", - "title": "Setting database password" - }, + "url": "https://supabase.com/docs/guides/api/creating-routes", + "title": "Creating API Routes" + } + ], + "resultChars": 26664 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"secret key apikey header server-side sb_secret REST API Authorization\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#how-it-works", - "title": "How it works" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#before-you-begin", - "title": "Before you begin" + "url": "https://supabase.com/docs/guides/realtime/getting_started", + "title": "Getting Started with Realtime" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#adding-the-new-keys", - "title": "Adding the new keys" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#new-api-keys-format", - "title": "New API keys format" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + } + ], + "resultChars": 121204 + } + ] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6-no-skills/build-dataapi-002-restock-alert-report.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "build-database-001-migrate-postgres-to-supabase", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true + }, + { + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true + }, + { + "name": "foreign key constraints survived the restore", + "passed": true + }, + { + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"restore pg_dump custom format existing Postgres database into local Supabase CLI database roles extensions\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#verifying-the-setup", - "title": "Verifying the setup" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#environment-variables-configuration", - "title": "Environment variables configuration" + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#differences-from-the-supabase-platform", - "title": "Differences from the Supabase platform" + "url": "https://supabase.com/docs/guides/self-hosting/custom-postgres-extensions", + "title": "Custom Postgres Extensions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#backward-compatibility", - "title": "Backward compatibility" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", + "title": "Migrate from Vercel Postgres to Supabase" + } + ], + "resultChars": 53836 + } + ] + }, + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6-no-skills/build-database-001-migrate-postgres-to-supabase.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "build-functions-004-service-role-bypass", + "stage": "build", + "product": [ + "edge-functions", + "auth", + "database" + ], + "topic": [ + "rls", + "security", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "rejects missing auth", + "passed": true, + "notes": "status=401" + }, + { + "name": "user A reads own note", + "passed": true, + "notes": "status=200" + }, + { + "name": "reads only with the caller's JWT", + "passed": true, + "notes": "bearer_tokens=2, all_match=true" + }, + { + "name": "user A cannot force-read user B note", + "passed": true, + "notes": "status=200" + }, + { + "name": "user B cannot force-read user A note", + "passed": true, + "notes": "status=200" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions authenticate user Authorization header createClient SUPABASE_ANON_KEY RLS auth.getUser\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#rotating-the-new-api-keys", - "title": "Rotating the new API keys" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#regenerating-asymmetric-key-pair", - "title": "Regenerating asymmetric key pair" + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#what-client-sdk-sends", - "title": "What client SDK sends" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#kong-api-gateway-routing", - "title": "Kong API gateway routing" - }, + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + } + ], + "resultChars": 44695 + } + ] + }, + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6-no-skills/build-functions-004-service-role-bypass.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "build-functions-005-dual-auth-user-secret", + "stage": "build", + "product": [ + "edge-functions", + "auth", + "database" + ], + "topic": [ + "sdk", + "rls", + "security" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "seed rows present", + "passed": true, + "notes": "found 2/2 seeded rows" + }, + { + "name": "rejects request with no credentials", + "passed": true, + "notes": "status 401: {\"error\":\"Unauthorized\"}" + }, + { + "name": "user with JWT reads only their own rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"1bd2cc99-8c87-40df-82a9-3e9b69fb51ef\",\"metric\":\"steps_a_mst1rlj1\",\"value\":111}]" + }, + { + "name": "user cannot read another user's rows by passing user_id", + "passed": true, + "notes": "status 200: [{\"user_id\":\"1bd2cc99-8c87-40df-82a9-3e9b69fb51ef\",\"metric\":\"steps_a_mst1rlj1\",\"value\":111}]" + }, + { + "name": "service key bypasses RLS to read the target user's rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"6dde3a35-96a5-436a-b8f8-73c953b2fed1\",\"metric\":\"steps_b_mst1rlj1\",\"value\":222}]" + }, + { + "name": "non-service key is not granted service access", + "passed": true, + "notes": "status 401: {\"error\":\"Unauthorized\"}" + }, + { + "name": "rejects an unverified (forged) user token", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "a user token in the apikey slot is not treated as the service key", + "passed": true, + "notes": "status 401: {\"error\":\"Unauthorized\"}" + }, + { + "name": "implementation uses @supabase/server", + "passed": true, + "notes": "imports @supabase/server / withSupabase" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions authenticate JWT Authorization header service_role apikey header createClient auth getUser\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#request-flows", - "title": "Request flows" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#unauthenticated-requests-api-key-only-no-user-session-jwt", - "title": "Unauthenticated requests (API key only, no user session JWT)" + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#authenticated-requests-user-session-jwt", - "title": "Authenticated requests (user session JWT)" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#additional-resources", - "title": "Additional resources" + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + } + ], + "resultChars": 36992 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions config.toml verify_jwt false per function apikey service role environment SUPABASE_SERVICE_ROLE_KEY\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-4-add-seed-data", - "title": "Step 4: Add seed data" + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-5-verify", - "title": "Step 5: Verify" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-6-commit", - "title": "Step 6: Commit" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#the-daily-workflow", - "title": "The daily workflow" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#making-schema-changes", - "title": "Making schema changes" - }, + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + } + ], + "resultChars": 39226 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"@supabase/server withSupabase auth user secret ctx.authMode ctx.supabase ctx.supabaseAdmin Edge Function Deno.serve\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#generating-types", - "title": "Generating types" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#staying-in-sync-with-your-team", - "title": "Staying in sync with your team" + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", + "title": "Which package to use" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#pushing-to-a-remote-project", - "title": "Pushing to a remote project" - }, + "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", + "title": "Resumable WebSockets with Edge Functions" + } + ], + "resultChars": 25740 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"@supabase/server auth service_role legacy key auth mode verifyCredentials\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#resetting-a-remote-dev-or-staging-project", - "title": "Resetting a remote dev or staging project" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#key-commands-at-a-glance", - "title": "Key commands at a glance" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#cleaning-up-generated-migrations", - "title": "Cleaning up generated migrations" + "url": "https://supabase.com/docs/guides/auth/passkeys", + "title": "Passkey authentication" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#grants", - "title": "Grants" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#revokere-grant-patterns", - "title": "Revoke/re-grant patterns" - }, + "url": "https://supabase.com/docs/guides/auth/third-party/auth0", + "title": "Auth0" + } + ], + "resultChars": 54418 + } + ] + }, + "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", + "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6-no-skills/build-functions-005-dual-auth-user-secret.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "build-storage-001-private-bucket-access", + "stage": "build", + "product": [ + "storage", + "database" + ], + "topic": [ + "rls", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "bucket user-files exists", + "passed": true + }, + { + "name": "bucket user-files is private", + "passed": true + }, + { + "name": "RLS still enabled on storage.objects", + "passed": true + }, + { + "name": "user A lists only own files", + "passed": true, + "notes": "saw: 01a000ac-88e7-7066-979f-f9bd7fd9fec5/receipt-alpha.pdf, 01a000ac-88e7-7066-979f-f9bd7fd9fec5/receipt-beta.pdf" + }, + { + "name": "user B cannot read user A files", + "passed": true + }, + { + "name": "anon reads no files", + "passed": true + }, + { + "name": "user A can upload into own folder", + "passed": true + }, + { + "name": "user B cannot upload into user A folder", + "passed": true + }, + { + "name": "configured private per-user storage access", + "passed": true, + "judgeNotes": "Meets all requirements: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies on storage.objects with RLS left enabled, and supabase-js createSignedUrl with expiry for temporary sharing." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase Storage RLS policy foldername auth.uid private bucket signed URL createSignedUrl\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#extension-statements", - "title": "Extension statements" + "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", + "title": "Storage Helper Functions" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#known-limitations-of-db-diff", - "title": "Known limitations of db diff" + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#troubleshooting", - "title": "Troubleshooting" + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-3-create-your-schema", - "title": "Step 3: Create your schema" + "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", + "title": "Custom Roles" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-2-start-the-local-stack", - "title": "Step 2: Start the local stack" - }, + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + } + ], + "resultChars": 17747 + } + ] + }, + "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", + "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6-no-skills/build-storage-001-private-bucket-access.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "build-tests-001-rls-tenant-isolation", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "tests", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "pgTAP test file(s) written under supabase/tests/", + "passed": true, + "notes": "1 file(s): supabase/tests/tenant_isolation.test.sql" + }, + { + "name": "pgTAP isolation tests ran and pass", + "passed": true, + "notes": "4 passed, 2 failed" + }, + { + "name": "agent correctly identifies the posts isolation bug from test results", + "passed": true, + "judgeNotes": "The agent correctly identifies `posts` as having the tenant isolation flaw, states that cross-tenant post reads returned 1 instead of expected 0 in pgTAP, and distinguishes that `notes` is correctly isolated." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI database testing pgTAP RLS auth.uid tenant isolation tests\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-1-initialize-1", - "title": "Step 1: Initialize" + "url": "https://supabase.com/docs/guides/local-development/testing/overview", + "title": "Testing Overview" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#start-a-new-project-from-scratch", - "title": "Start a new project from scratch" + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", + "title": "Testing and linting" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-7-commit", - "title": "Step 7: Commit" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-6-verify", - "title": "Step 6: Verify" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-5-create-seed-data", - "title": "Step 5: Create seed data" - }, + "url": "https://supabase.com/docs/guides/database/extensions/pgtap", + "title": "pgTAP: Unit Testing" + } + ], + "resultChars": 71814 + } + ] + }, + "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", + "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6-no-skills/build-tests-001-rls-tenant-isolation.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "build-vectors-001-rag-with-permissions", + "stage": "build", + "product": [ + "database", + "vectors" + ], + "topic": [ + "sql", + "rls" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "document_sections.embedding is vector(384)", + "passed": true, + "notes": "vector(384)" + }, + { + "name": "HNSW index on the embedding column", + "passed": true, + "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" + }, + { + "name": "index operator class matches the search operator", + "passed": true, + "notes": "function operators: <#>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" + }, + { + "name": "user A search returns only own sections, best match first", + "passed": true + }, + { + "name": "user B search returns only own sections, best match first", + "passed": true + }, + { + "name": "user A reads only own sections through the API", + "passed": true + }, + { + "name": "user A reads only own documents through the API", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"pgvector semantic search match_documents row level security security invoker rpc auth.uid\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-4-pull-the-remote-schema", - "title": "Step 4: Pull the remote schema" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-3-link-to-your-remote-project", - "title": "Step 3: Link to your remote project" + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-2-authenticate", - "title": "Step 2: Authenticate" + "url": "https://supabase.com/docs/guides/ai/hybrid-search", + "title": "Hybrid search" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#before-you-begin", - "title": "Before you begin" + "url": "https://supabase.com/docs/guides/database/full-text-search", + "title": "Full Text Search" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#the-supabase-directory", - "title": "The ./supabase directory" - }, + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + } + ], + "resultChars": 95111 + } + ] + }, + "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", + "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6-no-skills/build-vectors-001-rag-with-permissions.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "deploy-database-001-prometheus-metrics", + "stage": "deploy", + "product": [ + "database" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "preserved existing app scrape job", + "passed": true + }, + { + "name": "configured the Supabase Metrics API scrape correctly", + "passed": true, + "judgeNotes": "Configuration meets the rubric: app scrape is preserved, Supabase scrape uses HTTPS with the required metrics path and project target, HTTP Basic Auth with password_file, and docker-compose wires the password file via a Compose secret." + }, + { + "name": "documented live deployment and verification steps", + "passed": true, + "judgeNotes": "README includes Secret API key creation, matching Compose secret file placement, Compose restart/recreate steps, and concrete verification via Prometheus targets/PromQL. Endpoint and auth setup match the provided configuration." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase project metrics Prometheus endpoint service role key metrics\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#move-an-existing-project-to-local-development", - "title": "Move an existing project to local development" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", + "title": "Metrics API" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-1-initialize", - "title": "Step 1: Initialize" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", - "title": "Supabase CLI" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#how-to-opt-out", - "title": "How to opt out" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#telemetry", - "title": "Telemetry" - }, + "url": "https://supabase.com/docs/guides/platform/read-replicas", + "title": "Read Replicas" + } + ], + "resultChars": 29095 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Prometheus scrape_config Supabase Metrics API basic_auth password_file 60 seconds\", limit: 3) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#stopping-local-services", - "title": "Stopping local services" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#access-your-projects-services", - "title": "Access your project's services" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#running-supabase-locally", - "title": "Running Supabase locally" - }, + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" + } + ], + "resultChars": 15264 + } + ] + }, + "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", + "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6-no-skills/deploy-database-001-prometheus-metrics.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "deploy-functions-001-edge-function-secrets", + "stage": "deploy", + "product": [ + "edge-functions" + ], + "topic": [ + "security" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "WEATHER_API_KEY is set as a Function secret on the project", + "passed": true + }, + { + "name": "the weather function is deployed to the project", + "passed": true, + "notes": "status ACTIVE" + }, + { + "name": "the weather function reads WEATHER_API_KEY from the environment", + "passed": true, + "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + }, + { + "name": "WEATHER_API_KEY value is not committed to the repo", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions environment variables secrets deploy supabase functions secrets set env file Deno serve CORS\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#updating-the-supabase-cli", - "title": "Updating the Supabase CLI" + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#beta-channel", - "title": "Beta channel" + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#installing-the-supabase-cli", - "title": "Installing the Supabase CLI" + "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", + "title": "Inspecting edge function environment variables" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#learn-more", - "title": "Learn more" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting", - "title": "Self-Hosting" - }, + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + } + ], + "resultChars": 44193 + } + ] + }, + "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", + "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6-no-skills/deploy-functions-001-edge-function-secrets.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "deploy-self-hosting-001-docker-compose", + "stage": "deploy", + "product": [ + "database", + "auth", + "storage" + ], + "topic": [ + "self-hosting" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", + "passed": true + }, + { + "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", + "passed": true + }, + { + "name": "secrets rotated off the shipped defaults", + "passed": true + }, + { + "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"self-hosting Docker compose docker .env secrets JWT_SECRET ANON_KEY SERVICE_ROLE_KEY SUPABASE_PUBLIC_URL API_EXTERNAL_URL SITE_URL SMTP dashboard basic auth\", limit: 8) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting#community-driven-projects", - "title": "Community-driven projects" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/self-hosting#about-self-hosting", - "title": "About self-hosting" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, { - "url": "https://supabase.com/docs/guides/self-hosting#how-self-hosted-supabase-differs", - "title": "How self-hosted Supabase differs" + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/self-hosting#your-responsibilities-when-self-hosting", - "title": "Your responsibilities when self-hosting" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" }, { - "url": "https://supabase.com/docs/guides/self-hosting#telemetry", - "title": "Telemetry" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https", + "title": "Configure Reverse Proxy and HTTPS" }, { - "url": "https://supabase.com/docs/guides/self-hosting#support-and-community", - "title": "Support and community" + "url": "https://supabase.com/docs/guides/self-hosting", + "title": "Self-Hosting" }, { - "url": "https://supabase.com/docs/guides/self-hosting#enterprise-self-hosting", - "title": "Enterprise self-hosting" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" }, { - "url": "https://supabase.com/docs/guides/self-hosting#get-started", - "title": "Get started" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", + "title": "Configure Social Login (OAuth) Providers" } ], - "resultChars": 353761 + "resultChars": 146677 } ] }, "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.6-no-skills/deploy-self-hosting-001-docker-compose.json" }, { @@ -18273,7 +18371,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "Meets rubric: identifies soft-delete-only root cause, implements auth user deletion plus RLS live-account gate, correctly explains JWTs remain cryptographically valid while protected data path has no post-commit window due to RLS, and correctly distinguishes publishable frontend keys from secret backend-only RLS-bypassing keys." + "judgeNotes": "Meets all rubric requirements: diagnoses soft-delete-only flow, implements real auth user deletion/session revocation path with RLS active-user mitigation, consistently explains remaining JWT validity/local-validation caveat, and correctly distinguishes publishable vs secret keys and RLS behavior." } ], "skills": { @@ -18284,154 +18382,88 @@ "calls": [ { "source": "search_docs", - "query": "query { keys: searchDocs(query: \"publishable key secret key legacy anon service_role RLS frontend\", limit: 5) { nodes { ... on Guide { title href content } ... on ManagementApiReference { title href content } } } sessions: searchDocs(query: \"delete user access token remains valid until expiry session sign out JWT revoke\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } delete: searchDocs(query: \"delete user admin API JWT remains valid delete account\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", - "title": "Should I set a shorter Max-Age parameter on the cookies?" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso/testing-best-practices", - "title": "SSO Testing and Best Practices" - } - ], - "resultChars": 230051 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"API keys publishable secret key sb_publishable sb_secret RLS\", limit: 10) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"delete user access token remains valid until expiry sign out sessions JWT RLS\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + { + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", - "title": "Build a User Management App with SvelteKit" + "url": "https://supabase.com/docs/reference/swift/auth-admin-signout", + "title": "signOut()" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-solidjs", - "title": "Build a User Management App with SolidJS" + "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", + "title": "Should I set a shorter Max-Age parameter on the cookies?" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-vue-3", - "title": "Build a User Management App with Vue 3" + "url": "https://supabase.com/docs/reference/javascript/auth-signout" }, { "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", "title": "Token Security and Row Level Security" } ], - "resultChars": 225824 + "resultChars": 29024 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"sign out JWT remains valid until expires revoke refresh tokens access token\", limit: 8) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"publishable key secret key legacy anon service_role RLS frontend\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", - "title": "Should I set a shorter Max-Age parameter on the cookies?" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { "url": "https://supabase.com/docs/guides/auth/signing-keys", "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/auth/server-side/advanced-guide", - "title": "Advanced guide" + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" }, { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" + } + ], + "resultChars": 90535 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Understanding API keys publishable secret key sb_publishable sb_secret RLS bypass\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/how-do-i-make-the-cookies-httponly-vwweFx", - "title": "How do I make the cookies HttpOnly?" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" } ], - "resultChars": 103031 + "resultChars": 53732 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"admin deleteUser delete user existing JWT access token valid until expiry\", limit: 6) { nodes { ... on ClientLibraryFunctionReference { title href content language methodName } ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"delete user auth.users SQL database function deleteUser admin API access token\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/auth/managing-user-data", "title": "User Management" }, - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, { "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" }, @@ -18439,17 +18471,21 @@ "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" }, { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-admin-deleteuser", + "title": "deleteUser()" } ], - "resultChars": 33037 + "resultChars": 22702 } ] }, "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.6-no-skills/investigate-auth-001-deleted-user-access.json" }, { @@ -18498,7 +18534,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "The assistant correctly identified that orders was absent from the supabase_realtime publication despite the channel being subscribed, applied exactly ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, preserved courier_locations and RLS/policies, and did not blame or alter RLS/client/networking." + "judgeNotes": "The assistant correctly identified orders missing from supabase_realtime as the root cause, added only public.orders to the existing publication, verified courier_locations remained included, and did not alter RLS or policies." } ], "skills": { @@ -18509,7 +18545,7 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Postgres Changes add table supabase_realtime publication ALTER PUBLICATION\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"Postgres Changes add table supabase_realtime publication ALTER PUBLICATION\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -18523,23 +18559,15 @@ { "url": "https://supabase.com/docs/guides/database/replication/pipelines", "title": "Set up Pipelines" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", - "title": "Backup and Restore using the CLI" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks", - "title": "Benchmarks" } ], - "resultChars": 121815 + "resultChars": 74883 } ] }, "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.6-no-skills/investigate-realtime-001-subscribed-no-events.json" }, { @@ -18566,17 +18594,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform as affected and described eight intermittent HTTP 503s across 07:00–12:00 UTC on 2026-04-28, noting the recurring gateway pattern." + "judgeNotes": "Identified image-transform as affected and described the recurring pattern of eight HTTP 503 gateway failures across 07:00–12:00Z on 2026-04-28." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "Attributes the 503s to the gateway/platform layer before function execution, grounded in gateway logs without corresponding Edge Function logs and nearby successful invocations on the same deployment. It also distinguishes the separate avatar-upload 500 as a runtime-level issue." + "judgeNotes": "Attributes the recurring image-transform 503s to the gateway/platform/pre-handler layer rather than function code, grounded in the observation that failed gateway 503s had no corresponding Edge Function execution records while successful calls used the same deployment. It also distinguishes avatar-upload's isolated 500 from the gateway 503s. Although it suggests redeploying/rolling back to force fresh workers, the primary attribution is clearly platform/gateway rather than application code." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Recommended concrete next steps, including escalating to Supabase support with request IDs and timestamps, plus retry and alerting actions." + "judgeNotes": "The assistant provided concrete actionable next steps, including redeploy/rollback, retries, decoupling processing, and escalating to Supabase Support with project ref, deployment, UTC window, and gateway request IDs." } ], "skills": { @@ -18588,7 +18616,7 @@ }, "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.6-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" }, { @@ -18642,7 +18670,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed RLS deny-all due to no policies, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." + "judgeNotes": "Diagnosed RLS deny-all due to no policies, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() using USING and WITH CHECK. Did not disable RLS or use permissive/public policies." } ], "skills": { @@ -18654,7 +18682,7 @@ }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.6-no-skills/resolve-dataapi-001-empty-results.json" }, { @@ -18701,7 +18729,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "Avatar migration was applied by `supabase db push --db-url \"$DB_URL\" --yes` in action #11, which succeeded and left remote migration history containing `20240220000000 | add_avatar_url`. The orphan bio migration was reconciled by adding local file `supabase/migrations/20240115000000_add_profile_bio.sql` in action #10, then running the same Supabase CLI push. No forbidden direct SQL mutation or prepared-statement workaround was seen; psql usage was read-only inspection." + "judgeNotes": "Avatar migration was applied by `supabase db push --db-url \"$DB_URL\" --yes`, with output `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration was reconciled by adding the local file `supabase/migrations/20240115000000_add_profile_bio.sql`, after which `supabase db push` proceeded with only the avatar migration pending. Read-only `psql` inspection was used; no disallowed workaround or direct mutation was seen." } ], "skills": { @@ -18712,37 +18740,37 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI db push migration history mismatch repair deploy hosted project\", limit: 5) { nodes { __typename title href content } } }", + "query": "query { searchDocs(query: \"Supabase CLI db push migration history remote hosted project migration repair\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" + "url": "https://supabase.com/docs/reference/cli/supabase-db-push", + "title": "Push new migrations to the remote database" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" }, { "url": "https://supabase.com/docs/reference/cli/supabase-migration-repair", "title": "Repair the migration history table" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", - "title": "Backup and Restore using the CLI" + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" }, { "url": "https://supabase.com/docs/guides/local-development/database-migrations", "title": "Database migrations" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" } ], - "resultChars": 96189 + "resultChars": 61657 } ] }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.6-no-skills/resolve-database-001-migration-history-mismatch.json" }, { @@ -18794,11 +18822,32 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Postgres index ORDER BY LIMIT multicolumn index query performance\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/database/orioledb", + "title": "OrioleDB Overview" + }, + { + "url": "https://supabase.com/docs/guides/database/inspect", + "title": "Database debugging and monitoring" + } + ], + "resultChars": 26407 + } + ] }, "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.6-no-skills/resolve-performance-001-slow-query-cpu-spike.json" }, { @@ -18870,7 +18919,7 @@ }, "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.6-no-skills/resolve-security-002-rls-cross-tenant-leak.json" }, { @@ -18904,7 +18953,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user cbbc700e-775b-4489-94a7-6a2ce1ab7257, signUp returned {\"userId\":\"cbbc700e-775b-4489-94a7-6a2ce1ab7257\"}" + "notes": "db user 81126c67-9af1-49e1-b922-21b35b3a3665, signUp returned {\"userId\":\"81126c67-9af1-49e1-b922-21b35b3a3665\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -18919,7 +18968,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"cbbc700e-775b-4489-94a7-6a2ce1ab7257\"}" + "notes": "{\"userId\":\"81126c67-9af1-49e1-b922-21b35b3a3665\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -18961,36 +19010,41 @@ }, { "source": "search_docs", - "query": "{ searchDocs(query: \"signUp email password user metadata display_name options data\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"supabase-js signUp with email password user metadata data display name signInWithPassword getUser session\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" + "url": "https://supabase.com/docs/guides/auth/passkeys", + "title": "Passkey authentication" }, { - "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", - "title": "Passwordless email logins" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" + "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", + "title": "Customizing Emails by Language" + }, + { + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" } ], - "resultChars": 36311 + "resultChars": 82542 } ] }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/build-auth-001-email-password-flow.json" }, { @@ -19058,11 +19112,23 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 93795 + } + ] }, "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/build-cli-001-bootstrap-app.json" }, { @@ -19118,7 +19184,7 @@ }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/build-cli-002-declarative-schema.json" }, { @@ -19158,7 +19224,7 @@ { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 3) from the queue" + "notes": "function removed the seeded message (id 39) from the queue" } ], "skills": { @@ -19174,44 +19240,42 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "{ searchDocs(query: \"pgmq queue cron schedule edge function pop read delete messages\", limit: 6) { nodes { title href content } } }", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md 2>/dev/null | grep -i -B2 -A2 -E \"breaking\" | grep -i -E \"cron|pgmq|queue|edge\" | head -20; echo \"---done---\"", "pages": [ { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 1461 + }, + { + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/queues/quickstart.md", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" + "url": "https://supabase.com/docs/guides/queues/quickstart.md" } ], - "resultChars": 68777 + "resultChars": 8777 }, { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md | grep -i -B2 -A2 \"breaking\" | grep -i -B1 -A1 -E \"cron|pgmq|queue|edge function\" | head -40; echo \"---done---\"", + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/cron/quickstart.md", + "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/cron/quickstart.md" } ], - "resultChars": 1492 + "resultChars": 6155 } ] }, "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/build-cli-003-pg-cron-queue-workflow.json" }, { @@ -19396,11 +19460,50 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"migrate postgres database to supabase pg_dump pg_restore import dump file\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", + "title": "Migrate from Vercel Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/neon", + "title": "Migrate from Neon to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/heroku", + "title": "Migrate from Heroku to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" + } + ], + "resultChars": 42930 + }, + { + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md | grep -i -B2 -A2 \"breaking\" | head -60", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 4462 + } + ] }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/build-database-001-migrate-postgres-to-supabase.json" }, { @@ -19465,48 +19568,48 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge function verify JWT auth getUser getClaims authorization header\", limit: 5) { nodes { title href content } } }", + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 93795 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge function get authenticated user from Authorization header createClient SUPABASE_ANON_KEY auth.getUser\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/reference/javascript/auth-getuser" }, { "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-getclaims" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/reference/swift/auth-getclaims", - "title": "getClaims()" - } - ], - "resultChars": 52010 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/functions/auth.md", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, { - "url": "https://supabase.com/docs/guides/functions/auth.md" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" } ], - "resultChars": 7275 + "resultChars": 53866 } ] }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/build-functions-004-service-role-bypass.json" }, { @@ -19542,37 +19645,37 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"error\":\"Invalid credentials\"}" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"1f7a732a-f911-4415-80cd-89a9f4e1e75e\",\"metric\":\"steps_a_msj163uc\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"95537d3d-0f88-4b70-b074-0e4195a95fcd\",\"metric\":\"steps_a_mst21be9\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: [{\"user_id\":\"1f7a732a-f911-4415-80cd-89a9f4e1e75e\",\"metric\":\"steps_a_msj163uc\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"95537d3d-0f88-4b70-b074-0e4195a95fcd\",\"metric\":\"steps_a_mst21be9\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"9a247c74-daaa-4e1f-9bd6-4c24ce42abcd\",\"metric\":\"steps_b_msj163uc\",\"value\":222}]" + "notes": "status 200: [{\"user_id\":\"b13e982b-a3b1-42b2-a638-c5f9e3e27fe2\",\"metric\":\"steps_b_mst21be9\",\"value\":222}]" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"error\":\"Invalid credentials\"}" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"error\":\"Invalid credentials\"}" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"error\":\"Invalid credentials\"}" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "implementation uses @supabase/server", @@ -19603,76 +19706,49 @@ "resultChars": 93795 }, { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge function environment variables SUPABASE_URL SUPABASE_SERVICE_ROLE_KEY secret key publishable key\", limit: 5) { nodes { title href content } } }", + "source": "web_fetch", + "query": "https://supabase.com/changelog/45702-developer-update-may-2026", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" + "url": "https://supabase.com/changelog/45702-developer-update-may-2026" } ], - "resultChars": 94601 + "resultChars": 6161 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"@supabase/server SDK edge functions createServerClient\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"edge function environment variables SUPABASE_SECRET_KEY publishable key\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/auth/server-side/migrating-to-ssr-from-auth-helpers", - "title": "Migrating to the SSR package from Auth Helpers" + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" }, { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" - } - ], - "resultChars": 59816 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/functions/auth.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth.md" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" } ], - "resultChars": 7275 + "resultChars": 67435 } ] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/build-functions-005-dual-auth-user-secret.json" }, { @@ -19712,7 +19788,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019fdc8e-ae2d-7481-a05e-6e5700e0781e/receipt-alpha.pdf, 019fdc8e-ae2d-7481-a05e-6e5700e0781e/receipt-beta.pdf" + "notes": "saw: 01a000b3-a589-76d5-b1dd-fceb8c905dca/receipt-alpha.pdf, 01a000b3-a589-76d5-b1dd-fceb8c905dca/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -19733,7 +19809,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Meets all criteria: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies with WITH CHECK, no RLS disabling or permissive/public policies, and supabase-js createSignedUrl with expiry for temporary sharing." + "judgeNotes": "Meets all requirements: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies with WITH CHECK, RLS kept in place, and supabase-js createSignedUrl with expiry for temporary sharing." } ], "skills": { @@ -19742,43 +19818,73 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { "calls": [ + { + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 51147 + }, { "source": "search_docs", - "query": "{ searchDocs(query: \"storage createSignedUrl supabase-js expiresIn private bucket\", limit: 5) { nodes { title href content } } }", + "query": "{\n searchDocs(query: \"storage access control RLS policies private bucket authenticated user folder\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", - "title": "Storage Image Transformations" + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" }, { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations", - "title": "Manage Storage Image Transformations usage" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" }, { - "url": "https://supabase.com/docs/guides/storage/uploads/resumable-uploads", - "title": "Resumable Uploads" + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" }, + { + "url": "https://supabase.com/docs/guides/auth/third-party/clerk", + "title": "Clerk" + } + ], + "resultChars": 25926 + }, + { + "source": "search_docs", + "query": "{\n searchDocs(query: \"createSignedUrl temporary signed url private bucket expiresIn download\", limit: 3) {\n nodes {\n title\n href\n content\n }\n }\n}", + "hasContent": true, + "pages": [ { "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" + }, + { + "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" + }, + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" } ], - "resultChars": 44528 + "resultChars": 2471 } ] }, "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/build-storage-001-private-bucket-access.json" }, { @@ -19805,17 +19911,17 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" + "notes": "3 file(s): supabase/tests/rls_notes.sql, supabase/tests/rls_posts.sql, supabase/tests/rls_memberships.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "3 passed, 1 failed" + "notes": "1 passed, 1 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as the table with broken tenant isolation, explains that authenticated members can read posts from orgs they are not members of, and grounds this in the pgTAP result where the `posts` negative test fails. It also correctly states `notes` is isolated and treats test results as authoritative." + "judgeNotes": "The agent correctly identifies `posts` as having a broken tenant isolation SELECT policy, specifically that authenticated members of any org can read posts from other orgs because the policy does not correlate `m.org_id` with `posts.org_id`. It grounds this in the pgTAP failures for `rls_posts.sql` tests 3–4 and also states `notes` isolation is correct." } ], "skills": { @@ -19833,7 +19939,7 @@ }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/build-tests-001-rls-tenant-isolation.json" }, { @@ -19902,20 +20008,9 @@ }, "docs": { "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 93795 - }, { "source": "search_docs", - "query": "{ searchDocs(query: \"semantic search pgvector match_document_sections embedding function\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"embedding gte-small vector dimensions match_documents pgvector semantic search\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -19923,40 +20018,29 @@ "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" - }, - { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" + "url": "https://supabase.com/docs/guides/ai/vector-columns", + "title": "Vector columns" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" + "url": "https://supabase.com/docs/guides/ai/concepts", + "title": "Concepts" }, { "url": "https://supabase.com/docs/guides/database/extensions/pgvector", "title": "pgvector: Embeddings and vector similarity" - } - ], - "resultChars": 68270 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/ai/semantic-search.md", - "hasContent": true, - "pages": [ + }, { - "url": "https://supabase.com/docs/guides/ai/semantic-search.md" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" } ], - "resultChars": 12827 + "resultChars": 59892 } ] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/build-vectors-001-rag-with-permissions.json" }, { @@ -19985,12 +20069,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "Meets all requirements: HTTPS Supabase Metrics API scrape for the project target, correct metrics path, Basic Auth with password_file, app scrape preserved, and docker-compose mounts the secrets directory containing the password file." + "judgeNotes": "Supabase scrape uses HTTPS, correct metrics path, Basic Auth with password_file, project target on supabase.co, preserves app job, and docker-compose mounts the secrets directory containing the password file." }, { "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "README includes Secret API key creation, matching secret file placement, stack restart/reload, and concrete verification via Prometheus targets plus direct curl check. Endpoint/auth and mounted secret path are consistent, with no hardcoded secret." + "judgeNotes": "README includes Secret API key creation, matching secret file path, restart/reload steps, and concrete verification via Prometheus targets and PromQL/Grafana. Endpoint/auth and secret mounting are consistent with config, with no hardcoded secret." } ], "skills": { @@ -20004,22 +20088,15 @@ }, "docs": { "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 93795 - }, { "source": "search_docs", - "query": "{ searchDocs(query: \"metrics API prometheus endpoint scrape\") { nodes { title href content } } }", + "query": "{\n searchDocs(query: \"metrics endpoint prometheus scrape observability\", limit: 8) {\n nodes {\n title\n href\n content\n }\n }\n}", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", + "title": "Metrics API" + }, { "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", "title": "Vendor-agnostic Metrics API setup" @@ -20028,30 +20105,22 @@ "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - }, { "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", "title": "Metrics API with Grafana Cloud" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/how-to-view-database-metrics-uqf2z_", - "title": "How to View Database Metrics" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" + "url": "https://supabase.com/docs/reference/api/v1-scrape-project-metrics", + "title": "Scrape a project's metrics" } ], - "resultChars": 29060 + "resultChars": 19934 } ] }, "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/deploy-database-001-prometheus-metrics.json" }, { @@ -20103,11 +20172,23 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 93795 + } + ] }, "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/deploy-functions-001-edge-function-secrets.json" }, { @@ -20155,38 +20236,38 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { "calls": [ { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/self-hosting/docker.md", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -s --max-time 20 \"https://supabase.com/changelog/48048-self-hosted-supabase-envoy-becomes-the-default-api-gateway-b.md\" | head -80", "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/docker.md" + "url": "https://supabase.com/changelog/48048-self-hosted-supabase-envoy-becomes-the-default-api-gateway-b.md" } ], - "resultChars": 30709 + "resultChars": 5943 }, { "source": "web_fetch", - "query": "https://supabase.com/changelog.md", + "query": "https://supabase.com/docs/guides/self-hosting/docker", "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/self-hosting/docker" } ], - "resultChars": 93795 + "resultChars": 30885 } ] }, "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/deploy-self-hosting-001-docker-compose.json" }, { @@ -20238,7 +20319,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "The answer diagnoses the soft-delete-only bug, implements real auth user/session/refresh-token removal, explains the remaining stale JWT access-token window consistently with its fix (including mitigation), and correctly distinguishes publishable frontend keys from secret/server keys that bypass RLS." + "judgeNotes": "The assistant identifies the root cause as soft-deleting only public.profiles without deleting auth.users/sessions, implements a SECURITY DEFINER delete_account that deletes app data and auth.users, cascading to identities/sessions/refresh_tokens, and restricts execution to authenticated users. It explains that deleting auth.users revokes sessions/refresh tokens and prevents future sign-in. The provided transcript does not show the final explanatory text about stateless JWT access-token lifetime or publishable vs secret keys, but it does indicate the assistant was preparing to verify docs. Based on the included answer, the core fix is correct; however, if judging only visible final prose, the JWT/key clarification is not present." } ], "skills": { @@ -20255,37 +20336,37 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key anon service_role API keys RLS\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"publishable key secret key vs anon service_role RLS migration\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, { "url": "https://supabase.com/docs/guides/auth/signing-keys", "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" }, { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" } ], - "resultChars": 101422 + "resultChars": 84462 } ] }, "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/investigate-auth-001-deleted-user-access.json" }, { @@ -20333,7 +20414,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Identified orders missing from supabase_realtime publication as root cause, added public.orders to the existing publication, and preserved courier_locations, RLS, and policies. Did not blame client/RLS/networking as root cause or weaken security." + "judgeNotes": "The answer correctly identifies the root cause as orders missing from the supabase_realtime publication despite the channel reaching SUBSCRIBED, applies exactly ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, and preserves RLS/policies and the existing courier_locations feed." } ], "skills": { @@ -20346,11 +20427,23 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/realtime/postgres-changes.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/realtime/postgres-changes.md" + } + ], + "resultChars": 62299 + } + ] }, "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/investigate-realtime-001-subscribed-no-events.json" }, { @@ -20371,65 +20464,36 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform as the affected function and explicitly described the recurring 8 HTTP 503 gateway failures across the morning of 2026-04-28 from 07:00Z to 12:00Z." + "judgeNotes": "Identified image-transform as affected and described the recurring 503 pattern across the morning of 2026-04-28, covering all 8 gateway failures from 07:00Z to 12:00Z. Also correctly distinguished old billing-webhook 503s as unrelated." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": false, - "judgeNotes": "Although it cites a valid observation (503s only in gateway logs with no function execution logs), it ultimately attributes the primary cause to the function's unpinned npm dependency/worker boot and recommends pinning/redeploying the functions, rather than attributing the 503s to the gateway/platform layer in front of the function." + "passed": true, + "judgeNotes": "Attributes image-transform 503s to the gateway/platform layer, not function code, and grounds it in valid observations: gateway-only 503s with no edge-function runtime logs, nearby successful invocations on unchanged deployment/version, and distinction from a function-level logged 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended concrete next steps: pinning Edge Function dependencies, vendoring dependencies with lockfile checks, checking npm publish/registry history for the affected time window, and adding gateway 5xx alerting." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge functions 503 error npm dependencies pin version deno.json import map cold start\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-dependency-analysis", - "title": "Edge Function dependency analysis" - }, - { - "url": "https://supabase.com/docs/guides/security/npm-security", - "title": "Securing npm installs" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/importing-stripe-or-other-modules-from-esmsh-on-deno-edge-functions-throws-an-error-TmbB5p", - "title": "Importing Stripe or other modules from esm.sh on Deno Edge Functions throws an error" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", - "title": "Building an MCP Server with mcp-lite" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - } - ], - "resultChars": 43609 - } + "judgeNotes": "The assistant recommended multiple concrete next steps, including auditing scheduled callers, adding retry/backoff and jitter, checking edge-function capacity limits, adding alerts, and escalating to Supabase support with exact timestamps and region if needed." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" ] }, + "docs": { + "calls": [] + }, "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", "attempts": 2, @@ -20485,7 +20549,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies as the cause of empty Data API results, kept RLS enabled, and created authenticated SELECT and INSERT owner-scoped policies using auth.uid(), including WITH CHECK for inserts." + "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies causing deny-all Data API results, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid(), with INSERT enforced via WITH CHECK." } ], "skills": { @@ -20503,7 +20567,7 @@ }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/resolve-dataapi-001-empty-results.json" }, { @@ -20549,7 +20613,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "PASS: The avatar_url migration was applied through the Supabase CLI with `supabase db push` in action #26, which shows `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration history was reconciled by adding the missing local migration file `supabase/migrations/20240115000000_add_bio.sql` in actions #22-#24, after which `supabase migration list` (#25/#28) showed local and remote history aligned. Read-only psql inspections were used, but no prohibited direct SQL mutation or prepared-statement workaround was seen." + "judgeNotes": "Avatar migration was applied through `supabase db push` in #19, with output `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration was reconciled by adding the local file `20240115000000_add_profile_bio.sql` in #18 and then letting `supabase db push` proceed; #21 confirms local/remote history aligned. No disallowed workaround or direct mutation was used." } ], "skills": { @@ -20558,8 +20622,7 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { @@ -20567,7 +20630,7 @@ }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/resolve-database-001-migration-history-mismatch.json" }, { @@ -20628,7 +20691,7 @@ }, "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/resolve-performance-001-slow-query-cpu-spike.json" }, { @@ -20705,7 +20768,7 @@ }, "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/resolve-security-002-rls-cross-tenant-leak.json" }, { @@ -20739,7 +20802,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 21c579b2-ae12-44ca-83f7-24fe17e54af5, signUp returned {\"userId\":\"21c579b2-ae12-44ca-83f7-24fe17e54af5\"}" + "notes": "db user c51c7c73-5572-47e5-9cf6-6689087021cc, signUp returned {\"userId\":\"c51c7c73-5572-47e5-9cf6-6689087021cc\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -20754,7 +20817,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"21c579b2-ae12-44ca-83f7-24fe17e54af5\"}" + "notes": "{\"userId\":\"c51c7c73-5572-47e5-9cf6-6689087021cc\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -20781,7 +20844,7 @@ }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/build-auth-001-email-password-flow.json" }, { @@ -20817,7 +20880,7 @@ { "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "found 3 rows" + "notes": "found 2 rows" }, { "name": "row level security is enabled on todos", @@ -20835,7 +20898,7 @@ { "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "3 rows" + "notes": "2 rows" } ], "skills": { @@ -20847,7 +20910,7 @@ }, "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/build-cli-001-bootstrap-app.json" }, { @@ -20897,7 +20960,7 @@ }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/build-cli-002-declarative-schema.json" }, { @@ -20932,12 +20995,12 @@ { "name": "cron command enqueues to the 'tasks' queue", "passed": true, - "notes": "queue depth 0 -> 1" + "notes": "queue depth 1 -> 2" }, { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 5) from the queue" + "notes": "function removed the seeded message (id 36) from the queue" } ], "skills": { @@ -20948,7 +21011,7 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"pgmq queues consume messages with edge function read delete\", limit: 5) { nodes { title href content } } }", + "query": "{\n searchDocs(query: \"consume queue messages edge function pgmq_public read delete\", limit: 3) {\n nodes {\n title\n href\n content\n }\n }\n}", "hasContent": true, "pages": [ { @@ -20959,52 +21022,38 @@ "url": "https://supabase.com/docs/guides/queues/pgmq", "title": "PGMQ Extension" }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, { "url": "https://supabase.com/docs/guides/queues/api", "title": "API" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" } ], - "resultChars": 68440 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"expose queues PostgREST edge function pgmq_public read delete example\", limit: 3) { nodes { title href subsections { nodes { title content } } } } }", - "hasContent": true, - "pages": [] + "resultChars": 27216 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"expose queues PostgREST edge function pgmq_public read delete example\", limit: 3) { nodes { title href ... on Guide { subsections { nodes { title content } } } } } }", + "query": "{\n searchDocs(query: \"pgmq queue cron schedule send message every minute pg_cron\", limit: 3) {\n nodes {\n title\n href\n content\n }\n }\n}", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" }, { - "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", - "title": "Expose Queues for local and self-hosted Supabase" + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" } ], - "resultChars": 22663 + "resultChars": 35611 } ] }, "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/build-cli-003-pg-cron-queue-workflow.json" }, { @@ -21177,7 +21226,7 @@ }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/build-database-001-migrate-postgres-to-supabase.json" }, { @@ -21228,229 +21277,101 @@ "name": "user B cannot force-read user A note", "passed": true, "notes": "status=200" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/build-functions-004-service-role-bypass.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-functions-005-dual-auth-user-secret", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "sdk", - "rls", - "security" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"error\":\"Invalid credentials\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"8de5d728-6f94-47d9-833a-f4104a74539f\",\"metric\":\"steps_a_msj15kas\",\"value\":111}]" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: [{\"user_id\":\"8de5d728-6f94-47d9-833a-f4104a74539f\",\"metric\":\"steps_a_msj15kas\",\"value\":111}]" - }, - { - "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"b33c6f83-e1ca-49a7-8520-64ff05e415e9\",\"metric\":\"steps_b_msj15kas\",\"value\":222}]" - }, - { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"error\":\"Invalid credentials\"}" - }, - { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 401: {\"error\":\"Invalid credentials\"}" - }, - { - "name": "a user token in the apikey slot is not treated as the service key", - "passed": true, - "notes": "status 401: {\"error\":\"Invalid credentials\"}" - }, - { - "name": "implementation uses @supabase/server", - "passed": true, - "notes": "imports @supabase/server / withSupabase" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{searchDocs(query: \"edge function verify JWT get user from access token service role bypass RLS\") {nodes {title href content}}}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac", - "title": "Custom Claims & Role-based Access Control (RBAC)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-getclaims" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-getclaims", - "title": "getClaims()" - }, - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, - { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/getting-started", - "title": "Getting Started with OAuth 2.1 Server" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-select-returning-an-empty-data-array-and-i-have-data-in-the-table-xvOPgx", - "title": "Why is my select returning an empty data array and I have data in the table?" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/ownership", - "title": "Ownership" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-getuser", - "title": "user()" - }, - { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0012_auth_allow_anonymous_sign_ins", - "title": "Database Advisor: Lint 0012_auth_allow_anonymous_sign_ins" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" - } - ], - "resultChars": 271984 - } - ] + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", + "attempts": 2, + "sourcePath": "opencode-kimi-k3-no-skills/build-functions-004-service-role-bypass.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-functions-005-dual-auth-user-secret", + "stage": "build", + "product": [ + "edge-functions", + "auth", + "database" + ], + "topic": [ + "sdk", + "rls", + "security" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "seed rows present", + "passed": true, + "notes": "found 2/2 seeded rows" + }, + { + "name": "rejects request with no credentials", + "passed": true, + "notes": "status 401: {\"error\":\"unauthorized\"}" + }, + { + "name": "user with JWT reads only their own rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"38e0a1aa-f14e-4c97-97d4-1d2e10bf890f\",\"metric\":\"steps_a_mst20kk4\",\"value\":111}]" + }, + { + "name": "user cannot read another user's rows by passing user_id", + "passed": true, + "notes": "status 200: [{\"user_id\":\"38e0a1aa-f14e-4c97-97d4-1d2e10bf890f\",\"metric\":\"steps_a_mst20kk4\",\"value\":111}]" + }, + { + "name": "service key bypasses RLS to read the target user's rows", + "passed": false, + "notes": "status 401: {\"error\":\"unauthorized\"}" + }, + { + "name": "non-service key is not granted service access", + "passed": true, + "notes": "status 401: {\"error\":\"unauthorized\"}" + }, + { + "name": "rejects an unverified (forged) user token", + "passed": true, + "notes": "status 401: {\"error\":\"invalid or expired access token\"}" + }, + { + "name": "a user token in the apikey slot is not treated as the service key", + "passed": true, + "notes": "status 401: {\"error\":\"unauthorized\"}" + }, + { + "name": "implementation uses @supabase/server", + "passed": false, + "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/build-functions-005-dual-auth-user-secret.json" }, { @@ -21490,7 +21411,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019fdc8d-4da9-7774-82e4-97e7592afbcb/receipt-alpha.pdf, 019fdc8d-4da9-7774-82e4-97e7592afbcb/receipt-beta.pdf" + "notes": "saw: 01a000ad-c6e6-7739-aca3-3fb31cc88f8a/receipt-alpha.pdf, 01a000ad-c6e6-7739-aca3-3fb31cc88f8a/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -21511,7 +21432,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Meets the rubric: creates a private user-files bucket, adds authenticated owner-scoped SELECT and INSERT policies on storage.objects using the user-id path segment, does not disable RLS or use public access, and provides supabase-js createSignedUrl code with an expiry." + "judgeNotes": "Configures a private user-files bucket, authenticated owner-scoped SELECT and INSERT policies on storage.objects, does not disable RLS, and provides supabase-js createSignedUrl code with an expiry for temporary sharing." } ], "skills": { @@ -21523,7 +21444,7 @@ }, "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/build-storage-001-private-bucket-access.json" }, { @@ -21550,17 +21471,17 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation.sql" + "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "10 passed, 2 failed" + "notes": "7 passed, 1 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as the broken tenant isolation policy, explains that authenticated members can read posts from organizations they do not belong to, and grounds the conclusion in pgTAP failures. It also correctly notes `notes` is isolated and treats the test results as authoritative." + "judgeNotes": "The agent correctly identifies `posts` as the broken tenant isolation policy, explains that authenticated members can read posts from orgs they do not belong to, and grounds the conclusion in pgTAP results showing the `posts` negative test failed while `notes` passed." } ], "skills": { @@ -21572,7 +21493,7 @@ }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/build-tests-001-rls-tenant-isolation.json" }, { @@ -21634,11 +21555,32 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"gte-small embedding dimensions vector 384 semantic search match_documents\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", + "title": "Choosing your Compute Add-on" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" + }, + { + "url": "https://supabase.com/docs/guides/ai/concepts", + "title": "Concepts" + } + ], + "resultChars": 37288 + } + ] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/build-vectors-001-rag-with-permissions.json" }, { @@ -21658,7 +21600,7 @@ "observability" ], "suite": "benchmark", - "passed": true, + "passed": false, "checks": [ { "name": "preserved existing app scrape job", @@ -21667,12 +21609,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "Meets all requirements: app scrape preserved, Supabase HTTPS metrics endpoint configured with Basic Auth password_file, project target present, and docker-compose mounts the secrets directory containing the password file." + "judgeNotes": "Prometheus preserves the app scrape and adds a Supabase HTTPS scrape to /customer/v1/privileged/metrics with Basic Auth using password_file. docker-compose mounts the secrets directory containing that password file path." }, { "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README includes correct live setup steps: create/copy a Supabase Secret API key, write it to the mounted secret file matching prometheus.yml, restart/recreate or reload the Compose stack, and verify via Prometheus targets and PromQL/Grafana. Endpoint/auth and secret setup are consistent, with no hardcoded real secret." + "passed": false, + "judgeNotes": "README.md does not explain how to make the Supabase integration live. It lacks steps to create a Secret API key, place it in observability/secrets/supabase_metrics_api_key, restart/reload the Compose stack, and verify via Prometheus targets/PromQL/Grafana. It only documents starting the stack and lists app:8080." } ], "skills": { @@ -21683,47 +21625,138 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"metrics endpoint prometheus scrape hosted project observability\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"metrics endpoint prometheus scrape observability\") { nodes { title href content } } }", + "hasContent": true, + "pages": [], + "resultChars": 344 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"management api create api key secret sb_secret\") { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", + "title": "Build a User Management App with Angular" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-react", + "title": "Build a User Management App with Ionic React" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-angular", + "title": "Build a User Management App with Ionic Angular" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-vue-3", + "title": "Build a User Management App with Vue 3" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-vue", + "title": "Build a User Management App with Ionic Vue" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-svelte", + "title": "Build a User Management App with Svelte" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-swift", + "title": "Build a User Management App with Swift and SwiftUI" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-solidjs", + "title": "Build a User Management App with SolidJS" + }, + { + "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", + "title": "Build a Social Auth App with Expo React Native" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-react", + "title": "Build a User Management App with React" }, { "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", "title": "Vendor-agnostic Metrics API setup" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", + "title": "Build a User Management App with SvelteKit" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - } - ], - "resultChars": 19334 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"management api create project api key secret sb_secret\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-expo-react-native", + "title": "Build a User Management App with Expo React Native" + }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nuxt-3", + "title": "Build a User Management App with Nuxt 3" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-refine", + "title": "Build a User Management App with Refine" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter", + "title": "Build a User Management App with Flutter" + }, + { + "url": "https://supabase.com/docs/reference/api/v1-bulk-create-secrets", + "title": "Bulk create secrets" }, { "url": "https://supabase.com/docs/reference/api/v1-create-project-api-key", "title": "Creates a new API key for the project" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", + "title": "Supabase for Platforms" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/getting-started", + "title": "Getting Started with OAuth 2.1 Server" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-kakao", + "title": "Login with Kakao" } ], - "resultChars": 63685 + "resultChars": 767213 } ] }, @@ -21780,7 +21813,7 @@ }, "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/deploy-functions-001-edge-function-secrets.json" }, { @@ -21827,11 +21860,40 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"self-hosting supabase with docker compose setup env secrets\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", + "title": "Configure Social Login (OAuth) Providers" + } + ], + "resultChars": 88525 + } + ] }, "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/deploy-self-hosting-001-docker-compose.json" }, { @@ -21883,7 +21945,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "The answer identifies the soft-delete-only root cause, implements real auth/session/refresh-token revocation by deleting sessions and auth user, hardens RLS to close the data API stale-JWT window, and explains the remaining stateless JWT validity caveat consistently. It also correctly distinguishes publishable frontend keys with RLS from secret server-only keys that bypass RLS." + "judgeNotes": "The answer identifies the root cause: delete_account only soft-deleted the profile and did not remove the auth user/sessions. It implements a real fix by deleting auth.users via a SECURITY DEFINER function, relying on cascades to remove identities, sessions, refresh tokens, and profile. It also hardens RLS so stale JWTs cannot access notes/profile after deletion, while correctly explaining already-issued JWTs remain cryptographically valid until exp for local validation. It correctly distinguishes publishable frontend key with user JWT/RLS from secret server-only key that bypasses RLS and must not be shipped to clients." } ], "skills": { @@ -21895,7 +21957,7 @@ }, "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/investigate-auth-001-deleted-user-access.json" }, { @@ -21943,7 +22005,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Identifies orders missing from supabase_realtime publication as root cause despite SUBSCRIBED status, applies ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, verifies existing courier_locations remains, and does not weaken RLS/policies or blame client/networking." + "judgeNotes": "Identified the missing orders table in supabase_realtime as the root cause, explained SUBSCRIBED vs no INSERT events, added only public.orders to the existing publication, and did not weaken RLS/policies or disturb courier_locations." } ], "skills": { @@ -21955,7 +22017,7 @@ }, "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/investigate-realtime-001-subscribed-no-events.json" }, { @@ -21981,17 +22043,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "The assistant clearly identified image-transform as the affected function and described the recurring pattern of 8 HTTP 503 gateway responses on 2026-04-28 between 07:00Z and 12:00Z, while correctly distinguishing unrelated old billing-webhook 503s." + "judgeNotes": "The assistant clearly identified `image-transform` as the affected function and described the recurring HTTP 503 gateway failures throughout the morning of 2026-04-28, covering all 8 failures from 07:00Z to 12:00Z. It also correctly distinguished the older billing-webhook 503s as a red herring." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "Attributes the 503s to the gateway/edge platform layer, not function code, and grounds this in valid evidence: gateway-only 503s with no failed function invocations, successful nearby executions on same deployment/version, and distinction from a function-level avatar-upload 500." + "judgeNotes": "Attributes the recurring image-transform 503s to the API gateway/platform layer, explicitly says they never reached the function and all actual runtime invocations succeeded on the same deployment. Also distinguishes them from avatar-upload's real function-level 500 and recommends not redeploying/changing function code." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended concrete next steps, including opening a Supabase support ticket with project ref, request IDs, and time window, checking platform incident/status, adding retries, and setting up alerting." + "judgeNotes": "The assistant recommended concrete next steps, including checking Supabase status/edge-runtime health for the specific time window, opening a support ticket with request IDs and timestamps if recurring, adding retries/queueing, alerting on gateway 5xxs, and spot-checking a specific application error." } ], "skills": { @@ -22056,7 +22118,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies causing deny-all Data API results, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to auth.uid() = user_id using USING and WITH CHECK." + "judgeNotes": "Diagnosed RLS enabled with no policies causing deny-all Data API results, kept RLS enabled, and added authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." } ], "skills": { @@ -22068,7 +22130,7 @@ }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/resolve-dataapi-001-empty-results.json" }, { @@ -22114,7 +22176,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "Avatar_url was applied through `supabase db push` in #17, which shows `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration was reconciled by adding the missing local migration file `20240115000000_add_profile_bio.sql` in #15, after which Supabase CLI migration list showed local and remote aligned. No prohibited workaround or direct mutation was used." + "judgeNotes": "Applied avatar_url via `supabase db push` (#22), which showed `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` Reconciled the orphan bio migration by adding local file `supabase/migrations/20240115000000_add_profile_bio.sql` (#18), after which `supabase migration list` showed local and remote aligned (#19/#23). Read-only psql inspections were used; no disallowed direct remote mutation or prepared-statement reset workaround seen." } ], "skills": { @@ -22126,7 +22188,7 @@ }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/resolve-database-001-migration-history-mismatch.json" }, { @@ -22165,7 +22227,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on idx_events_user_created_at (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on idx_events_user_id_created_at (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -22181,7 +22243,7 @@ }, "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/resolve-performance-001-slow-query-cpu-spike.json" }, { @@ -22252,7 +22314,7 @@ }, "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/resolve-security-002-rls-cross-tenant-leak.json" } ] diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index f8ca77b3..f2ac4a45 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -32,7 +32,7 @@ { "name": "suggested relevant next steps for a Supabase + Next.js project", "passed": true, - "judgeNotes": "The response provides multiple concrete Supabase/Next.js next steps: running `supabase start`, checking Docker, installing `@supabase/supabase-js`/`@supabase/ssr`, setting `.env.local` URL and anon key, linking a remote project, and creating a first migration." + "judgeNotes": "The response includes multiple concrete Supabase/Next.js next steps: running `supabase start`, installing `@supabase/supabase-js`/`@supabase/ssr`, setting `NEXT_PUBLIC_SUPABASE_URL` and anon key env vars, linking a remote project, and creating a first migration." }, { "name": "attempted `npx plugins add supabase-community/supabase-plugin`", @@ -48,7 +48,7 @@ }, "prompt": "Help me get set up with Supabase. Do the following: 1. Install the Supabase CLI globally with `npm install -g supabase`. 2. Install the Supabase Plugin with `npx plugins add supabase-community/supabase-plugin`. 3. Review my project and determine whether Supabase is already initialized. If it is not initialized, run `supabase init`. 4. Suggest the most relevant next steps.", "promptSourcePath": "evals/build-cli-004-quickstart-nextjs/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/build-cli-004-quickstart-nextjs.json" }, { @@ -90,17 +90,17 @@ { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"24ebdf94-3b1a-4f68-92ed-d030e5f27f7f\",\"metric\":\"steps_a_msr5rarn\",\"value\":111}]}" + "notes": "status 200: [{\"user_id\":\"2e8689db-4aed-4219-bb38-dee7588fdc13\",\"metric\":\"steps_a_mst1rqp0\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"24ebdf94-3b1a-4f68-92ed-d030e5f27f7f\",\"metric\":\"steps_a_msr5rarn\",\"value\":111}]}" + "notes": "status 200: [{\"user_id\":\"2e8689db-4aed-4219-bb38-dee7588fdc13\",\"metric\":\"steps_a_mst1rqp0\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"b23c8b7b-030c-45e9-bc10-ca4e3c8c88b4\",\"metric\":\"steps_b_msr5rarn\",\"value\":222}]}" + "notes": "status 200: [{\"user_id\":\"763cc1c1-359e-4e2d-8111-f51030e2853f\",\"metric\":\"steps_b_mst1rqp0\",\"value\":222}]" }, { "name": "non-service key is not granted service access", @@ -133,40 +133,11 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"@supabase/server withSupabase edge function multi-auth\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/enterprise-sso", - "title": "Enterprise Single Sign-On" - } - ], - "resultChars": 32262 - } - ] + "calls": [] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nImplement it with the **`@supabase/server`** package, which is built for exactly\nthis kind of multi-auth Edge Function. Import it directly in your function:\n\n```ts\nimport { withSupabase } from \"npm:@supabase/server\";\n```\n\nOur product stores per-user metrics in a `user_stats` table that already exists\n(see `supabase/migrations/`), protected by row-level security so a user can read\nonly their own rows.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.\n\nGet the local stack running so the function is reachable at the path above.", "promptSourcePath": "evals/build-functions-006-dual-auth-with-server/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/build-functions-006-dual-auth-with-server.json" }, { @@ -198,7 +169,7 @@ { "name": "did not recommend read replicas for Realtime", "passed": true, - "judgeNotes": "The assistant correctly treats the request as Supabase Realtime/Postgres Changes setup, adds the messages table to the supabase_realtime publication, provides a postgres_changes client subscription, and does not recommend or confuse read replicas." + "judgeNotes": "The answer correctly treats the issue as Supabase Realtime/Postgres Changes setup by adding the messages table to the supabase_realtime publication and providing a postgres_changes client subscription. It does not recommend read replicas or confuse them with logical replication/publications." } ], "skills": { @@ -211,11 +182,77 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"enable realtime postgres changes replication table RLS\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/realtime/authorization", + "title": "Realtime Authorization" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" + }, + { + "url": "https://supabase.com/docs/guides/realtime/architecture", + "title": "Realtime Architecture" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/realtime/benchmarks", + "title": "Benchmarks" + } + ], + "resultChars": 91439 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Postgres Changes enable table replication supabase_realtime publication\", limit: 3) { nodes { ... on Guide { title href subsections(first: 3) { nodes { title content } } } } } }", + "hasContent": true, + "pages": [] + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Postgres Changes enable table replication supabase_realtime publication\", limit: 3) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", + "title": "Backup and Restore using the CLI" + }, + { + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", + "title": "Subscribing to Database Changes" + }, + { + "url": "https://supabase.com/docs/guides/database/replication/pipelines", + "title": "Set up Pipelines" + } + ], + "resultChars": 61182 + }, + { + "source": "web_fetch", + "query": "Explain exactly how to enable Postgres Changes for a table: what SQL is needed to add a table to the supabase_realtime publication, whether RLS SELECT policies affect which change events a client receives, and any current caveats/breaking changes about this feature.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/realtime/postgres-changes.md" + } + ], + "resultChars": 1439 + } + ] }, "prompt": "I'm building a simple chat app on Supabase.\n\nUsers can send messages, and I want everyone in the same room to see new\nmessages appear automatically without refreshing the page.\n\nCan you inspect the project and set up whatever Supabase needs for live updates?", "promptSourcePath": "evals/build-realtime-001-live-chat-updates/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/build-realtime-001-live-chat-updates.json" }, { @@ -241,17 +278,17 @@ { "name": "identified video-thumbnails and the 546 resource-limit pattern", "passed": true, - "judgeNotes": "Identified video thumbnail generation as the affected function/workload and correctly recognized HTTP 546 WORKER_LIMIT/resource-limit CPUTime errors, not 500/503 server errors." + "judgeNotes": "Identified `video-thumbnails` as the affected function and correctly described failures as HTTP 546 / WORKER_RESOURCE_LIMIT due to CPU resource limits, not 500/503 server errors." }, { "name": "attributed the 546s to CPU time exhaustion", "passed": true, - "judgeNotes": "The assistant explicitly attributes the 546 failures to CPU time exhaustion, citing log evidence: shutdown reason CPUTime and cpu_time_used 2000ms / cpu_time_limit 2000ms. It distinguishes this from wall-clock time, memory, exceptions, or unrelated errors." + "judgeNotes": "The assistant explicitly attributes the 546 failures to CPU time exhaustion, citing log evidence: shutdown reason CPUTime and cpu_time_used 2000ms matching cpu_time_limit 2000ms. It does not blame memory, wall-clock time, code exceptions, or unrelated errors." }, { "name": "recommended reducing/offloading CPU work as the fix", "passed": true, - "judgeNotes": "The assistant correctly identified the CPUTime limit and recommended reducing CPU work per request, optimizing thumbnail generation, and moving transcoding to a background/external worker/service rather than retrying or raising limits." + "judgeNotes": "The assistant explicitly recommends reducing per-call CPU work (seek-and-grab keyframe, avoid full decode) and moving thumbnail generation off Edge Functions to a background/external worker. It also notes the CPU limit is hard and cannot be raised." } ], "skills": { @@ -297,7 +334,7 @@ }, "prompt": "Our `video-thumbnails` edge function has been failing intermittently since this morning. It generates a thumbnail from a user-uploaded video, and about half the calls are erroring out.\n\nCan you investigate the project logs and tell me what's going on and what we should do about it?", "promptSourcePath": "evals/investigate-functions-001-546-resource-limit/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/investigate-functions-001-546-resource-limit.json" }, { @@ -367,7 +404,7 @@ { "name": "diagnosed secure default grants without weakening RLS", "passed": true, - "judgeNotes": "Diagnoses secure-by-default/Data API explicit grant issue, distinguishes grants from RLS, preserves owner-scoped RLS, grants SELECT/INSERT only to authenticated, and verifies behavior without granting anon/public or weakening policies." + "judgeNotes": "Diagnosed missing Data API table grants under secure-by-default behavior, distinguished grants from RLS, preserved owner-scoped RLS, and granted only SELECT/INSERT on public.journal_entries to authenticated." } ], "skills": { @@ -384,7 +421,7 @@ }, "prompt": "Our app lets signed-in users keep a private journal. Entries are stored in the `journal_entries` table. A user should only ever see their own, and they need to be able to create new entries from the app.\n\nI can see the rows when I query the table directly, but the journal page shows an empty list for every signed-in user. New entries don't show up after saving either. Please figure out why and fix it.", "promptSourcePath": "evals/resolve-dataapi-002-secure-default-grants/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/resolve-dataapi-002-secure-default-grants.json" }, { @@ -415,7 +452,7 @@ { "name": "user A's update actually checks off their own task", "passed": true, - "notes": "saw: [{\"id\":\"e6664c25-a1a6-4c61-8097-cd7bbb62626c\",\"is_done\":true}]" + "notes": "saw: [{\"id\":\"4bd7b118-354b-448a-ae7f-3986f005e2be\",\"is_done\":true}]" }, { "name": "user B cannot update user A's task", @@ -424,7 +461,7 @@ { "name": "diagnosed the missing USING clause and added it", "passed": true, - "judgeNotes": "Diagnosed the zero-row UPDATE as the tasks UPDATE RLS policy missing a USING clause, explained WITH CHECK vs row targeting, and fixed it with an authenticated-only ownership-scoped USING plus retained WITH CHECK on user_id = auth.uid(). RLS was not disabled and the fix was verified." + "judgeNotes": "Diagnosed the zero-row UPDATE as the tasks UPDATE policy missing a USING clause while having only WITH CHECK, explained why Postgres RLS silently updates 0 rows, and fixed it with USING (user_id = auth.uid()) while retaining WITH CHECK (user_id = auth.uid()) scoped to authenticated/user ownership." } ], "skills": { @@ -441,7 +478,7 @@ }, "prompt": "Our app lets signed-in users manage a personal `tasks` list. Users can create tasks and check them off (`is_done`).\n\nCreating a task works fine, and I can see the row in the table. But when a user checks off a task, the app's update call succeeds with no error, yet `is_done` never actually changes, and the API doesn't return the updated row either.\n\nFind out why the update has no effect and fix it.", "promptSourcePath": "evals/resolve-dataapi-002-update-zero-rows-affected/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/resolve-dataapi-002-update-zero-rows-affected.json" }, { @@ -467,7 +504,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Answer correctly says pause/restore is not the right recovery path, recommends restart as first step, distinguishes restore from restart/backups, suggests checking resource pressure and scaling/tuning, and escalating to support if recurring. Minor omission: could have mentioned logs/advisors/troubleshooting docs more explicitly, but guidance is concrete and safe." + "judgeNotes": "The answer clearly says restart first, not pause/restore; distinguishes restart from restore; and recommends concrete next steps including restart, checking advisors/logs, tuning or scaling for overload. It does not mention contacting Supabase support if the project remains unhealthy, but the core recovery guidance is safe and aligned with the rubric." } ], "skills": { @@ -483,7 +520,7 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"project unhealthy restart pause restore dashboard unresponsive\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"project unhealthy restart pause restore dashboard unusable\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -503,17 +540,17 @@ "title": "Project Pausing" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/failed-to-run-sql-query-connection-terminated-due-to-connection-timeout", - "title": "Error: Failed to run sql query: Connection terminated due to connection timeout" + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" } ], - "resultChars": 28453 + "resultChars": 28992 } ] }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", "promptSourcePath": "evals/resolve-reliability-001-unhealthy-project-recovery/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/resolve-reliability-001-unhealthy-project-recovery.json" }, { @@ -581,11 +618,40 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"publishable key secret key migrate from anon key service_role key\", limit: 5) { nodes { title href ... on Guide { content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", + "title": "Supabase for Platforms" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" + } + ], + "resultChars": 107177 + } + ] }, "prompt": "Heads-up from the platform team: the legacy JWT-based API keys (`anon` /\n`service_role`) are going away for our projects soon, in favor of the new\npublishable/secret keys. The little blog tooling app in `app/` still uses the\nlegacy keys.\n\nMigrate it over. Both scripts need to keep working — `npm run posts` and\n`npm run stats` (run them from `app/`). The local Supabase project in\n`supabase/` is already running.", "promptSourcePath": "evals/resolve-sdk-001-legacy-key-migration/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/resolve-sdk-001-legacy-key-migration.json" }, { @@ -629,7 +695,7 @@ { "name": "user A can replace their own avatar via upsert", "passed": true, - "notes": "saw: [{\"name\":\"019ff9df-beb8-74c8-b84b-21f4e91826eb/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" + "notes": "saw: [{\"name\":\"01a000ac-562a-776e-9f4a-69a3e2a879f5/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" }, { "name": "user B cannot overwrite user A's avatar", @@ -638,7 +704,7 @@ { "name": "added an owner-scoped UPDATE policy without weakening public reads", "passed": true, - "judgeNotes": "The answer correctly diagnoses missing UPDATE RLS policy on storage.objects for upsert replacements, notes public bucket only affects read/download behavior, keeps bucket public/RLS enabled, and adds an authenticated owner-scoped UPDATE policy with USING and WITH CHECK based on path user id." + "judgeNotes": "Diagnosed missing UPDATE RLS policy for upsert replacement on storage.objects, kept public-read bucket/RLS intact, and added an authenticated owner-scoped UPDATE policy with USING and WITH CHECK." } ], "skills": { @@ -646,14 +712,16 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [] + "loaded": [ + "supabase" + ] }, "docs": { "calls": [] }, "prompt": "Our app has a public `avatars` bucket so profile photos have a public URL. Each user's avatar is stored at `/avatar.png`, and the app uploads it with `upsert: true` so a new photo replaces the old one at that same path.\n\nThe very first upload for a user always works, but replacing an existing avatar fails. Find out why and fix it.", "promptSourcePath": "evals/resolve-storage-001-upsert-missing-update-policy/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/resolve-storage-001-upsert-missing-update-policy.json" }, { @@ -695,17 +763,17 @@ { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"4ddfbb4b-389b-48ab-a2e9-5379f47338e3\",\"metric\":\"steps_a_msr5t2ph\",\"value\":111}]}" + "notes": "status 200: [{\"user_id\":\"cc1eb6ac-f741-4f8d-b5e9-4a5d58c22654\",\"metric\":\"steps_a_mst1va5l\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"4ddfbb4b-389b-48ab-a2e9-5379f47338e3\",\"metric\":\"steps_a_msr5t2ph\",\"value\":111}]}" + "notes": "status 200: [{\"user_id\":\"cc1eb6ac-f741-4f8d-b5e9-4a5d58c22654\",\"metric\":\"steps_a_mst1va5l\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"aae20fc0-9396-43c2-bd78-2df3e1594f54\",\"metric\":\"steps_b_msr5t2ph\",\"value\":222}]}" + "notes": "status 200: [{\"user_id\":\"afe9748e-9c84-4bc8-a9c4-1a3b09bf75f3\",\"metric\":\"steps_b_mst1va5l\",\"value\":222}]" }, { "name": "non-service key is not granted service access", @@ -733,167 +801,11 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "web_search", - "query": "npm @supabase/server withSupabase edge function package", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/blog/introducing-supabase-server", - "title": "Introducing @supabase/server" - }, - { - "url": "https://supabase.com/blog/edge-functions-node-npm", - "title": "Edge Functions: Node and native npm compatibility" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres) | Supabase Docs" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart", - "title": "Getting Started with Edge Functions | Supabase Docs" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions | Supabase Docs" - }, - { - "url": "https://supabase.com/docs/guides/functions/wasm", - "title": "Using Wasm modules - Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/routing", - "title": "Handling Routing in Functions | Supabase Docs" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions | Supabase Docs" - } - ], - "resultChars": 2726 - }, - { - "source": "web_fetch", - "query": "Extract the full API documentation and code examples for the withSupabase function, including all auth modes (user, service, dual/either), how it reads the apikey/authorization headers, how ctx.supabase and ctx.supabaseAdmin are constructed, and any example of handling both a user JWT and a service-role caller in the same function.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/blog/introducing-supabase-server" - } - ], - "resultChars": 1789 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"@supabase/server withSupabase auth mode secret user dual authentication edge function\", limit: 10) { totalCount nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", - "title": "Getting Started with Edge Functions (Dashboard)" - } - ], - "resultChars": 78210 - }, - { - "source": "web_search", - "query": "\"@supabase/server\" withSupabase \"authMode\" OR \"ctx.authMode\" OR \"auth: ['user', 'secret']\" example", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions | Supabase Docs" - }, - { - "url": "https://supabase.com/blog/introducing-supabase-server", - "title": "Introducing @supabase/server" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth | Supabase Docs" - }, - { - "url": "https://supabase.com/docs/reference/server/types-credentials", - "title": "Server API Reference | Supabase Docs" - }, - { - "url": "https://supabase.com/features/server-side-auth", - "title": "Server-side Auth | Supabase Features" - } - ], - "resultChars": 2668 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"verify_jwt config.toml functions combining auth modes secret user apikey header disable\", limit: 5) { totalCount nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/deploy", - "title": "Deploy to Production" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - } - ], - "resultChars": 51704 - } - ] + "calls": [] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nImplement it with the **`@supabase/server`** package, which is built for exactly\nthis kind of multi-auth Edge Function. Import it directly in your function:\n\n```ts\nimport { withSupabase } from \"npm:@supabase/server\";\n```\n\nOur product stores per-user metrics in a `user_stats` table that already exists\n(see `supabase/migrations/`), protected by row-level security so a user can read\nonly their own rows.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.\n\nGet the local stack running so the function is reachable at the path above.", "promptSourcePath": "evals/build-functions-006-dual-auth-with-server/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/build-functions-006-dual-auth-with-server.json" }, { @@ -925,7 +837,7 @@ { "name": "did not recommend read replicas for Realtime", "passed": true, - "judgeNotes": "The assistant correctly treats the issue as Supabase Realtime/Postgres Changes setup, adds the table to the supabase_realtime publication, discusses RLS and client subscription code, and does not recommend or imply read replicas are needed or useful for live updates." + "judgeNotes": "The answer correctly treats the issue as Supabase Realtime/Postgres Changes setup, adds the messages table to the supabase_realtime publication, and does not recommend or conflate read replicas." } ], "skills": { @@ -937,7 +849,7 @@ }, "prompt": "I'm building a simple chat app on Supabase.\n\nUsers can send messages, and I want everyone in the same room to see new\nmessages appear automatically without refreshing the page.\n\nCan you inspect the project and set up whatever Supabase needs for live updates?", "promptSourcePath": "evals/build-realtime-001-live-chat-updates/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/build-realtime-001-live-chat-updates.json" }, { @@ -963,17 +875,17 @@ { "name": "identified video-thumbnails and the 546 resource-limit pattern", "passed": true, - "judgeNotes": "Identified `video-thumbnails` as the affected function and correctly recognized HTTP 546 resource/CPU limit failures rather than 500/503 server errors." + "judgeNotes": "The assistant identified `video-thumbnails` as the affected function and correctly described failures as HTTP 546 due to CPU/resource limit shutdowns, not 500/503 server errors." }, { "name": "attributed the 546s to CPU time exhaustion", "passed": true, - "judgeNotes": "The assistant explicitly attributes the 546 failures to CPU time exhaustion, citing the log shutdown reason CPUTime and cpu_time_used/cpu_time_limit of 2000ms." + "judgeNotes": "The assistant specifically attributes the 546 failures to CPU time exhaustion, citing the log evidence: shutdown reason CPUTime and cpu_time_used/limit at 2000ms. It explicitly rules out code exceptions, memory, network/storage, and wall-clock time." }, { "name": "recommended reducing/offloading CPU work as the fix", "passed": true, - "judgeNotes": "The assistant clearly identifies the fixed CPU limit and recommends reducing CPU work per call, optimizing seeking/decoding, reducing input size, and moving heavy thumbnail generation to a background/external worker. It also explicitly rejects retries/scaling as fixes." + "judgeNotes": "The assistant correctly identifies the fixed CPU-time limit and recommends reducing/offloading CPU-intensive thumbnail generation work via background/external workers and optimizing extraction to reduce per-invocation CPU." } ], "skills": { @@ -985,7 +897,7 @@ }, "prompt": "Our `video-thumbnails` edge function has been failing intermittently since this morning. It generates a thumbnail from a user-uploaded video, and about half the calls are erroring out.\n\nCan you investigate the project logs and tell me what's going on and what we should do about it?", "promptSourcePath": "evals/investigate-functions-001-546-resource-limit/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/investigate-functions-001-546-resource-limit.json" }, { @@ -1055,7 +967,7 @@ { "name": "diagnosed secure default grants without weakening RLS", "passed": true, - "judgeNotes": "Diagnoses secure-by-default/missing Data API grants, distinguishes grants from RLS, preserves existing owner-scoped RLS, grants only SELECT/INSERT on public.journal_entries to authenticated, and keeps anon/public ungranted." + "judgeNotes": "Diagnoses missing authenticated table grants as distinct from RLS, preserves owner-scoped RLS, grants only SELECT/INSERT on public.journal_entries to authenticated, and does not weaken security or grant anon/public. Mentions secure-default grants theme though not explicitly api.auto_expose_new_tables; the handled issue matches the rubric." } ], "skills": { @@ -1067,7 +979,7 @@ }, "prompt": "Our app lets signed-in users keep a private journal. Entries are stored in the `journal_entries` table. A user should only ever see their own, and they need to be able to create new entries from the app.\n\nI can see the rows when I query the table directly, but the journal page shows an empty list for every signed-in user. New entries don't show up after saving either. Please figure out why and fix it.", "promptSourcePath": "evals/resolve-dataapi-002-secure-default-grants/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-dataapi-002-secure-default-grants.json" }, { @@ -1098,7 +1010,7 @@ { "name": "user A's update actually checks off their own task", "passed": true, - "notes": "saw: [{\"id\":\"ef42dd34-2cac-4bd1-9c47-bbf8019b7e0d\",\"is_done\":true}]" + "notes": "saw: [{\"id\":\"22c44602-3376-4fde-9e40-b2921542fe6d\",\"is_done\":true}]" }, { "name": "user B cannot update user A's task", @@ -1107,7 +1019,7 @@ { "name": "diagnosed the missing USING clause and added it", "passed": true, - "judgeNotes": "The answer correctly identifies the missing UPDATE USING clause as the cause of silent zero-row updates, explains WITH CHECK vs USING, and fixes the authenticated task-owner policy with USING (user_id = auth.uid()) while preserving WITH CHECK (user_id = auth.uid()) and RLS." + "judgeNotes": "The assistant correctly identified the missing USING clause on the authenticated UPDATE policy as the cause of silent 0-row updates, explained WITH CHECK vs USING, and fixed it with USING (user_id = auth.uid()) while preserving WITH CHECK (user_id = auth.uid())." } ], "skills": { @@ -1115,40 +1027,11 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"row level security policy UPDATE USING WITH CHECK default behavior\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/auth/enterprise-sso/auth-sso-saml", - "title": "Single Sign-On with SAML 2.0 for Projects" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/rls-simplified-BJTcS8", - "title": "RLS Simplified" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes", - "title": "Postgres Changes" - } - ], - "resultChars": 125460 - } - ] + "calls": [] }, "prompt": "Our app lets signed-in users manage a personal `tasks` list. Users can create tasks and check them off (`is_done`).\n\nCreating a task works fine, and I can see the row in the table. But when a user checks off a task, the app's update call succeeds with no error, yet `is_done` never actually changes, and the API doesn't return the updated row either.\n\nFind out why the update has no effect and fix it.", "promptSourcePath": "evals/resolve-dataapi-002-update-zero-rows-affected/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-dataapi-002-update-zero-rows-affected.json" }, { @@ -1174,7 +1057,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "The answer clearly says restart is the better first recovery step and that pause/restore is heavier and not the right first move. It distinguishes restart from pause/restore and offers concrete safe steps such as restarting and checking logs/advisors. It could be stronger by mentioning waiting, resource pressure/scaling, troubleshooting docs, and contacting Supabase support if unresolved." + "judgeNotes": "Answer correctly rejects pause/restore as first-line recovery, distinguishes restore from restart, recommends restart and checking logs/advisors/resource pressure. It could be stronger by explicitly saying to wait a few minutes, reduce workload/scale, and contact Supabase support if unresolved, but it provides concrete safe first steps and does not conflate restore with restart." } ], "skills": { @@ -1182,11 +1065,40 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"restart project unhealthy vs pause restore dashboard unresponsive\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/upgrading", + "title": "Upgrading" + }, + { + "url": "https://supabase.com/docs/guides/platform/backups", + "title": "Database Backups" + }, + { + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" + }, + { + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "title": "Project Status reports unhealthy services" + } + ], + "resultChars": 28992 + } + ] }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", "promptSourcePath": "evals/resolve-reliability-001-unhealthy-project-recovery/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-reliability-001-unhealthy-project-recovery.json" }, { @@ -1253,7 +1165,7 @@ }, "prompt": "Heads-up from the platform team: the legacy JWT-based API keys (`anon` /\n`service_role`) are going away for our projects soon, in favor of the new\npublishable/secret keys. The little blog tooling app in `app/` still uses the\nlegacy keys.\n\nMigrate it over. Both scripts need to keep working — `npm run posts` and\n`npm run stats` (run them from `app/`). The local Supabase project in\n`supabase/` is already running.", "promptSourcePath": "evals/resolve-sdk-001-legacy-key-migration/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-sdk-001-legacy-key-migration.json" }, { @@ -1297,7 +1209,7 @@ { "name": "user A can replace their own avatar via upsert", "passed": true, - "notes": "saw: [{\"name\":\"019ff9df-4b74-73e9-a360-757cc38f38f8/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" + "notes": "saw: [{\"name\":\"01a000ad-30a5-730c-82d5-4d68e28db114/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" }, { "name": "user B cannot overwrite user A's avatar", @@ -1306,7 +1218,7 @@ { "name": "added an owner-scoped UPDATE policy without weakening public reads", "passed": true, - "judgeNotes": "Diagnosed missing UPDATE RLS policy for upsert on storage.objects, kept public bucket/RLS intact, and added authenticated owner-scoped UPDATE policy with USING and WITH CHECK." + "judgeNotes": "Diagnoses missing UPDATE RLS policy for upsert replacement, explains public bucket only covers read URLs, keeps public/RLS setup intact, and adds an authenticated owner-scoped UPDATE policy with USING and WITH CHECK on storage.objects." } ], "skills": { @@ -1318,7 +1230,7 @@ }, "prompt": "Our app has a public `avatars` bucket so profile photos have a public URL. Each user's avatar is stored at `/avatar.png`, and the app uploads it with `upsert: true` so a new photo replaces the old one at that same path.\n\nThe very first upload for a user always works, but replacing an existing avatar fails. Find out why and fix it.", "promptSourcePath": "evals/resolve-storage-001-upsert-missing-update-policy/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-storage-001-upsert-missing-update-policy.json" } ] From cf1e91cecbfd0ea7b14d7c29c6949d50fa062d4a Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Fri, 14 Aug 2026 15:25:19 +0100 Subject: [PATCH 3/3] test: cover any-pass attempt aggregation Exports aggregateAttempts and adds cases for: passes if any attempt passed (keeps the passing tree), fails only when all failed, counts only attempts that produced a result, and throws when none did. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../scripts/run-vercel-evals.test.ts | 97 ++++++++++++++++++- apps/framework/scripts/run-vercel-evals.ts | 2 +- 2 files changed, 96 insertions(+), 3 deletions(-) diff --git a/apps/framework/scripts/run-vercel-evals.test.ts b/apps/framework/scripts/run-vercel-evals.test.ts index cc364a30..14d1d6c7 100644 --- a/apps/framework/scripts/run-vercel-evals.test.ts +++ b/apps/framework/scripts/run-vercel-evals.test.ts @@ -1,6 +1,20 @@ +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { APIError } from '@vercel/sandbox'; -import { describe, expect, it } from 'vitest'; -import { parsePairs, runBounded } from './run-vercel-evals.js'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { + aggregateAttempts, + parsePairs, + runBounded, + type EvalPair, +} from './run-vercel-evals.js'; import { isRetryableSandboxCreateError, tagValue } from './vercel-sandbox.js'; describe('Vercel eval controller', () => { @@ -63,3 +77,82 @@ describe('Vercel eval controller', () => { ); }); }); + +describe('aggregateAttempts (any-pass)', () => { + const pair: EvalPair = { + eval_id: 'e1', + experiment: 'exp1', + experiment_suite: 'benchmark', + eval_suite: 'benchmark', + }; + let root: string; + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'agg-test-')); + }); + afterEach(() => rmSync(root, { recursive: true, force: true })); + + const attemptDir = ( + name: string, + result: { passed: boolean; marker: string } | null + ): string => { + const dir = join(root, name); + mkdirSync(dir, { recursive: true }); + if (result) { + writeFileSync( + join(dir, `${pair.eval_id}.json`), + JSON.stringify({ ...result, attempts: 1, checks: [] }) + ); + } + return dir; + }; + const output = () => + JSON.parse( + readFileSync( + join(root, 'out', `raw-results-${pair.experiment}__${pair.eval_id}`, `${pair.eval_id}.json`), + 'utf8' + ) + ) as { passed: boolean; attempts: number; marker: string }; + + it('passes if any attempt passed and keeps the passing tree', () => { + aggregateAttempts( + pair, + [ + attemptDir('a1', { passed: false, marker: 'fail' }), + attemptDir('a2', { passed: true, marker: 'pass' }), + ], + join(root, 'out') + ); + expect(output()).toMatchObject({ passed: true, attempts: 2, marker: 'pass' }); + }); + + it('fails only when every attempt failed', () => { + aggregateAttempts( + pair, + [ + attemptDir('a1', { passed: false, marker: 'a1' }), + attemptDir('a2', { passed: false, marker: 'a2' }), + ], + join(root, 'out') + ); + expect(output()).toMatchObject({ passed: false, attempts: 2 }); + }); + + it('counts only attempts that produced a result file', () => { + aggregateAttempts( + pair, + [ + attemptDir('a1', { passed: true, marker: 'pass' }), + attemptDir('a2', null), + ], + join(root, 'out') + ); + expect(output()).toMatchObject({ passed: true, attempts: 1, marker: 'pass' }); + }); + + it('throws when no attempt produced a result file', () => { + expect(() => + aggregateAttempts(pair, [attemptDir('a1', null)], join(root, 'out')) + ).toThrow('no attempt produced a result'); + }); +}); diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts index a68486ea..a3c4dcb2 100644 --- a/apps/framework/scripts/run-vercel-evals.ts +++ b/apps/framework/scripts/run-vercel-evals.ts @@ -294,7 +294,7 @@ async function runParallelAttempts( * passing attempt if there is one, else the first) is kept, with its `attempts` * count set to the number of attempts that produced a result. */ -function aggregateAttempts( +export function aggregateAttempts( pair: EvalPair, attemptDirs: string[], outputDir: string