diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index 006be91c..5a4b4a5a 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -1662,6 +1662,135 @@ "attempts": 2, "sourcePath": "claude-code-opus-5/investigate-auth-001-deleted-user-access.json" }, + { + "experiment": "claude-code-opus-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-5", + "reasoningEffort": "high" + }, + "eval": "investigate-functions-001-546-resource-limit", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "identified video-thumbnails and the 546 resource-limit pattern", + "passed": true, + "judgeNotes": "Identified `video-thumbnails` as the affected function and correctly described failures as HTTP 546 due to CPU/resource limit, while treating `welcome-email` 500 as unrelated." + }, + { + "name": "attributed the 546s to CPU time exhaustion", + "passed": true, + "judgeNotes": "The assistant specifically attributes the HTTP 546 failures to CPU time exhaustion, citing log evidence: shutdown reason CPUTime and cpu_time_used/cpu_time_limit at 2000ms/2000ms. It does not blame memory, wall-clock time, code exceptions, or the unrelated welcome-email error." + }, + { + "name": "recommended reducing/offloading CPU work as the fix", + "passed": true, + "judgeNotes": "The assistant recommends reducing/offloading CPU-intensive thumbnail work: moving decode off the isolate to a media service/transform API, cheaper extraction, capping input size, and notes retries/scaling won't help because CPU time is fixed." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "How to debug Edge Function failures: what log sources/tables exist (edge_logs, function_edge_logs, function_logs), how to query them, and what common causes of intermittent edge function errors (timeouts, memory limits, CPU limits, WORKER_LIMIT, boot errors) look like in logs.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging.md" + } + ], + "resultChars": 786 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge function shutdown reason CPUTime cpu_time_limit 546 error\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-546-error-response", + "title": "546 - WORKER_RESOURCE_LIMIT Exceeded / WORKER_LIMIT Exceeded" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-shutdown-reasons-explained", + "title": "Edge Function shutdown reasons explained" + }, + { + "url": "https://supabase.com/docs/guides/functions/status-codes", + "title": "Status codes" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-504-error-response", + "title": "Edge Function 504 error response" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-wall-clock-time-limit-reached-Nk38bW", + "title": "Edge Function 'wall clock time limit reached'" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" + } + ], + "resultChars": 48463 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge functions CPU time limit wall clock limits quotas\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/limits", + "title": "Limits" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-cpu-limits", + "title": "Understanding Edge Function CPU limits" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-wall-clock-time-limit-reached-Nk38bW", + "title": "Edge Function 'wall clock time limit reached'" + }, + { + "url": "https://supabase.com/docs/guides/functions/recursive-functions", + "title": "Recursive / Nested Function Calls" + }, + { + "url": "https://supabase.com/docs/guides/platform/compute-and-disk", + "title": "Compute and Disk" + }, + { + "url": "https://supabase.com/docs/guides/platform/billing-faq", + "title": "Billing FAQ" + } + ], + "resultChars": 56359 + } + ] + }, + "prompt": "Our `video-thumbnails` edge function has been failing intermittently since this morning. It generates a thumbnail from a user-uploaded video, and about half the calls are erroring out.\n\nCan you investigate the project logs and tell me what's going on and what we should do about it?", + "promptSourcePath": "evals/investigate-functions-001-546-resource-limit/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-5/investigate-functions-001-546-resource-limit.json" + }, { "experiment": "claude-code-opus-5", "experimentSuite": "benchmark", @@ -1893,6 +2022,63 @@ "attempts": 1, "sourcePath": "claude-code-opus-5/resolve-dataapi-001-empty-results.json" }, + { + "experiment": "claude-code-opus-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-5", + "reasoningEffort": "high" + }, + "eval": "resolve-dataapi-002-update-zero-rows-affected", + "stage": "resolve", + "product": [ + "data-api", + "database" + ], + "topic": [ + "rls" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "RLS still enabled on tasks", + "passed": true + }, + { + "name": "user A's update actually checks off their own task", + "passed": true, + "notes": "saw: [{\"id\":\"1df0ce54-ee29-48ef-ad23-a414658d4581\",\"is_done\":true}]" + }, + { + "name": "user B cannot update user A's task", + "passed": true + }, + { + "name": "diagnosed the missing USING clause and added it", + "passed": true, + "judgeNotes": "Diagnoses missing USING clause on the tasks UPDATE RLS policy as the cause of silent 0-row updates, explains WITH CHECK vs USING, and fixes by recreating the authenticated UPDATE policy with user_id/auth.uid() in both USING and WITH CHECK while keeping RLS/security boundaries intact." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app lets signed-in users manage a personal `tasks` list. Users can create tasks and check them off (`is_done`).\n\nCreating a task works fine, and I can see the row in the table. But when a user checks off a task, the app's update call succeeds with no error, yet `is_done` never actually changes, and the API doesn't return the updated row either.\n\nFind out why the update has no effect and fix it.", + "promptSourcePath": "evals/resolve-dataapi-002-update-zero-rows-affected/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-5/resolve-dataapi-002-update-zero-rows-affected.json" + }, { "experiment": "claude-code-opus-5", "experimentSuite": "benchmark", @@ -2019,6 +2205,70 @@ "attempts": 1, "sourcePath": "claude-code-opus-5/resolve-performance-001-slow-query-cpu-spike.json" }, + { + "experiment": "claude-code-opus-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-5", + "reasoningEffort": "high" + }, + "eval": "resolve-performance-002-rls-auth-initplan", + "stage": "resolve", + "product": [ + "database", + "data-api" + ], + "topic": [ + "rls", + "sql" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS still enabled on documents", + "passed": true + }, + { + "name": "user A still reads exactly their own documents", + "passed": true, + "notes": "saw 2 rows" + }, + { + "name": "user B cannot read user A's documents", + "passed": true + }, + { + "name": "SELECT policy evaluates auth.uid() as an InitPlan (wrapped in a subquery)", + "passed": true, + "notes": "(user_id = ( SELECT auth.uid() AS uid))" + }, + { + "name": "diagnosed per-row auth.uid() re-evaluation and applied the InitPlan fix", + "passed": true, + "judgeNotes": "Diagnosed the RLS policy's per-row auth.uid()/current_setting evaluation, fixed it with (select auth.uid()) in owner-scoped policies, kept RLS semantics and authenticated-only access. Extra index/analyze changes do not invalidate the required fix." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Our `documents` table has Row Level Security enabled so each signed-in user only sees their own rows. It worked fine in testing, but now that the table has grown the document list has become painfully slow — a simple \"list my documents\" query that should be instant now takes seconds and gets slower as the table grows, even though each user only owns a handful of rows.\n\nWe don't want to weaken the security model — users must still only ever see their own documents. Find out why the query is so slow and fix it.", + "promptSourcePath": "evals/resolve-performance-002-rls-auth-initplan/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-5/resolve-performance-002-rls-auth-initplan.json" + }, { "experiment": "claude-code-opus-5", "experimentSuite": "benchmark", @@ -2096,6 +2346,76 @@ "attempts": 1, "sourcePath": "claude-code-opus-5/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "claude-code-opus-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-5", + "reasoningEffort": "high" + }, + "eval": "resolve-storage-001-upsert-missing-update-policy", + "stage": "resolve", + "product": [ + "storage", + "database" + ], + "topic": [ + "rls" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "bucket avatars exists", + "passed": true + }, + { + "name": "bucket avatars stays public", + "passed": true, + "notes": "the bucket being public is intentional (avatars need a public URL); it is not the bug" + }, + { + "name": "RLS still enabled on storage.objects", + "passed": true + }, + { + "name": "anon can still read the public avatar", + "passed": true + }, + { + "name": "user A can replace their own avatar via upsert", + "passed": true, + "notes": "saw: [{\"name\":\"019ff5ef-8f84-7599-a509-e05c95a370b6/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" + }, + { + "name": "user B cannot overwrite user A's avatar", + "passed": true + }, + { + "name": "added an owner-scoped UPDATE policy without weakening public reads", + "passed": true, + "judgeNotes": "Diagnoses missing UPDATE RLS policy for storage.objects during upsert replacement, notes public bucket only covers SELECT/download, keeps public read/RLS intact, and adds an authenticated owner-scoped UPDATE policy with both USING and WITH CHECK." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app has a public `avatars` bucket so profile photos have a public URL. Each user's avatar is stored at `/avatar.png`, and the app uploads it with `upsert: true` so a new photo replaces the old one at that same path.\n\nThe very first upload for a user always works, but replacing an existing avatar fails. Find out why and fix it.", + "promptSourcePath": "evals/resolve-storage-001-upsert-missing-update-policy/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-5/resolve-storage-001-upsert-missing-update-policy.json" + }, { "experiment": "claude-code-opus-5-no-skills", "experimentSuite": "no-skills", @@ -3757,10 +4077,58 @@ "modelId": "claude-opus-5", "reasoningEffort": "high" }, - "eval": "investigate-realtime-001-subscribed-no-events", + "eval": "investigate-functions-001-546-resource-limit", "stage": "investigate", "product": [ - "realtime", + "edge-functions" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "identified video-thumbnails and the 546 resource-limit pattern", + "passed": true, + "judgeNotes": "Identified `video-thumbnails` as the affected function and correctly characterized failures as HTTP 546/resource limit CPU-time shutdowns, not 500/503 errors. Also treated `welcome-email` 500 as unrelated." + }, + { + "name": "attributed the 546s to CPU time exhaustion", + "passed": true, + "judgeNotes": "The assistant specifically attributes the 546 failures to CPU time exhaustion, citing log evidence: `shutdown (reason: CPUTime, cpu_time_used: 2000ms, cpu_time_limit: 2000ms)`, and distinguishes it from exceptions, wall time, memory, and the unrelated welcome-email error." + }, + { + "name": "recommended reducing/offloading CPU work as the fix", + "passed": true, + "judgeNotes": "Recommended offloading CPU-intensive video decoding/thumbnailing to a container or media service, making it async via queue/worker, client-side generation, and reducing in-function CPU by using cheaper thumbnail extraction." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our `video-thumbnails` edge function has been failing intermittently since this morning. It generates a thumbnail from a user-uploaded video, and about half the calls are erroring out.\n\nCan you investigate the project logs and tell me what's going on and what we should do about it?", + "promptSourcePath": "evals/investigate-functions-001-546-resource-limit/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-5-no-skills/investigate-functions-001-546-resource-limit.json" + }, + { + "experiment": "claude-code-opus-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-5", + "reasoningEffort": "high" + }, + "eval": "investigate-realtime-001-subscribed-no-events", + "stage": "investigate", + "product": [ + "realtime", "database" ], "topic": [ @@ -3924,6 +4292,58 @@ "attempts": 1, "sourcePath": "claude-code-opus-5-no-skills/resolve-dataapi-001-empty-results.json" }, + { + "experiment": "claude-code-opus-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-5", + "reasoningEffort": "high" + }, + "eval": "resolve-dataapi-002-update-zero-rows-affected", + "stage": "resolve", + "product": [ + "data-api", + "database" + ], + "topic": [ + "rls" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "RLS still enabled on tasks", + "passed": true + }, + { + "name": "user A's update actually checks off their own task", + "passed": true, + "notes": "saw: [{\"id\":\"6a99b1c7-9f21-49f4-9cd2-92bf4c62d0db\",\"is_done\":true}]" + }, + { + "name": "user B cannot update user A's task", + "passed": true + }, + { + "name": "diagnosed the missing USING clause and added it", + "passed": true, + "judgeNotes": "The assistant correctly identifies the missing USING clause on the UPDATE policy as the cause of silent zero-row updates, explains USING vs WITH CHECK, and fixes it with an authenticated policy scoped to user_id = auth.uid() while retaining WITH CHECK and RLS." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app lets signed-in users manage a personal `tasks` list. Users can create tasks and check them off (`is_done`).\n\nCreating a task works fine, and I can see the row in the table. But when a user checks off a task, the app's update call succeeds with no error, yet `is_done` never actually changes, and the API doesn't return the updated row either.\n\nFind out why the update has no effect and fix it.", + "promptSourcePath": "evals/resolve-dataapi-002-update-zero-rows-affected/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-5-no-skills/resolve-dataapi-002-update-zero-rows-affected.json" + }, { "experiment": "claude-code-opus-5-no-skills", "experimentSuite": "no-skills", @@ -4039,6 +4459,65 @@ "attempts": 1, "sourcePath": "claude-code-opus-5-no-skills/resolve-performance-001-slow-query-cpu-spike.json" }, + { + "experiment": "claude-code-opus-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-5", + "reasoningEffort": "high" + }, + "eval": "resolve-performance-002-rls-auth-initplan", + "stage": "resolve", + "product": [ + "database", + "data-api" + ], + "topic": [ + "rls", + "sql" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS still enabled on documents", + "passed": true + }, + { + "name": "user A still reads exactly their own documents", + "passed": true, + "notes": "saw 2 rows" + }, + { + "name": "user B cannot read user A's documents", + "passed": true + }, + { + "name": "SELECT policy evaluates auth.uid() as an InitPlan (wrapped in a subquery)", + "passed": true, + "notes": "(user_id = ( SELECT auth.uid() AS uid))" + }, + { + "name": "diagnosed per-row auth.uid() re-evaluation and applied the InitPlan fix", + "passed": true, + "judgeNotes": "Identified auth.uid() being evaluated per row in the RLS predicate, fixed policies by wrapping auth.uid() in a scalar subquery so it becomes an InitPlan evaluated once, and preserved row-owner scoping/RLS semantics." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our `documents` table has Row Level Security enabled so each signed-in user only sees their own rows. It worked fine in testing, but now that the table has grown the document list has become painfully slow — a simple \"list my documents\" query that should be instant now takes seconds and gets slower as the table grows, even though each user only owns a handful of rows.\n\nWe don't want to weaken the security model — users must still only ever see their own documents. Find out why the query is so slow and fix it.", + "promptSourcePath": "evals/resolve-performance-002-rls-auth-initplan/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-5-no-skills/resolve-performance-002-rls-auth-initplan.json" + }, { "experiment": "claude-code-opus-5-no-skills", "experimentSuite": "no-skills", @@ -4111,6 +4590,71 @@ "attempts": 1, "sourcePath": "claude-code-opus-5-no-skills/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "claude-code-opus-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-5", + "reasoningEffort": "high" + }, + "eval": "resolve-storage-001-upsert-missing-update-policy", + "stage": "resolve", + "product": [ + "storage", + "database" + ], + "topic": [ + "rls" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "bucket avatars exists", + "passed": true + }, + { + "name": "bucket avatars stays public", + "passed": true, + "notes": "the bucket being public is intentional (avatars need a public URL); it is not the bug" + }, + { + "name": "RLS still enabled on storage.objects", + "passed": true + }, + { + "name": "anon can still read the public avatar", + "passed": true + }, + { + "name": "user A can replace their own avatar via upsert", + "passed": true, + "notes": "saw: [{\"name\":\"019ff5ee-1a1e-7609-bbc0-7c5c40be6d74/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" + }, + { + "name": "user B cannot overwrite user A's avatar", + "passed": true + }, + { + "name": "added an owner-scoped UPDATE policy without weakening public reads", + "passed": true, + "judgeNotes": "Diagnoses missing UPDATE RLS policy for upsert replacement, explains public bucket only covers read/download, keeps bucket public/RLS enabled, and adds an authenticated owner-scoped UPDATE policy with USING and WITH CHECK." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app has a public `avatars` bucket so profile photos have a public URL. Each user's avatar is stored at `/avatar.png`, and the app uploads it with `upsert: true` so a new photo replaces the old one at that same path.\n\nThe very first upload for a user always works, but replacing an existing avatar fails. Find out why and fix it.", + "promptSourcePath": "evals/resolve-storage-001-upsert-missing-update-policy/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-5-no-skills/resolve-storage-001-upsert-missing-update-policy.json" + }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", @@ -5650,6 +6194,82 @@ "attempts": 2, "sourcePath": "claude-code-sonnet-5/investigate-auth-001-deleted-user-access.json" }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "investigate-functions-001-546-resource-limit", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "identified video-thumbnails and the 546 resource-limit pattern", + "passed": true, + "judgeNotes": "Identified video-thumbnails as affected and correctly characterized failures as HTTP 546 resource/CPU limit shutdowns, not 500/503 server errors." + }, + { + "name": "attributed the 546s to CPU time exhaustion", + "passed": true, + "judgeNotes": "The assistant specifically attributes the 546 failures to CPU time exhaustion, citing log evidence: shutdown reason CPUTime and cpu_time_used/cpu_time_limit at 2000ms. It does not blame memory, wall-clock time, exceptions, or unrelated errors." + }, + { + "name": "recommended reducing/offloading CPU work as the fix", + "passed": true, + "judgeNotes": "The assistant clearly identifies CPUTime limit and recommends reducing CPU cost of thumbnail generation and moving transcoding to a background worker/external CPU-budgeted service, with enqueueing from the Edge Function. This matches the required fix and does not suggest raising limits as the solution." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "Summarize how to debug edge function errors using logs, especially intermittent failures.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging.md" + } + ], + "resultChars": 1186 + }, + { + "source": "web_fetch", + "query": "What does a 546 status code and \"shutdown (reason: CPUTime, cpu_time_used, cpu_time_limit)\" message mean for edge functions? What are the CPU time limits and how can this be fixed?", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/limits.md" + } + ], + "resultChars": 1094 + } + ] + }, + "prompt": "Our `video-thumbnails` edge function has been failing intermittently since this morning. It generates a thumbnail from a user-uploaded video, and about half the calls are erroring out.\n\nCan you investigate the project logs and tell me what's going on and what we should do about it?", + "promptSourcePath": "evals/investigate-functions-001-546-resource-limit/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/investigate-functions-001-546-resource-limit.json" + }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", @@ -5850,32 +6470,118 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "resolve-database-001-migration-history-mismatch", + "eval": "resolve-dataapi-002-update-zero-rows-affected", "stage": "resolve", "product": [ + "data-api", "database" ], "topic": [ - "migrations" + "rls" ], "suite": "benchmark", - "interface": "cli", "passed": true, "checks": [ { - "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true - }, - { - "name": "migration 20240220000000 is recorded in the remote history", + "name": "RLS still enabled on tasks", "passed": true }, { - "name": "remote migration history matches local migration files", - "passed": true + "name": "user A's update actually checks off their own task", + "passed": true, + "notes": "saw: [{\"id\":\"7bb872a2-4722-4b8c-a5be-14ee4bd4571d\",\"is_done\":true}]" }, { - "name": "local migrations are a valid reconciled sequence", + "name": "user B cannot update user A's task", + "passed": true + }, + { + "name": "diagnosed the missing USING clause and added it", + "passed": true, + "judgeNotes": "The assistant correctly identified the missing USING clause on the UPDATE RLS policy as the cause of silent 0-row updates, explained USING vs WITH CHECK, and fixed the tasks policy for authenticated users with USING auth.uid() = user_id while keeping WITH CHECK auth.uid() = user_id and RLS enabled." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"row level security policy USING clause default WITH CHECK UPDATE\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/rls-simplified-BJTcS8", + "title": "RLS Simplified" + }, + { + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0002_auth_users_exposed", + "title": "Database Advisor: Lint 0002_auth_users_exposed" + }, + { + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0024_permissive_rls_policy", + "title": "Database Advisor: Lint 0024_permissive_rls_policy" + } + ], + "resultChars": 62534 + } + ] + }, + "prompt": "Our app lets signed-in users manage a personal `tasks` list. Users can create tasks and check them off (`is_done`).\n\nCreating a task works fine, and I can see the row in the table. But when a user checks off a task, the app's update call succeeds with no error, yet `is_done` never actually changes, and the API doesn't return the updated row either.\n\nFind out why the update has no effect and fix it.", + "promptSourcePath": "evals/resolve-dataapi-002-update-zero-rows-affected/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/resolve-dataapi-002-update-zero-rows-affected.json" + }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "resolve-database-001-migration-history-mismatch", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "the avatar_url column is applied on the hosted profiles table", + "passed": true + }, + { + "name": "migration 20240220000000 is recorded in the remote history", + "passed": true + }, + { + "name": "remote migration history matches local migration files", + "passed": true + }, + { + "name": "local migrations are a valid reconciled sequence", "passed": true }, { @@ -5967,6 +6673,71 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5/resolve-performance-001-slow-query-cpu-spike.json" }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "resolve-performance-002-rls-auth-initplan", + "stage": "resolve", + "product": [ + "database", + "data-api" + ], + "topic": [ + "rls", + "sql" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS still enabled on documents", + "passed": true + }, + { + "name": "user A still reads exactly their own documents", + "passed": true, + "notes": "saw 2 rows" + }, + { + "name": "user B cannot read user A's documents", + "passed": true + }, + { + "name": "SELECT policy evaluates auth.uid() as an InitPlan (wrapped in a subquery)", + "passed": true, + "notes": "(user_id = ( SELECT auth.uid() AS uid))" + }, + { + "name": "diagnosed per-row auth.uid() re-evaluation and applied the InitPlan fix", + "passed": true, + "judgeNotes": "The answer identifies the RLS auth.uid()/current_setting predicate as the scaling issue, fixes policies by wrapping auth.uid() in a scalar subquery so it becomes an InitPlan evaluated once per query, keeps owner scoping with user_id equality, and keeps RLS/policies for authenticated users rather than widening access." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Our `documents` table has Row Level Security enabled so each signed-in user only sees their own rows. It worked fine in testing, but now that the table has grown the document list has become painfully slow — a simple \"list my documents\" query that should be instant now takes seconds and gets slower as the table grows, even though each user only owns a handful of rows.\n\nWe don't want to weaken the security model — users must still only ever see their own documents. Find out why the query is so slow and fix it.", + "promptSourcePath": "evals/resolve-performance-002-rls-auth-initplan/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/resolve-performance-002-rls-auth-initplan.json" + }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", @@ -6044,6 +6815,76 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "resolve-storage-001-upsert-missing-update-policy", + "stage": "resolve", + "product": [ + "storage", + "database" + ], + "topic": [ + "rls" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "bucket avatars exists", + "passed": true + }, + { + "name": "bucket avatars stays public", + "passed": true, + "notes": "the bucket being public is intentional (avatars need a public URL); it is not the bug" + }, + { + "name": "RLS still enabled on storage.objects", + "passed": true + }, + { + "name": "anon can still read the public avatar", + "passed": true + }, + { + "name": "user A can replace their own avatar via upsert", + "passed": true, + "notes": "saw: [{\"name\":\"019ff5ed-eaf5-761a-9cbb-62fa4f7ff340/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" + }, + { + "name": "user B cannot overwrite user A's avatar", + "passed": true + }, + { + "name": "added an owner-scoped UPDATE policy without weakening public reads", + "passed": true, + "judgeNotes": "Diagnosed missing storage.objects UPDATE policy for avatar upsert replacement, explained public bucket only covers read/download, and added an authenticated owner-scoped UPDATE policy with USING and WITH CHECK while keeping public read/RLS intact." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app has a public `avatars` bucket so profile photos have a public URL. Each user's avatar is stored at `/avatar.png`, and the app uploads it with `upsert: true` so a new photo replaces the old one at that same path.\n\nThe very first upload for a user always works, but replacing an existing avatar fails. Find out why and fix it.", + "promptSourcePath": "evals/resolve-storage-001-upsert-missing-update-policy/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/resolve-storage-001-upsert-missing-update-policy.json" + }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", @@ -7135,6 +7976,83 @@ "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/investigate-auth-001-deleted-user-access.json" }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "investigate-functions-001-546-resource-limit", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "identified video-thumbnails and the 546 resource-limit pattern", + "passed": true, + "judgeNotes": "Identified `video-thumbnails` as the affected function and correctly described failures as HTTP 546 WORKER_RESOURCE_LIMIT/CPUTime resource-limit responses, while noting the welcome-email 500 as unrelated." + }, + { + "name": "attributed the 546s to CPU time exhaustion", + "passed": true, + "judgeNotes": "The assistant specifically attributes the 546 failures to CPU time exhaustion, citing log evidence including shutdown reason CPUTime and cpu_time_used/cpu_time_limit at 2000ms. It distinguishes this from wall-clock/network timeout and notes the welcome-email error is unrelated." + }, + { + "name": "recommended reducing/offloading CPU work as the fix", + "passed": true, + "judgeNotes": "The assistant correctly identifies CPUTime shutdown as the cause and recommends reducing/offloading CPU-intensive thumbnail work: optimizing decode/seek/downscale, moving heavy video processing to background workers or external services, and capping input size. Although it also mentions retry logic, it is not the only recommendation and the primary fix reduces/offloads CPU usage." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge function CPU time limit exceeded shutdown\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-wall-clock-time-limit-reached-Nk38bW", + "title": "Edge Function 'wall clock time limit reached'" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-cpu-limits", + "title": "Understanding Edge Function CPU limits" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-546-error-response", + "title": "546 - WORKER_RESOURCE_LIMIT Exceeded / WORKER_LIMIT Exceeded" + }, + { + "url": "https://supabase.com/docs/guides/functions/status-codes", + "title": "Status codes" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-shutdown-reasons-explained", + "title": "Edge Function shutdown reasons explained" + } + ], + "resultChars": 31353 + } + ] + }, + "prompt": "Our `video-thumbnails` edge function has been failing intermittently since this morning. It generates a thumbnail from a user-uploaded video, and about half the calls are erroring out.\n\nCan you investigate the project logs and tell me what's going on and what we should do about it?", + "promptSourcePath": "evals/investigate-functions-001-546-resource-limit/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/investigate-functions-001-546-resource-limit.json" + }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", @@ -7349,32 +8267,113 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "resolve-database-001-migration-history-mismatch", + "eval": "resolve-dataapi-002-update-zero-rows-affected", "stage": "resolve", "product": [ + "data-api", "database" ], "topic": [ - "migrations" + "rls" ], "suite": "benchmark", - "interface": "cli", "passed": true, "checks": [ { - "name": "the avatar_url column is applied on the hosted profiles table", + "name": "RLS still enabled on tasks", "passed": true }, { - "name": "migration 20240220000000 is recorded in the remote history", - "passed": true + "name": "user A's update actually checks off their own task", + "passed": true, + "notes": "saw: [{\"id\":\"dc442e02-31fa-4fc8-b8f7-3679b7f03176\",\"is_done\":true}]" }, { - "name": "remote migration history matches local migration files", + "name": "user B cannot update user A's task", "passed": true }, { - "name": "local migrations are a valid reconciled sequence", + "name": "diagnosed the missing USING clause and added it", + "passed": true, + "judgeNotes": "Diagnosed missing UPDATE USING clause, fixed policy with USING (user_id = auth.uid()) and retained WITH CHECK for authenticated users without disabling RLS." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"row level security policy UPDATE USING WITH CHECK default missing\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/rls-simplified-BJTcS8", + "title": "RLS Simplified" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-react", + "title": "Build a User Management App with Ionic React" + }, + { + "url": "https://supabase.com/docs/guides/auth/enterprise-sso/auth-sso-saml", + "title": "Single Sign-On with SAML 2.0 for Projects" + } + ], + "resultChars": 107724 + } + ] + }, + "prompt": "Our app lets signed-in users manage a personal `tasks` list. Users can create tasks and check them off (`is_done`).\n\nCreating a task works fine, and I can see the row in the table. But when a user checks off a task, the app's update call succeeds with no error, yet `is_done` never actually changes, and the API doesn't return the updated row either.\n\nFind out why the update has no effect and fix it.", + "promptSourcePath": "evals/resolve-dataapi-002-update-zero-rows-affected/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/resolve-dataapi-002-update-zero-rows-affected.json" + }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "resolve-database-001-migration-history-mismatch", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "the avatar_url column is applied on the hosted profiles table", + "passed": true + }, + { + "name": "migration 20240220000000 is recorded in the remote history", + "passed": true + }, + { + "name": "remote migration history matches local migration files", + "passed": true + }, + { + "name": "local migrations are a valid reconciled sequence", "passed": true }, { @@ -7455,6 +8454,86 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-performance-001-slow-query-cpu-spike.json" }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "resolve-performance-002-rls-auth-initplan", + "stage": "resolve", + "product": [ + "database", + "data-api" + ], + "topic": [ + "rls", + "sql" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS still enabled on documents", + "passed": true + }, + { + "name": "user A still reads exactly their own documents", + "passed": true, + "notes": "saw 2 rows" + }, + { + "name": "user B cannot read user A's documents", + "passed": true + }, + { + "name": "SELECT policy evaluates auth.uid() as an InitPlan (wrapped in a subquery)", + "passed": true, + "notes": "(( SELECT auth.uid() AS uid) = user_id)" + }, + { + "name": "diagnosed per-row auth.uid() re-evaluation and applied the InitPlan fix", + "passed": true, + "judgeNotes": "The assistant correctly identified per-row auth.uid() evaluation in RLS as the cause, fixed policies by wrapping auth.uid() in a scalar subquery to create an InitPlan, preserved owner scoping and authenticated role, and kept RLS/security intact." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"RLS performance auth.uid() wrap select initplan\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/rls-performance-and-best-practices-Z5Jjwv", + "title": "RLS Performance and Best Practices" + }, + { + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0003_auth_rls_initplan", + "title": "Database Advisor: Lint 0003_auth_rls_initplan" + } + ], + "resultChars": 41842 + } + ] + }, + "prompt": "Our `documents` table has Row Level Security enabled so each signed-in user only sees their own rows. It worked fine in testing, but now that the table has grown the document list has become painfully slow — a simple \"list my documents\" query that should be instant now takes seconds and gets slower as the table grows, even though each user only owns a handful of rows.\n\nWe don't want to weaken the security model — users must still only ever see their own documents. Find out why the query is so slow and fix it.", + "promptSourcePath": "evals/resolve-performance-002-rls-auth-initplan/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/resolve-performance-002-rls-auth-initplan.json" + }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", @@ -7527,6 +8606,71 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "resolve-storage-001-upsert-missing-update-policy", + "stage": "resolve", + "product": [ + "storage", + "database" + ], + "topic": [ + "rls" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "bucket avatars exists", + "passed": true + }, + { + "name": "bucket avatars stays public", + "passed": true, + "notes": "the bucket being public is intentional (avatars need a public URL); it is not the bug" + }, + { + "name": "RLS still enabled on storage.objects", + "passed": true + }, + { + "name": "anon can still read the public avatar", + "passed": true + }, + { + "name": "user A can replace their own avatar via upsert", + "passed": true, + "notes": "saw: [{\"name\":\"019ff5ed-975e-738b-9b80-de4fd834abcf/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" + }, + { + "name": "user B cannot overwrite user A's avatar", + "passed": true + }, + { + "name": "added an owner-scoped UPDATE policy without weakening public reads", + "passed": true, + "judgeNotes": "Diagnosed missing storage.objects UPDATE policy for upsert replacements, noted public bucket only affects reads, and added authenticated owner-scoped UPDATE policy with USING and WITH CHECK while keeping public read/RLS intact." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app has a public `avatars` bucket so profile photos have a public URL. Each user's avatar is stored at `/avatar.png`, and the app uploads it with `upsert: true` so a new photo replaces the old one at that same path.\n\nThe very first upload for a user always works, but replacing an existing avatar fails. Find out why and fix it.", + "promptSourcePath": "evals/resolve-storage-001-upsert-missing-update-policy/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/resolve-storage-001-upsert-missing-update-policy.json" + }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", @@ -9913,44 +11057,31 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "investigate-realtime-001-subscribed-no-events", + "eval": "investigate-functions-001-546-resource-limit", "stage": "investigate", "product": [ - "realtime", - "database" + "edge-functions" ], "topic": [ - "sdk" + "observability" ], "suite": "benchmark", - "interface": "mcp", "passed": true, "checks": [ { - "name": "orders table added to supabase_realtime publication", - "passed": true - }, - { - "name": "courier_locations still in supabase_realtime publication", - "passed": true - }, - { - "name": "publication still publishes INSERT events", - "passed": true - }, - { - "name": "RLS still enabled on orders", - "passed": true + "name": "identified video-thumbnails and the 546 resource-limit pattern", + "passed": true, + "judgeNotes": "Identified `video-thumbnails` as the affected function and correctly recognized the failures as HTTP 546 resource-limit/CPU limit responses, not 500s or 503s." }, { - "name": "staff can still read orders through RLS", + "name": "attributed the 546s to CPU time exhaustion", "passed": true, - "notes": "authenticated sees 2 of 2 orders" + "judgeNotes": "The assistant specifically attributed the 546 failures to CPU time exhaustion, citing the shutdown reason `CPUTime` and `cpu_time_used: 2000ms` / `cpu_time_limit: 2000ms` as log evidence." }, { - "name": "diagnosed missing publication membership", + "name": "recommended reducing/offloading CPU work as the fix", "passed": true, - "judgeNotes": "Identified the root cause as orders missing from supabase_realtime despite successful subscription, fixed by ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, and preserved courier_locations plus RLS/policies." + "judgeNotes": "The assistant correctly identifies CPUTime limit and recommends reducing/offloading CPU-intensive thumbnail generation via asynchronous worker/background job, reducing work per invocation, avoiding full scans, and caching. It does not rely on retrying or increasing limits." } ], "skills": { @@ -9967,55 +11098,179 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Realtime postgres_changes publication supabase_realtime table not receiving events\", limit: 5) { nodes { __typename title href content ... on Guide { subsections { nodes { title href content } } } } } }", + "query": "query { searchDocs(query: \"monitoring and debugging edge functions logs\", limit: 5) { nodes { __typename title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", - "title": "Subscribing to Database Changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#broadcast-authorization", - "title": "Broadcast authorization" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#streaming-updates", - "title": "Streaming updates" + "url": "https://supabase.com/docs/guides/functions/logging", + "title": "Logging" }, { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#streaming-inserts", - "title": "Streaming inserts" + "url": "https://supabase.com/docs/guides/functions/debugging-tools", + "title": "Local Debugging" }, { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#enable-postgres-changes", - "title": "Enable Postgres Changes" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/debugging", + "title": "Debugging guide" }, { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#using-postgres-changes", - "title": "Using Postgres Changes" + "url": "https://supabase.com/docs/guides/functions/unit-test", + "title": "Testing your Edge Functions" }, { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#listening-on-client-side", - "title": "Listening on client side" - }, + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-546-error-response", + "title": "546 - WORKER_RESOURCE_LIMIT Exceeded / WORKER_LIMIT Exceeded" + } + ], + "resultChars": 43128 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Function 546 CPUTime optimize heavy processing video thumbnail\", limit: 5) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#create-a-trigger", - "title": "Create a trigger" + "url": "https://supabase.com/docs/guides/storage/vector/storing-vectors", + "title": "Storing Vectors" }, { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#create-a-trigger-function", - "title": "Create a trigger function" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#using-broadcast", - "title": "Using Broadcast" + "url": "https://supabase.com/docs/guides/functions/storage-caching", + "title": "Integrating with Supabase Storage" }, { - "url": "https://supabase.com/docs/guides/realtime/benchmarks", - "title": "Benchmarks" + "url": "https://supabase.com/docs/guides/functions/wasm", + "title": "Using Wasm modules" }, { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#methodology", + "url": "https://supabase.com/docs/guides/functions/recursive-functions", + "title": "Recursive / Nested Function Calls" + } + ], + "resultChars": 50150 + } + ] + }, + "prompt": "Our `video-thumbnails` edge function has been failing intermittently since this morning. It generates a thumbnail from a user-uploaded video, and about half the calls are erroring out.\n\nCan you investigate the project logs and tell me what's going on and what we should do about it?", + "promptSourcePath": "evals/investigate-functions-001-546-resource-limit/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini/investigate-functions-001-546-resource-limit.json" + }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "investigate-realtime-001-subscribed-no-events", + "stage": "investigate", + "product": [ + "realtime", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "orders table added to supabase_realtime publication", + "passed": true + }, + { + "name": "courier_locations still in supabase_realtime publication", + "passed": true + }, + { + "name": "publication still publishes INSERT events", + "passed": true + }, + { + "name": "RLS still enabled on orders", + "passed": true + }, + { + "name": "staff can still read orders through RLS", + "passed": true, + "notes": "authenticated sees 2 of 2 orders" + }, + { + "name": "diagnosed missing publication membership", + "passed": true, + "judgeNotes": "Identified the root cause as orders missing from supabase_realtime despite successful subscription, fixed by ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, and preserved courier_locations plus RLS/policies." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase Realtime postgres_changes publication supabase_realtime table not receiving events\", limit: 5) { nodes { __typename title href content ... on Guide { subsections { nodes { title href content } } } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", + "title": "Subscribing to Database Changes" + }, + { + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#broadcast-authorization", + "title": "Broadcast authorization" + }, + { + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#streaming-updates", + "title": "Streaming updates" + }, + { + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#streaming-inserts", + "title": "Streaming inserts" + }, + { + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#enable-postgres-changes", + "title": "Enable Postgres Changes" + }, + { + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#using-postgres-changes", + "title": "Using Postgres Changes" + }, + { + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#listening-on-client-side", + "title": "Listening on client side" + }, + { + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#create-a-trigger", + "title": "Create a trigger" + }, + { + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#create-a-trigger-function", + "title": "Create a trigger function" + }, + { + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#using-broadcast", + "title": "Using Broadcast" + }, + { + "url": "https://supabase.com/docs/guides/realtime/benchmarks", + "title": "Benchmarks" + }, + { + "url": "https://supabase.com/docs/guides/realtime/benchmarks#methodology", "title": "Methodology" }, { @@ -10575,6 +11830,152 @@ "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/resolve-dataapi-001-empty-results.json" }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "resolve-dataapi-002-update-zero-rows-affected", + "stage": "resolve", + "product": [ + "data-api", + "database" + ], + "topic": [ + "rls" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "RLS still enabled on tasks", + "passed": true + }, + { + "name": "user A's update actually checks off their own task", + "passed": true, + "notes": "saw: [{\"id\":\"6e0d06e4-d7b5-4c3f-80aa-ae655c1cd8db\",\"is_done\":true}]" + }, + { + "name": "user B cannot update user A's task", + "passed": true + }, + { + "name": "diagnosed the missing USING clause and added it", + "passed": true, + "judgeNotes": "The assistant correctly identified the UPDATE policy had only WITH CHECK and no USING, explained that UPDATE needs USING to find/qualify the existing row causing silent 0-row updates, and fixed it with a user-owned USING clause while preserving WITH CHECK, scoped to authenticated/user_id with RLS still enabled." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,140p'\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 7789 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and debugging Supabase RLS update no rows\", limit: 3) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/overview", + "title": "Testing Overview" + }, + { + "url": "https://supabase.com/docs/guides/database/testing", + "title": "Testing Your Database" + } + ], + "resultChars": 64670 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"UPDATE requires SELECT policy RLS Supabase\", limit: 5) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", + "title": "Storage Helper Functions" + } + ], + "resultChars": 71556 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase Postgres RLS empty result no rows\", limit: 5) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-select-returning-an-empty-data-array-and-i-have-data-in-the-table-xvOPgx", + "title": "Why is my select returning an empty data array and I have data in the table?" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/debugging", + "title": "Debugging guide" + }, + { + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" + }, + { + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0008_rls_enabled_no_policy", + "title": "Database Advisor: Lint 0008_rls_enabled_no_policy" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" + } + ], + "resultChars": 42939 + } + ] + }, + "prompt": "Our app lets signed-in users manage a personal `tasks` list. Users can create tasks and check them off (`is_done`).\n\nCreating a task works fine, and I can see the row in the table. But when a user checks off a task, the app's update call succeeds with no error, yet `is_done` never actually changes, and the API doesn't return the updated row either.\n\nFind out why the update has no effect and fix it.", + "promptSourcePath": "evals/resolve-dataapi-002-update-zero-rows-affected/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini/resolve-dataapi-002-update-zero-rows-affected.json" + }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", @@ -10732,27 +12133,149 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "resolve-security-002-rls-cross-tenant-leak", + "eval": "resolve-performance-002-rls-auth-initplan", "stage": "resolve", "product": [ "database", - "auth" + "data-api" ], "topic": [ "rls", - "security" + "sql" ], "suite": "benchmark", "interface": "mcp", "passed": true, "checks": [ { - "name": "RLS enabled on notes", + "name": "RLS still enabled on documents", "passed": true }, { - "name": "tenant A sees only org A notes", - "passed": true + "name": "user A still reads exactly their own documents", + "passed": true, + "notes": "saw 2 rows" + }, + { + "name": "user B cannot read user A's documents", + "passed": true + }, + { + "name": "SELECT policy evaluates auth.uid() as an InitPlan (wrapped in a subquery)", + "passed": true, + "notes": "(( SELECT auth.uid() AS uid) = user_id)" + }, + { + "name": "diagnosed per-row auth.uid() re-evaluation and applied the InitPlan fix", + "passed": true, + "judgeNotes": "The assistant correctly identified the direct auth.uid() RLS predicate as the bottleneck due to per-row evaluation, changed the policies to ((select auth.uid()) = user_id), verified an InitPlan and index usage, and kept RLS scoped to authenticated row owners." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Row Level Security performance auth.uid index select policy\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on CLICommandReference { title href content } } totalCount } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0003_auth_rls_initplan", + "title": "Database Advisor: Lint 0003_auth_rls_initplan" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", + "title": "Column Level Security" + } + ], + "resultChars": 72267 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging slow query explain analyze missing indexes\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } totalCount } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, + { + "url": "https://supabase.com/docs/guides/database/database-advisors", + "title": "Performance and Security Advisors" + }, + { + "url": "https://supabase.com/docs/guides/database/inspect", + "title": "Debugging and monitoring" + } + ], + "resultChars": 74452 + } + ] + }, + "prompt": "Our `documents` table has Row Level Security enabled so each signed-in user only sees their own rows. It worked fine in testing, but now that the table has grown the document list has become painfully slow — a simple \"list my documents\" query that should be instant now takes seconds and gets slower as the table grows, even though each user only owns a handful of rows.\n\nWe don't want to weaken the security model — users must still only ever see their own documents. Find out why the query is so slow and fix it.", + "promptSourcePath": "evals/resolve-performance-002-rls-auth-initplan/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini/resolve-performance-002-rls-auth-initplan.json" + }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "resolve-security-002-rls-cross-tenant-leak", + "stage": "resolve", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS enabled on notes", + "passed": true + }, + { + "name": "tenant A sees only org A notes", + "passed": true }, { "name": "tenant B cannot read org A notes", @@ -10830,6 +12353,122 @@ "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "resolve-storage-001-upsert-missing-update-policy", + "stage": "resolve", + "product": [ + "storage", + "database" + ], + "topic": [ + "rls" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "bucket avatars exists", + "passed": true + }, + { + "name": "bucket avatars stays public", + "passed": true, + "notes": "the bucket being public is intentional (avatars need a public URL); it is not the bug" + }, + { + "name": "RLS still enabled on storage.objects", + "passed": true + }, + { + "name": "anon can still read the public avatar", + "passed": true + }, + { + "name": "user A can replace their own avatar via upsert", + "passed": true, + "notes": "saw: [{\"name\":\"019ff5ed-fe38-7411-a832-b4aa09ff87f8/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" + }, + { + "name": "user B cannot overwrite user A's avatar", + "passed": true + }, + { + "name": "added an owner-scoped UPDATE policy without weakening public reads", + "passed": true, + "judgeNotes": "The assistant correctly diagnosed missing UPDATE RLS policy for storage.objects during avatar upsert/replacement, noted existing SELECT/INSERT and public bucket, added an authenticated owner-scoped UPDATE policy with USING and WITH CHECK using owner_id = auth.uid()::text, and did not weaken public-read setup or disable RLS." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"storage upsert requires insert select update policy\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", + "title": "Storage Helper Functions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter", + "title": "Build a User Management App with Flutter" + }, + { + "url": "https://supabase.com/docs/guides/storage/management/copy-move-objects", + "title": "Copy Objects" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" + } + ], + "resultChars": 77961 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"storage update policy owner avatar upsert select update owner_id owner\", limit: 10) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/security/ownership", + "title": "Ownership" + }, + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + } + ], + "resultChars": 6639 + } + ] + }, + "prompt": "Our app has a public `avatars` bucket so profile photos have a public URL. Each user's avatar is stored at `/avatar.png`, and the app uploads it with `upsert: true` so a new photo replaces the old one at that same path.\n\nThe very first upload for a user always works, but replacing an existing avatar fails. Find out why and fix it.", + "promptSourcePath": "evals/resolve-storage-001-upsert-missing-update-policy/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini/resolve-storage-001-upsert-missing-update-policy.json" + }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", @@ -12442,6 +14081,54 @@ "attempts": 2, "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-auth-001-deleted-user-access.json" }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "investigate-functions-001-546-resource-limit", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "identified video-thumbnails and the 546 resource-limit pattern", + "passed": true, + "judgeNotes": "Identified video-thumbnails as the affected function and correctly stated failures return HTTP 546 due to CPU/resource limit, while noting welcome-email 500 is unrelated." + }, + { + "name": "attributed the 546s to CPU time exhaustion", + "passed": true, + "judgeNotes": "The assistant specifically attributes the 546 shutdowns to CPU time exhaustion, citing log evidence including `shutdown (reason: CPUTime, cpu_time_used: 2000ms, cpu_time_limit: 2000ms)`, and distinguishes it from the unrelated welcome-email error." + }, + { + "name": "recommended reducing/offloading CPU work as the fix", + "passed": true, + "judgeNotes": "The assistant identified CPU limit shutdowns and recommended moving expensive thumbnail work out of the edge function, reducing per-request work, using smaller/lower-resolution inputs, and making processing async via a background worker. These directly reduce/offload CPU-intensive work." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our `video-thumbnails` edge function has been failing intermittently since this morning. It generates a thumbnail from a user-uploaded video, and about half the calls are erroring out.\n\nCan you investigate the project logs and tell me what's going on and what we should do about it?", + "promptSourcePath": "evals/investigate-functions-001-546-resource-limit/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-functions-001-546-resource-limit.json" + }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", @@ -12627,33 +14314,85 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "resolve-database-001-migration-history-mismatch", + "eval": "resolve-dataapi-002-update-zero-rows-affected", "stage": "resolve", "product": [ + "data-api", "database" ], "topic": [ - "migrations" + "rls" ], "suite": "benchmark", - "interface": "cli", "passed": true, "checks": [ { - "name": "the avatar_url column is applied on the hosted profiles table", + "name": "RLS still enabled on tasks", "passed": true }, { - "name": "migration 20240220000000 is recorded in the remote history", - "passed": true + "name": "user A's update actually checks off their own task", + "passed": true, + "notes": "saw: [{\"id\":\"454d55fa-f3bf-476e-84a5-5c2815fd24dd\",\"is_done\":true}]" }, { - "name": "remote migration history matches local migration files", + "name": "user B cannot update user A's task", "passed": true }, { - "name": "local migrations are a valid reconciled sequence", - "passed": true + "name": "diagnosed the missing USING clause and added it", + "passed": true, + "judgeNotes": "Diagnosed the missing USING clause on the authenticated UPDATE policy, kept WITH CHECK scoped to user_id = auth.uid(), left RLS enabled, and applied the correct policy fix." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app lets signed-in users manage a personal `tasks` list. Users can create tasks and check them off (`is_done`).\n\nCreating a task works fine, and I can see the row in the table. But when a user checks off a task, the app's update call succeeds with no error, yet `is_done` never actually changes, and the API doesn't return the updated row either.\n\nFind out why the update has no effect and fix it.", + "promptSourcePath": "evals/resolve-dataapi-002-update-zero-rows-affected/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-dataapi-002-update-zero-rows-affected.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "resolve-database-001-migration-history-mismatch", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "the avatar_url column is applied on the hosted profiles table", + "passed": true + }, + { + "name": "migration 20240220000000 is recorded in the remote history", + "passed": true + }, + { + "name": "remote migration history matches local migration files", + "passed": true + }, + { + "name": "local migrations are a valid reconciled sequence", + "passed": true }, { "name": "production profile data is intact (not reset)", @@ -12733,6 +14472,65 @@ "attempts": 1, "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-performance-001-slow-query-cpu-spike.json" }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "resolve-performance-002-rls-auth-initplan", + "stage": "resolve", + "product": [ + "database", + "data-api" + ], + "topic": [ + "rls", + "sql" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": false, + "checks": [ + { + "name": "RLS still enabled on documents", + "passed": true + }, + { + "name": "user A still reads exactly their own documents", + "passed": true, + "notes": "saw 2 rows" + }, + { + "name": "user B cannot read user A's documents", + "passed": true + }, + { + "name": "SELECT policy evaluates auth.uid() as an InitPlan (wrapped in a subquery)", + "passed": false, + "notes": "(user_id = auth.uid())" + }, + { + "name": "diagnosed per-row auth.uid() re-evaluation and applied the InitPlan fix", + "passed": false, + "judgeNotes": "The assistant did not identify or fix the per-row auth.uid()/current_setting evaluation in the RLS policy. It instead added a composite/covering index and left the policy as user_id = auth.uid(), so it fails the rubric." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our `documents` table has Row Level Security enabled so each signed-in user only sees their own rows. It worked fine in testing, but now that the table has grown the document list has become painfully slow — a simple \"list my documents\" query that should be instant now takes seconds and gets slower as the table grows, even though each user only owns a handful of rows.\n\nWe don't want to weaken the security model — users must still only ever see their own documents. Find out why the query is so slow and fix it.", + "promptSourcePath": "evals/resolve-performance-002-rls-auth-initplan/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-performance-002-rls-auth-initplan.json" + }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", @@ -12805,6 +14603,71 @@ "attempts": 1, "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "resolve-storage-001-upsert-missing-update-policy", + "stage": "resolve", + "product": [ + "storage", + "database" + ], + "topic": [ + "rls" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "bucket avatars exists", + "passed": true + }, + { + "name": "bucket avatars stays public", + "passed": true, + "notes": "the bucket being public is intentional (avatars need a public URL); it is not the bug" + }, + { + "name": "RLS still enabled on storage.objects", + "passed": true + }, + { + "name": "anon can still read the public avatar", + "passed": true + }, + { + "name": "user A can replace their own avatar via upsert", + "passed": true, + "notes": "saw: [{\"name\":\"019ff5ed-885e-7640-b495-fabe292580ca/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" + }, + { + "name": "user B cannot overwrite user A's avatar", + "passed": true + }, + { + "name": "added an owner-scoped UPDATE policy without weakening public reads", + "passed": true, + "judgeNotes": "Diagnosed missing UPDATE RLS policy for upsert replacement, explained public bucket only covers reads, added authenticated owner-scoped UPDATE policy with USING and WITH CHECK, and kept public read/RLS setup intact." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app has a public `avatars` bucket so profile photos have a public URL. Each user's avatar is stored at `/avatar.png`, and the app uploads it with `upsert: true` so a new photo replaces the old one at that same path.\n\nThe very first upload for a user always works, but replacing an existing avatar fails. Find out why and fix it.", + "promptSourcePath": "evals/resolve-storage-001-upsert-missing-update-policy/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-storage-001-upsert-missing-update-policy.json" + }, { "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", @@ -14922,6 +16785,139 @@ "attempts": 1, "sourcePath": "codex-gpt-5.6/investigate-auth-001-deleted-user-access.json" }, + { + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "investigate-functions-001-546-resource-limit", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "identified video-thumbnails and the 546 resource-limit pattern", + "passed": true, + "judgeNotes": "Identified video-thumbnails as the affected function and correctly described failures as HTTP 546 due to CPU/resource limit, not 500/503." + }, + { + "name": "attributed the 546s to CPU time exhaustion", + "passed": true, + "judgeNotes": "The assistant specifically attributes the 546 failures to CPU time exhaustion, citing `shutdown (reason: CPUTime, cpu_time_used: 2000ms, cpu_time_limit: 2000ms)` as log evidence." + }, + { + "name": "recommended reducing/offloading CPU work as the fix", + "passed": true, + "judgeNotes": "The assistant clearly identifies CPU exhaustion and recommends reducing/offloading CPU-intensive thumbnail processing: moving decoding/frame extraction to a worker or external service, optimizing extraction, avoiding full decoding/buffering, and imposing limits. It also correctly says retries won’t help." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions CPUTime shutdown 546 CPU time limit video processing background tasks\", limit: 8) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/limits", + "title": "Limits" + }, + { + "url": "https://supabase.com/docs/guides/functions/background-tasks", + "title": "Background Tasks" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-cpu-limits", + "title": "Understanding Edge Function CPU limits" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-monitoring-resource-usage", + "title": "Monitoring Edge Function resource usage" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-shutdown-reasons-explained", + "title": "Edge Function shutdown reasons explained" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-takes-too-long-to-respond", + "title": "Edge Function takes too long to respond" + }, + { + "url": "https://supabase.com/docs/guides/functions/recursive-functions", + "title": "Recursive / Nested Function Calls" + } + ], + "resultChars": 47378 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging Edge Functions logs shutdown reasons CPU time\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-shutdown-reasons-explained", + "title": "Edge Function shutdown reasons explained" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-546-error-response", + "title": "546 - WORKER_RESOURCE_LIMIT Exceeded / WORKER_LIMIT Exceeded" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-takes-too-long-to-respond", + "title": "Edge Function takes too long to respond" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-monitoring-resource-usage", + "title": "Monitoring Edge Function resource usage" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/debugging", + "title": "Debugging guide" + } + ], + "resultChars": 45089 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'edge function|cpu|limit|breaking-change' | head -80\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 6135 + } + ] + }, + "prompt": "Our `video-thumbnails` edge function has been failing intermittently since this morning. It generates a thumbnail from a user-uploaded video, and about half the calls are erroring out.\n\nCan you investigate the project logs and tell me what's going on and what we should do about it?", + "promptSourcePath": "evals/investigate-functions-001-546-resource-limit/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6/investigate-functions-001-546-resource-limit.json" + }, { "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", @@ -15275,33 +17271,134 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "resolve-database-001-migration-history-mismatch", + "eval": "resolve-dataapi-002-update-zero-rows-affected", "stage": "resolve", "product": [ + "data-api", "database" ], "topic": [ - "migrations" + "rls" ], "suite": "benchmark", - "interface": "cli", "passed": true, "checks": [ { - "name": "the avatar_url column is applied on the hosted profiles table", + "name": "RLS still enabled on tasks", "passed": true }, { - "name": "migration 20240220000000 is recorded in the remote history", - "passed": true + "name": "user A's update actually checks off their own task", + "passed": true, + "notes": "saw: [{\"id\":\"c4d3551f-6cd7-458b-be4c-51a5f191129f\",\"is_done\":true}]" }, { - "name": "remote migration history matches local migration files", + "name": "user B cannot update user A's task", "passed": true }, { - "name": "local migrations are a valid reconciled sequence", - "passed": true + "name": "diagnosed the missing USING clause and added it", + "passed": true, + "judgeNotes": "The assistant correctly identified the missing USING clause on the tasks UPDATE policy as the cause of silent zero-row updates, added an owner-scoped USING condition while preserving WITH CHECK, kept the policy scoped to authenticated users and user_id, and left RLS enabled." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "web_search", + "query": "site:supabase.com/changelog.md Supabase changelog breaking change RLS update SELECT policy", + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"debugging UPDATE affects 0 rows RLS corresponding SELECT policy\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/debugging", + "title": "Debugging guide" + }, + { + "url": "https://supabase.com/docs/guides/storage/production/scaling", + "title": "Storage Optimizations" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", + "title": "Column Level Security" + }, + { + "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", + "title": "Declarative database schemas" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + } + ], + "resultChars": 69599 + } + ] + }, + "prompt": "Our app lets signed-in users manage a personal `tasks` list. Users can create tasks and check them off (`is_done`).\n\nCreating a task works fine, and I can see the row in the table. But when a user checks off a task, the app's update call succeeds with no error, yet `is_done` never actually changes, and the API doesn't return the updated row either.\n\nFind out why the update has no effect and fix it.", + "promptSourcePath": "evals/resolve-dataapi-002-update-zero-rows-affected/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6/resolve-dataapi-002-update-zero-rows-affected.json" + }, + { + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "resolve-database-001-migration-history-mismatch", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "the avatar_url column is applied on the hosted profiles table", + "passed": true + }, + { + "name": "migration 20240220000000 is recorded in the remote history", + "passed": true + }, + { + "name": "remote migration history matches local migration files", + "passed": true + }, + { + "name": "local migrations are a valid reconciled sequence", + "passed": true }, { "name": "production profile data is intact (not reset)", @@ -15504,6 +17601,114 @@ "attempts": 1, "sourcePath": "codex-gpt-5.6/resolve-performance-001-slow-query-cpu-spike.json" }, + { + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "resolve-performance-002-rls-auth-initplan", + "stage": "resolve", + "product": [ + "database", + "data-api" + ], + "topic": [ + "rls", + "sql" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS still enabled on documents", + "passed": true + }, + { + "name": "user A still reads exactly their own documents", + "passed": true, + "notes": "saw 2 rows" + }, + { + "name": "user B cannot read user A's documents", + "passed": true + }, + { + "name": "SELECT policy evaluates auth.uid() as an InitPlan (wrapped in a subquery)", + "passed": true, + "notes": "(( SELECT auth.uid() AS uid) = user_id)" + }, + { + "name": "diagnosed per-row auth.uid() re-evaluation and applied the InitPlan fix", + "passed": true, + "judgeNotes": "The assistant correctly diagnosed direct auth.uid() in the RLS ownership predicate as a per-row cost, rewrote policies to use (select auth.uid()) = user_id so Postgres uses an InitPlan once per query, kept policies scoped to authenticated owners, and kept RLS intact." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"row level security performance recommendations auth.uid select index policy column\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0003_auth_rls_initplan", + "title": "Database Advisor: Lint 0003_auth_rls_initplan" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", + "title": "Column Level Security" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/database/database-advisors", + "title": "Performance and Security Advisors" + } + ], + "resultChars": 50741 + }, + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "web_search", + "query": "site:supabase.com/changelog RLS performance auth.uid index Supabase", + "pages": [] + } + ] + }, + "prompt": "Our `documents` table has Row Level Security enabled so each signed-in user only sees their own rows. It worked fine in testing, but now that the table has grown the document list has become painfully slow — a simple \"list my documents\" query that should be instant now takes seconds and gets slower as the table grows, even though each user only owns a handful of rows.\n\nWe don't want to weaken the security model — users must still only ever see their own documents. Find out why the query is so slow and fix it.", + "promptSourcePath": "evals/resolve-performance-002-rls-auth-initplan/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6/resolve-performance-002-rls-auth-initplan.json" + }, { "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", @@ -15621,6 +17826,121 @@ "attempts": 1, "sourcePath": "codex-gpt-5.6/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "resolve-storage-001-upsert-missing-update-policy", + "stage": "resolve", + "product": [ + "storage", + "database" + ], + "topic": [ + "rls" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "bucket avatars exists", + "passed": true + }, + { + "name": "bucket avatars stays public", + "passed": true, + "notes": "the bucket being public is intentional (avatars need a public URL); it is not the bug" + }, + { + "name": "RLS still enabled on storage.objects", + "passed": true + }, + { + "name": "anon can still read the public avatar", + "passed": true + }, + { + "name": "user A can replace their own avatar via upsert", + "passed": true, + "notes": "saw: [{\"name\":\"019ff5ee-09a7-7623-bdd8-111df66fb6fd/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" + }, + { + "name": "user B cannot overwrite user A's avatar", + "passed": true + }, + { + "name": "added an owner-scoped UPDATE policy without weakening public reads", + "passed": true, + "judgeNotes": "Diagnosed missing UPDATE RLS policy for upsert replacement, explained public bucket only affects read/download, added authenticated owner-scoped UPDATE policy with USING and WITH CHECK, and kept bucket public/RLS intact." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Storage upload upsert existing object RLS SELECT UPDATE INSERT policy requirements troubleshooting\", limit: 6) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/storage-error-403-forbidden-new-row-violates-row-level-security-policy-on-upload-a94384", + "title": "Storage error: 403 Forbidden: 'new row violates row-level security policy' on upload" + }, + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" + }, + { + "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", + "title": "Storage Helper Functions" + }, + { + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" + }, + { + "url": "https://supabase.com/docs/guides/storage/schema/design", + "title": "The Storage Schema" + } + ], + "resultChars": 24619 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'storage|rls|upsert|breaking-change' | head -n 120\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 4460 + } + ] + }, + "prompt": "Our app has a public `avatars` bucket so profile photos have a public URL. Each user's avatar is stored at `/avatar.png`, and the app uploads it with `upsert: true` so a new photo replaces the old one at that same path.\n\nThe very first upload for a user always works, but replacing an existing avatar fails. Find out why and fix it.", + "promptSourcePath": "evals/resolve-storage-001-upsert-missing-update-policy/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6/resolve-storage-001-upsert-missing-update-policy.json" + }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", @@ -18382,75 +20702,172 @@ }, { "source": "search_docs", - "query": "query { searchDocs(query: \"sign out JWT remains valid until expires revoke refresh tokens access token\", limit: 8) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"sign out JWT remains valid until expires revoke refresh tokens access token\", limit: 8) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", + "title": "Should I set a shorter Max-Age parameter on the cookies?" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signout" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/server-side/advanced-guide", + "title": "Advanced guide" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", + "title": "OAuth 2.1 Flows" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/how-do-i-make-the-cookies-httponly-vwweFx", + "title": "How do I make the cookies HttpOnly?" + } + ], + "resultChars": 103031 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"admin deleteUser delete user existing JWT access token valid until expiry\", limit: 6) { nodes { ... on ClientLibraryFunctionReference { title href content language methodName } ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + }, + { + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + }, + { + "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" + }, + { + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" + } + ], + "resultChars": 33037 + } + ] + }, + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6-no-skills/investigate-auth-001-deleted-user-access.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "investigate-functions-001-546-resource-limit", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "identified video-thumbnails and the 546 resource-limit pattern", + "passed": true, + "judgeNotes": "Identified `video-thumbnails` as the affected function and correctly recognized failures as HTTP 546 due to CPU/resource limits, not 500/503 server errors." + }, + { + "name": "attributed the 546s to CPU time exhaustion", + "passed": true, + "judgeNotes": "The assistant specifically attributed status 546 failures to CPU time exhaustion and grounded it in log evidence: shutdown reason CPUTime with cpu_time_used 2000ms matching cpu_time_limit 2000ms." + }, + { + "name": "recommended reducing/offloading CPU work as the fix", + "passed": true, + "judgeNotes": "The assistant identified the CPU limit issue and recommended reducing/offloading CPU-intensive thumbnail work via queue/background processing on external compute, plus optimizations like seeking, extracting one frame, downscaling early, and enforcing input limits." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions CPUTime status 546 CPU time limit background tasks image video processing limits\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", - "title": "Should I set a shorter Max-Age parameter on the cookies?" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/functions/limits", + "title": "Limits" }, { - "url": "https://supabase.com/docs/guides/auth/server-side/advanced-guide", - "title": "Advanced guide" + "url": "https://supabase.com/docs/guides/functions/background-tasks", + "title": "Background Tasks" }, { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-cpu-limits", + "title": "Understanding Edge Function CPU limits" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-monitoring-resource-usage", + "title": "Monitoring Edge Function resource usage" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/how-do-i-make-the-cookies-httponly-vwweFx", - "title": "How do I make the cookies HttpOnly?" + "url": "https://supabase.com/docs/guides/functions/storage-caching", + "title": "Integrating with Supabase Storage" } ], - "resultChars": 103031 + "resultChars": 13135 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"admin deleteUser delete user existing JWT access token valid until expiry\", limit: 6) { nodes { ... on ClientLibraryFunctionReference { title href content language methodName } ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"Edge Function shutdown reason CPUTime 546\", limit: 3) { nodes { ... on TroubleshootingGuide { title href content } ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-shutdown-reasons-explained", + "title": "Edge Function shutdown reasons explained" }, { - "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-cpu-limits", + "title": "Understanding Edge Function CPU limits" }, { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" + "url": "https://supabase.com/docs/guides/functions/status-codes", + "title": "Status codes" } ], - "resultChars": 33037 + "resultChars": 17473 } ] }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "prompt": "Our `video-thumbnails` edge function has been failing intermittently since this morning. It generates a thumbnail from a user-uploaded video, and about half the calls are erroring out.\n\nCan you investigate the project logs and tell me what's going on and what we should do about it?", + "promptSourcePath": "evals/investigate-functions-001-546-resource-limit/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/investigate-auth-001-deleted-user-access.json" + "sourcePath": "codex-gpt-5.6-no-skills/investigate-functions-001-546-resource-limit.json" }, { "experiment": "codex-gpt-5.6-no-skills", @@ -18657,6 +21074,107 @@ "attempts": 1, "sourcePath": "codex-gpt-5.6-no-skills/resolve-dataapi-001-empty-results.json" }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "resolve-dataapi-002-update-zero-rows-affected", + "stage": "resolve", + "product": [ + "data-api", + "database" + ], + "topic": [ + "rls" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "RLS still enabled on tasks", + "passed": true + }, + { + "name": "user A's update actually checks off their own task", + "passed": true, + "notes": "saw: [{\"id\":\"7dc865b9-a257-4923-bf43-cb9dace055bd\",\"is_done\":true}]" + }, + { + "name": "user B cannot update user A's task", + "passed": true + }, + { + "name": "diagnosed the missing USING clause and added it", + "passed": true, + "judgeNotes": "Diagnosed the UPDATE policy missing a USING clause, explained the zero-row behavior, and fixed it with USING (user_id = auth.uid()) while retaining WITH CHECK and authenticated scope." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"row level security update policy USING WITH CHECK UPDATE requires SELECT policy\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/rls-simplified-BJTcS8", + "title": "RLS Simplified" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + } + ], + "resultChars": 90787 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase UPDATE policy using auth.uid() user_id example select policy required\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" + } + ], + "resultChars": 65229 + } + ] + }, + "prompt": "Our app lets signed-in users manage a personal `tasks` list. Users can create tasks and check them off (`is_done`).\n\nCreating a task works fine, and I can see the row in the table. But when a user checks off a task, the app's update call succeeds with no error, yet `is_done` never actually changes, and the API doesn't return the updated row either.\n\nFind out why the update has no effect and fix it.", + "promptSourcePath": "evals/resolve-dataapi-002-update-zero-rows-affected/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6-no-skills/resolve-dataapi-002-update-zero-rows-affected.json" + }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", @@ -18801,6 +21319,94 @@ "attempts": 1, "sourcePath": "codex-gpt-5.6-no-skills/resolve-performance-001-slow-query-cpu-spike.json" }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "resolve-performance-002-rls-auth-initplan", + "stage": "resolve", + "product": [ + "database", + "data-api" + ], + "topic": [ + "rls", + "sql" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS still enabled on documents", + "passed": true + }, + { + "name": "user A still reads exactly their own documents", + "passed": true, + "notes": "saw 2 rows" + }, + { + "name": "user B cannot read user A's documents", + "passed": true + }, + { + "name": "SELECT policy evaluates auth.uid() as an InitPlan (wrapped in a subquery)", + "passed": true, + "notes": "(( SELECT auth.uid() AS uid) = user_id)" + }, + { + "name": "diagnosed per-row auth.uid() re-evaluation and applied the InitPlan fix", + "passed": true, + "judgeNotes": "Diagnosed direct auth.uid() per-row RLS evaluation, rewrote policies to use (select auth.uid()) = user_id while preserving authenticated owner-only access and RLS, and verified InitPlan plus no widened access." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"RLS performance auth.uid select index policy\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/rls-performance-and-best-practices-Z5Jjwv", + "title": "RLS Performance and Best Practices" + }, + { + "url": "https://supabase.com/docs/guides/storage/production/scaling", + "title": "Storage Optimizations" + }, + { + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0003_auth_rls_initplan", + "title": "Database Advisor: Lint 0003_auth_rls_initplan" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + } + ], + "resultChars": 67690 + } + ] + }, + "prompt": "Our `documents` table has Row Level Security enabled so each signed-in user only sees their own rows. It worked fine in testing, but now that the table has grown the document list has become painfully slow — a simple \"list my documents\" query that should be instant now takes seconds and gets slower as the table grows, even though each user only owns a handful of rows.\n\nWe don't want to weaken the security model — users must still only ever see their own documents. Find out why the query is so slow and fix it.", + "promptSourcePath": "evals/resolve-performance-002-rls-auth-initplan/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6-no-skills/resolve-performance-002-rls-auth-initplan.json" + }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", @@ -18833,32 +21439,97 @@ "passed": true }, { - "name": "tenant B cannot read org A notes", + "name": "tenant B cannot read org A notes", + "passed": true + }, + { + "name": "tenant A author can update own note", + "passed": true + }, + { + "name": "tenant B cannot update org A note", + "passed": true + }, + { + "name": "tenant B author can delete own note", + "passed": true + }, + { + "name": "tenant B cannot delete org A note", + "passed": true + }, + { + "name": "tenant A can insert note in own org", + "passed": true + }, + { + "name": "tenant B cannot insert into org A", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6-no-skills/resolve-security-002-rls-cross-tenant-leak.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "resolve-storage-001-upsert-missing-update-policy", + "stage": "resolve", + "product": [ + "storage", + "database" + ], + "topic": [ + "rls" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "bucket avatars exists", "passed": true }, { - "name": "tenant A author can update own note", - "passed": true + "name": "bucket avatars stays public", + "passed": true, + "notes": "the bucket being public is intentional (avatars need a public URL); it is not the bug" }, { - "name": "tenant B cannot update org A note", + "name": "RLS still enabled on storage.objects", "passed": true }, { - "name": "tenant B author can delete own note", + "name": "anon can still read the public avatar", "passed": true }, { - "name": "tenant B cannot delete org A note", - "passed": true + "name": "user A can replace their own avatar via upsert", + "passed": true, + "notes": "saw: [{\"name\":\"019ff5ef-fc53-77f9-aa35-1ad7bb7f5f66/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" }, { - "name": "tenant A can insert note in own org", + "name": "user B cannot overwrite user A's avatar", "passed": true }, { - "name": "tenant B cannot insert into org A", - "passed": true + "name": "added an owner-scoped UPDATE policy without weakening public reads", + "passed": true, + "judgeNotes": "The assistant correctly diagnosed missing UPDATE RLS policy for storage.objects during upsert, noted public bucket only affects downloads, added an authenticated owner-scoped UPDATE policy with USING and WITH CHECK, and did not weaken public-read setup or disable RLS." } ], "skills": { @@ -18866,12 +21537,37 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Storage upsert existing file RLS UPDATE SELECT INSERT permissions\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", + "title": "Storage Helper Functions" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" + }, + { + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" + } + ], + "resultChars": 21335 + } + ] }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "prompt": "Our app has a public `avatars` bucket so profile photos have a public URL. Each user's avatar is stored at `/avatar.png`, and the app uploads it with `upsert: true` so a new photo replaces the old one at that same path.\n\nThe very first upload for a user always works, but replacing an existing avatar fails. Find out why and fix it.", + "promptSourcePath": "evals/resolve-storage-001-upsert-missing-update-policy/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/resolve-security-002-rls-cross-tenant-leak.json" + "sourcePath": "codex-gpt-5.6-no-skills/resolve-storage-001-upsert-missing-update-policy.json" }, { "experiment": "opencode-kimi-k3", @@ -20288,6 +22984,149 @@ "attempts": 1, "sourcePath": "opencode-kimi-k3/investigate-auth-001-deleted-user-access.json" }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "investigate-functions-001-546-resource-limit", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "identified video-thumbnails and the 546 resource-limit pattern", + "passed": true, + "judgeNotes": "Identified `video-thumbnails` as the affected function and correctly described failures as HTTP 546 resource/CPU limit shutdowns, not 500/503 server or boot/timeout errors." + }, + { + "name": "attributed the 546s to CPU time exhaustion", + "passed": true, + "judgeNotes": "The assistant explicitly attributes the 546 failures to CPU time exhaustion, citing log evidence including `shutdown (reason: CPUTime, cpu_time_used: 2000ms, cpu_time_limit: 2000ms)`, and distinguishes CPU time from wall-clock time." + }, + { + "name": "recommended reducing/offloading CPU work as the fix", + "passed": true, + "judgeNotes": "The assistant clearly identifies CPU time limit exhaustion and recommends reducing per-request CPU work via optimization, caps, lighter processing, offloading to external services/workers, and splitting into background jobs. It mentions retries only as a short-term band-aid, not the fix." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/monitoring-and-debugging.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging.md" + } + ], + "resultChars": 1990 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge function CPU time limit shutdown 546\") { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-546-error-response", + "title": "546 - WORKER_RESOURCE_LIMIT Exceeded / WORKER_LIMIT Exceeded" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-wall-clock-time-limit-reached-Nk38bW", + "title": "Edge Function 'wall clock time limit reached'" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-cpu-limits", + "title": "Understanding Edge Function CPU limits" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-functions-worker-timeouts-and-websocket-drops", + "title": "Edge Functions worker timeouts and WebSocket drops" + }, + { + "url": "https://supabase.com/docs/guides/functions/limits", + "title": "Limits" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-504-error-response", + "title": "Edge Function 504 error response" + }, + { + "url": "https://supabase.com/docs/guides/functions/status-codes", + "title": "Status codes" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/debugging", + "title": "Debugging guide" + }, + { + "url": "https://supabase.com/docs/guides/functions/recursive-functions", + "title": "Recursive / Nested Function Calls" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-takes-too-long-to-respond", + "title": "Edge Function takes too long to respond" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-shutdown-reasons-explained", + "title": "Edge Function shutdown reasons explained" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/storage/vector/storing-vectors", + "title": "Storing Vectors" + }, + { + "url": "https://supabase.com/docs/guides/functions/wasm", + "title": "Using Wasm modules" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" + } + ], + "resultChars": 122371 + }, + { + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/troubleshooting/edge-function-546-error-response.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-546-error-response.md" + } + ], + "resultChars": 13493 + } + ] + }, + "prompt": "Our `video-thumbnails` edge function has been failing intermittently since this morning. It generates a thumbnail from a user-uploaded video, and about half the calls are erroring out.\n\nCan you investigate the project logs and tell me what's going on and what we should do about it?", + "promptSourcePath": "evals/investigate-functions-001-546-resource-limit/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/investigate-functions-001-546-resource-limit.json" + }, { "experiment": "opencode-kimi-k3", "experimentSuite": "benchmark", @@ -20506,6 +23345,63 @@ "attempts": 1, "sourcePath": "opencode-kimi-k3/resolve-dataapi-001-empty-results.json" }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "resolve-dataapi-002-update-zero-rows-affected", + "stage": "resolve", + "product": [ + "data-api", + "database" + ], + "topic": [ + "rls" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "RLS still enabled on tasks", + "passed": true + }, + { + "name": "user A's update actually checks off their own task", + "passed": true, + "notes": "saw: [{\"id\":\"b742248e-f914-48c1-a10a-c727d0d0f41b\",\"is_done\":true}]" + }, + { + "name": "user B cannot update user A's task", + "passed": true + }, + { + "name": "diagnosed the missing USING clause and added it", + "passed": true, + "judgeNotes": "The assistant correctly identified the missing USING clause on the UPDATE RLS policy as the cause of silent 0-row updates, explained WITH CHECK vs USING, and fixed it with a policy scoped to authenticated users using the task owner user_id while preserving WITH CHECK to prevent reassignment. It did not disable RLS or use a blanket policy." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app lets signed-in users manage a personal `tasks` list. Users can create tasks and check them off (`is_done`).\n\nCreating a task works fine, and I can see the row in the table. But when a user checks off a task, the app's update call succeeds with no error, yet `is_done` never actually changes, and the API doesn't return the updated row either.\n\nFind out why the update has no effect and fix it.", + "promptSourcePath": "evals/resolve-dataapi-002-update-zero-rows-affected/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/resolve-dataapi-002-update-zero-rows-affected.json" + }, { "experiment": "opencode-kimi-k3", "experimentSuite": "benchmark", @@ -20626,10 +23522,73 @@ "docs": { "calls": [] }, - "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", - "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", + "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", + "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/resolve-performance-001-slow-query-cpu-spike.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "resolve-performance-002-rls-auth-initplan", + "stage": "resolve", + "product": [ + "database", + "data-api" + ], + "topic": [ + "rls", + "sql" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS still enabled on documents", + "passed": true + }, + { + "name": "user A still reads exactly their own documents", + "passed": true, + "notes": "saw 2 rows" + }, + { + "name": "user B cannot read user A's documents", + "passed": true + }, + { + "name": "SELECT policy evaluates auth.uid() as an InitPlan (wrapped in a subquery)", + "passed": true, + "notes": "(( SELECT auth.uid() AS uid) = user_id)" + }, + { + "name": "diagnosed per-row auth.uid() re-evaluation and applied the InitPlan fix", + "passed": true, + "judgeNotes": "The answer correctly diagnoses per-row auth.uid() evaluation in the RLS predicate, fixes policies using (select auth.uid()) so it is evaluated once as an InitPlan, keeps owner scoping for SELECT/INSERT, and verifies RLS remains enabled." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Our `documents` table has Row Level Security enabled so each signed-in user only sees their own rows. It worked fine in testing, but now that the table has grown the document list has become painfully slow — a simple \"list my documents\" query that should be instant now takes seconds and gets slower as the table grows, even though each user only owns a handful of rows.\n\nWe don't want to weaken the security model — users must still only ever see their own documents. Find out why the query is so slow and fix it.", + "promptSourcePath": "evals/resolve-performance-002-rls-auth-initplan/PROMPT.md", "attempts": 1, - "sourcePath": "opencode-kimi-k3/resolve-performance-001-slow-query-cpu-spike.json" + "sourcePath": "opencode-kimi-k3/resolve-performance-002-rls-auth-initplan.json" }, { "experiment": "opencode-kimi-k3", @@ -20708,6 +23667,76 @@ "attempts": 1, "sourcePath": "opencode-kimi-k3/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "resolve-storage-001-upsert-missing-update-policy", + "stage": "resolve", + "product": [ + "storage", + "database" + ], + "topic": [ + "rls" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "bucket avatars exists", + "passed": true + }, + { + "name": "bucket avatars stays public", + "passed": true, + "notes": "the bucket being public is intentional (avatars need a public URL); it is not the bug" + }, + { + "name": "RLS still enabled on storage.objects", + "passed": true + }, + { + "name": "anon can still read the public avatar", + "passed": true + }, + { + "name": "user A can replace their own avatar via upsert", + "passed": true, + "notes": "saw: [{\"name\":\"019ff5ef-b89d-70fe-ab82-411fe9675ebf/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" + }, + { + "name": "user B cannot overwrite user A's avatar", + "passed": true + }, + { + "name": "added an owner-scoped UPDATE policy without weakening public reads", + "passed": true, + "judgeNotes": "Diagnosed missing storage.objects UPDATE RLS policy for upsert replacement, kept public-read setup/RLS, and added authenticated owner-scoped UPDATE policy with USING and WITH CHECK." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app has a public `avatars` bucket so profile photos have a public URL. Each user's avatar is stored at `/avatar.png`, and the app uploads it with `upsert: true` so a new photo replaces the old one at that same path.\n\nThe very first upload for a user always works, but replacing an existing avatar fails. Find out why and fix it.", + "promptSourcePath": "evals/resolve-storage-001-upsert-missing-update-policy/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/resolve-storage-001-upsert-missing-update-policy.json" + }, { "experiment": "opencode-kimi-k3-no-skills", "experimentSuite": "no-skills", @@ -21898,6 +24927,82 @@ "attempts": 1, "sourcePath": "opencode-kimi-k3-no-skills/investigate-auth-001-deleted-user-access.json" }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "investigate-functions-001-546-resource-limit", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "identified video-thumbnails and the 546 resource-limit pattern", + "passed": true, + "judgeNotes": "Identified video-thumbnails as affected and correctly described failures as HTTP 546 WORKER_RESOURCE_LIMIT/CPUTime resource limit, while noting welcome-email 500 as unrelated." + }, + { + "name": "attributed the 546s to CPU time exhaustion", + "passed": true, + "judgeNotes": "The assistant specifically attributes the 546 failures to CPU time exhaustion, citing shutdown reason CPUTime and cpu_time_used/cpu_time_limit 2000/2000ms. It also distinguishes the unrelated welcome-email error." + }, + { + "name": "recommended reducing/offloading CPU work as the fix", + "passed": true, + "judgeNotes": "The assistant clearly identifies CPUTime limit as the cause and recommends offloading CPU-intensive thumbnail generation to a background worker/external service/client-side, constraining inputs, and notes code tweaks/input limits as mitigation. This satisfies the rubric by reducing/offloading CPU-intensive work rather than merely increasing timeout or retrying." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge function CPU time limit shutdown CPUTime 546\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-cpu-limits", + "title": "Understanding Edge Function CPU limits" + }, + { + "url": "https://supabase.com/docs/guides/functions/limits", + "title": "Limits" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-546-error-response", + "title": "546 - WORKER_RESOURCE_LIMIT Exceeded / WORKER_LIMIT Exceeded" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-wall-clock-time-limit-reached-Nk38bW", + "title": "Edge Function 'wall clock time limit reached'" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-504-error-response", + "title": "Edge Function 504 error response" + } + ], + "resultChars": 27605 + } + ] + }, + "prompt": "Our `video-thumbnails` edge function has been failing intermittently since this morning. It generates a thumbnail from a user-uploaded video, and about half the calls are erroring out.\n\nCan you investigate the project logs and tell me what's going on and what we should do about it?", + "promptSourcePath": "evals/investigate-functions-001-546-resource-limit/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/investigate-functions-001-546-resource-limit.json" + }, { "experiment": "opencode-kimi-k3-no-skills", "experimentSuite": "no-skills", @@ -22071,6 +25176,57 @@ "attempts": 1, "sourcePath": "opencode-kimi-k3-no-skills/resolve-dataapi-001-empty-results.json" }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "resolve-dataapi-002-update-zero-rows-affected", + "stage": "resolve", + "product": [ + "data-api", + "database" + ], + "topic": [ + "rls" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "RLS still enabled on tasks", + "passed": true + }, + { + "name": "user A's update actually checks off their own task", + "passed": true, + "notes": "saw: [{\"id\":\"4294c7c2-03a7-4049-a220-99a50a6c4bcf\",\"is_done\":true}]" + }, + { + "name": "user B cannot update user A's task", + "passed": true + }, + { + "name": "diagnosed the missing USING clause and added it", + "passed": true, + "judgeNotes": "The assistant correctly identified the missing USING clause on the UPDATE RLS policy as the cause of silent zero-row updates, explained WITH CHECK vs USING, and fixed it with FOR UPDATE TO authenticated USING (user_id = auth.uid()) WITH CHECK (user_id = auth.uid()). It kept RLS semantics scoped to authenticated users and row ownership." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app lets signed-in users manage a personal `tasks` list. Users can create tasks and check them off (`is_done`).\n\nCreating a task works fine, and I can see the row in the table. But when a user checks off a task, the app's update call succeeds with no error, yet `is_done` never actually changes, and the API doesn't return the updated row either.\n\nFind out why the update has no effect and fix it.", + "promptSourcePath": "evals/resolve-dataapi-002-update-zero-rows-affected/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/resolve-dataapi-002-update-zero-rows-affected.json" + }, { "experiment": "opencode-kimi-k3-no-skills", "experimentSuite": "no-skills", @@ -22184,6 +25340,64 @@ "attempts": 1, "sourcePath": "opencode-kimi-k3-no-skills/resolve-performance-001-slow-query-cpu-spike.json" }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "resolve-performance-002-rls-auth-initplan", + "stage": "resolve", + "product": [ + "database", + "data-api" + ], + "topic": [ + "rls", + "sql" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS still enabled on documents", + "passed": true + }, + { + "name": "user A still reads exactly their own documents", + "passed": true, + "notes": "saw 2 rows" + }, + { + "name": "user B cannot read user A's documents", + "passed": true + }, + { + "name": "SELECT policy evaluates auth.uid() as an InitPlan (wrapped in a subquery)", + "passed": true, + "notes": "(( SELECT auth.uid() AS uid) = user_id)" + }, + { + "name": "diagnosed per-row auth.uid() re-evaluation and applied the InitPlan fix", + "passed": true, + "judgeNotes": "The answer correctly identifies per-row auth.uid()/JWT setting evaluation in the RLS predicate as the cause, fixes the RLS policies with (select auth.uid()) so it becomes an InitPlan evaluated once per query, and preserves owner-scoped SELECT/INSERT policies without disabling RLS or widening access." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our `documents` table has Row Level Security enabled so each signed-in user only sees their own rows. It worked fine in testing, but now that the table has grown the document list has become painfully slow — a simple \"list my documents\" query that should be instant now takes seconds and gets slower as the table grows, even though each user only owns a handful of rows.\n\nWe don't want to weaken the security model — users must still only ever see their own documents. Find out why the query is so slow and fix it.", + "promptSourcePath": "evals/resolve-performance-002-rls-auth-initplan/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/resolve-performance-002-rls-auth-initplan.json" + }, { "experiment": "opencode-kimi-k3-no-skills", "experimentSuite": "no-skills", @@ -22254,5 +25468,69 @@ "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", "attempts": 1, "sourcePath": "opencode-kimi-k3-no-skills/resolve-security-002-rls-cross-tenant-leak.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "resolve-storage-001-upsert-missing-update-policy", + "stage": "resolve", + "product": [ + "storage", + "database" + ], + "topic": [ + "rls" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "bucket avatars exists", + "passed": true + }, + { + "name": "bucket avatars stays public", + "passed": true, + "notes": "the bucket being public is intentional (avatars need a public URL); it is not the bug" + }, + { + "name": "RLS still enabled on storage.objects", + "passed": true + }, + { + "name": "anon can still read the public avatar", + "passed": true + }, + { + "name": "user A can replace their own avatar via upsert", + "passed": true, + "notes": "saw: [{\"name\":\"019ff5f9-2d74-758c-bfe0-c692030d3745/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" + }, + { + "name": "user B cannot overwrite user A's avatar", + "passed": true + }, + { + "name": "added an owner-scoped UPDATE policy without weakening public reads", + "passed": true, + "judgeNotes": "The assistant correctly diagnosed the issue as a missing UPDATE RLS policy on storage.objects for upsert-based avatar replacement, noted public bucket only affects reads, added an authenticated owner-scoped UPDATE policy with both USING and WITH CHECK, and preserved public-read/RLS setup." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app has a public `avatars` bucket so profile photos have a public URL. Each user's avatar is stored at `/avatar.png`, and the app uploads it with `upsert: true` so a new photo replaces the old one at that same path.\n\nThe very first upload for a user always works, but replacing an existing avatar fails. Find out why and fix it.", + "promptSourcePath": "evals/resolve-storage-001-upsert-missing-update-policy/PROMPT.md", + "attempts": 2, + "sourcePath": "opencode-kimi-k3-no-skills/resolve-storage-001-upsert-missing-update-policy.json" } ] diff --git a/evals/investigate-functions-001-546-resource-limit/PROMPT.md b/evals/investigate-functions-001-546-resource-limit/PROMPT.md index cce52dae..f509b9b4 100644 --- a/evals/investigate-functions-001-546-resource-limit/PROMPT.md +++ b/evals/investigate-functions-001-546-resource-limit/PROMPT.md @@ -1,6 +1,6 @@ --- stage: investigate -suite: regression +suite: benchmark product: - edge-functions topic: diff --git a/evals/resolve-dataapi-002-update-zero-rows-affected/PROMPT.md b/evals/resolve-dataapi-002-update-zero-rows-affected/PROMPT.md index 5f910900..391d51cf 100644 --- a/evals/resolve-dataapi-002-update-zero-rows-affected/PROMPT.md +++ b/evals/resolve-dataapi-002-update-zero-rows-affected/PROMPT.md @@ -1,6 +1,6 @@ --- stage: resolve -suite: regression +suite: benchmark product: - data-api - database diff --git a/evals/resolve-performance-002-rls-auth-initplan/EVAL.ts b/evals/resolve-performance-002-rls-auth-initplan/EVAL.ts new file mode 100644 index 00000000..9919ad83 --- /dev/null +++ b/evals/resolve-performance-002-rls-auth-initplan/EVAL.ts @@ -0,0 +1,218 @@ +import { + judge, + serializeTranscript, + type CheckResult, + type SupabaseClient, + type ToolEvalContext, + type ToolScorer, +} from '@supabase-evals/core'; +import { stripIndent } from 'common-tags'; + +const PASSWORD = 'secret123'; + +const scorer: ToolScorer = async (ctx) => { + try { + const setup = await setupTestUsers(ctx); + if ('failure' in setup) { + return { passed: false, checks: [setup.failure] }; + } + const users = setup.users; + await seedOwnedDocuments(ctx, users); + + const checks: CheckResult[] = [ + await checkRlsStillEnabled(ctx), + await checkUserAReadsOnlyOwnDocuments(users), + await checkUserBCannotReadUserADocuments(users), + await checkSelectPolicyUsesInitplan(ctx), + await checkPerformanceDiagnosis(ctx), + ]; + + return { + passed: checks.every((check) => check.passed), + checks, + }; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + return { + passed: false, + checks: [ + { + name: 'scorer evaluated the RLS performance fix', + passed: false, + notes: msg, + }, + ], + }; + } +}; + +export default scorer; + +type TestUsers = { + clientA: SupabaseClient; + clientB: SupabaseClient; + userAId: string; + userBId: string; +}; + +async function setupTestUsers( + ctx: ToolEvalContext +): Promise<{ users: TestUsers } | { failure: CheckResult }> { + const clientA = ctx.client; + const clientB = ctx.getClient(); + + const { data: authA, error: authAError } = await clientA.auth.signUp({ + email: `rls-initplan-a-${Date.now()}@example.com`, + password: PASSWORD, + }); + const { data: authB, error: authBError } = await clientB.auth.signUp({ + email: `rls-initplan-b-${Date.now()}@example.com`, + password: PASSWORD, + }); + + if ( + authAError || + authBError || + !authA.user?.id || + !authA.session || + !authB.user?.id || + !authB.session + ) { + return { + failure: { + name: 'created auth sessions', + passed: false, + notes: authAError?.message ?? authBError?.message ?? 'missing session', + }, + }; + } + + return { + users: { clientA, clientB, userAId: authA.user.id, userBId: authB.user.id }, + }; +} + +async function seedOwnedDocuments( + ctx: ToolEvalContext, + users: TestUsers +): Promise { + await ctx.query(stripIndent` + INSERT INTO documents (user_id, title, body) VALUES + ('${users.userAId}', 'A first', 'owned by A'), + ('${users.userAId}', 'A second', 'owned by A'), + ('${users.userBId}', 'B first', 'owned by B'); + `); +} + +async function checkRlsStillEnabled( + ctx: ToolEvalContext +): Promise { + const { rows } = await ctx.query( + `SELECT relrowsecurity FROM pg_class WHERE relname = 'documents';` + ); + + return { + name: 'RLS still enabled on documents', + passed: rows[0]?.relrowsecurity === true, + notes: + rows[0]?.relrowsecurity === true + ? undefined + : 'RLS must stay enabled; disabling it is not an acceptable fix', + }; +} + +async function checkUserAReadsOnlyOwnDocuments( + users: TestUsers +): Promise { + const { data, error } = await users.clientA + .from('documents') + .select('title,user_id'); + + const allOwned = + !error && + Array.isArray(data) && + data.length === 2 && + data.every((row) => row.user_id === users.userAId); + + return { + name: 'user A still reads exactly their own documents', + passed: allOwned, + notes: error?.message ?? `saw ${data?.length ?? 0} rows`, + }; +} + +async function checkUserBCannotReadUserADocuments( + users: TestUsers +): Promise { + const { data, error } = await users.clientB + .from('documents') + .select('title,user_id') + .eq('user_id', users.userAId); + + return { + name: "user B cannot read user A's documents", + passed: !error && Array.isArray(data) && data.length === 0, + notes: error?.message, + }; +} + +/** + * The performance fix: the ownership predicate must evaluate the auth call once + * per query (an InitPlan) instead of once per row. In Postgres that means + * wrapping it in a scalar subquery -- `(select auth.uid())` -- which renders in + * pg_policies.qual with a nested SELECT. A bare `auth.uid()` predicate renders + * without one. + */ +async function checkSelectPolicyUsesInitplan( + ctx: ToolEvalContext +): Promise { + const { rows } = await ctx.query(stripIndent` + SELECT qual + FROM pg_policies + WHERE schemaname = 'public' + AND tablename = 'documents' + AND cmd = 'SELECT'; + `); + + const quals = rows + .map((row) => (typeof row.qual === 'string' ? row.qual : '')) + .filter((qual) => qual.length > 0); + + const wrapsAuthInSubquery = quals.some( + (qual) => /\bselect\b/i.test(qual) && /auth\.(uid|jwt)\b/i.test(qual) + ); + + return { + name: 'SELECT policy evaluates auth.uid() as an InitPlan (wrapped in a subquery)', + passed: wrapsAuthInSubquery, + notes: quals.length + ? quals.join(' | ') + : 'no SELECT policy found on documents', + }; +} + +async function checkPerformanceDiagnosis( + ctx: ToolEvalContext +): Promise { + const verdict = await judge({ + input: serializeTranscript(ctx.transcript, { + includeToolCallInputs: true, + }), + rubric: stripIndent` + Pass if the assistant diagnoses the slow list query as the RLS policy calling auth.uid() once per row and fixes it by making the auth call evaluate a single time per query. + + A passing answer should: + - Identify that the ownership predicate re-evaluates auth.uid() (or current_setting for the JWT) for every scanned row, so cost grows with table size. + - Fix it by wrapping the auth call in a scalar subquery, e.g. USING ((select auth.uid()) = user_id), so Postgres caches it as an InitPlan and runs it once. + - Keep the policy scoped to the row owner and keep RLS enabled. + + Fail if the assistant blames only a missing index and adds one without addressing the per-row auth.uid() call, disables RLS, replaces the predicate with a blanket USING (true) or otherwise widens who can read rows, scopes the policy to anon/public, or never touches the RLS policy. + `, + }); + + return { + name: 'diagnosed per-row auth.uid() re-evaluation and applied the InitPlan fix', + passed: verdict.passed, + judgeNotes: verdict.notes, + }; +} diff --git a/evals/resolve-performance-002-rls-auth-initplan/PROMPT.md b/evals/resolve-performance-002-rls-auth-initplan/PROMPT.md new file mode 100644 index 00000000..bef54f60 --- /dev/null +++ b/evals/resolve-performance-002-rls-auth-initplan/PROMPT.md @@ -0,0 +1,16 @@ +--- +stage: resolve +suite: benchmark +interface: mcp +product: + - database + - data-api +topic: + - rls + - sql +motivation: https://linear.app/supabase/issue/FDBKIN-32777/improve-rls-performance-on-tables-with-billions-of-rows-to-reduce, https://linear.app/supabase/issue/FDBKIN-25150/improve-rls-performance-by-documenting-or-automating-the-select +--- + +Our `documents` table has Row Level Security enabled so each signed-in user only sees their own rows. It worked fine in testing, but now that the table has grown the document list has become painfully slow — a simple "list my documents" query that should be instant now takes seconds and gets slower as the table grows, even though each user only owns a handful of rows. + +We don't want to weaken the security model — users must still only ever see their own documents. Find out why the query is so slow and fix it. diff --git a/evals/resolve-performance-002-rls-auth-initplan/remote/project.sql b/evals/resolve-performance-002-rls-auth-initplan/remote/project.sql new file mode 100644 index 00000000..cca36ca3 --- /dev/null +++ b/evals/resolve-performance-002-rls-auth-initplan/remote/project.sql @@ -0,0 +1,40 @@ +CREATE TABLE documents ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + user_id uuid NOT NULL DEFAULT auth.uid(), + title text NOT NULL, + body text NOT NULL DEFAULT '', + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX documents_user_id_idx ON documents (user_id); + +ALTER TABLE documents ENABLE ROW LEVEL SECURITY; + +GRANT SELECT, INSERT, UPDATE, DELETE ON documents TO authenticated; + +-- Bug: the ownership predicate calls auth.uid() directly, so Postgres +-- re-evaluates the function once per row scanned instead of once per query. +-- On a large table this turns every RLS-filtered read into a per-row function +-- call and the query slows down as the table grows. The fix is to wrap the +-- auth call in a scalar subquery -- (select auth.uid()) -- so the planner +-- caches it as an InitPlan and evaluates it a single time. +CREATE POLICY "read own documents" +ON documents +FOR SELECT +TO authenticated +USING (user_id = auth.uid()); + +CREATE POLICY "insert own documents" +ON documents +FOR INSERT +TO authenticated +WITH CHECK (user_id = auth.uid()); + +-- Bulk background rows owned by other users, so the RLS filter has to scan a +-- large table and the per-row auth.uid() re-evaluation is measurable. +INSERT INTO documents (user_id, title, body) +SELECT + gen_random_uuid(), + 'doc ' || g, + repeat('lorem ipsum ', 20) +FROM generate_series(1, 20000) AS g; diff --git a/evals/resolve-storage-001-upsert-missing-update-policy/PROMPT.md b/evals/resolve-storage-001-upsert-missing-update-policy/PROMPT.md index 15fc3550..09ee659a 100644 --- a/evals/resolve-storage-001-upsert-missing-update-policy/PROMPT.md +++ b/evals/resolve-storage-001-upsert-missing-update-policy/PROMPT.md @@ -1,6 +1,6 @@ --- stage: resolve -suite: regression +suite: benchmark product: - storage - database diff --git a/submodules/agent-skills b/submodules/agent-skills index 12077673..8331f910 160000 --- a/submodules/agent-skills +++ b/submodules/agent-skills @@ -1 +1 @@ -Subproject commit 1207767388a0ffb55f21fb4e6988fee96942431d +Subproject commit 8331f910845103c08d51f6ca1d86ebb7d1f745e3