feat: move eval execution from GitHub Actions to Vercel Sandboxes

.github/workflows/eval-refresh.yml

@@ -27,6 +27,10 @@ on:
         description: "Timeout per attempt in seconds"
         required: true
         default: "720"
+      sandbox_concurrency:
+        description: "Maximum Vercel Sandboxes running at once"
+        required: true
+        default: "250"
       merge:
         description: "Merge into existing results instead of overwriting (graft new experiment/eval pairs)"
         type: boolean
@@ -72,6 +76,7 @@ jobs:
       pairs: ${{ steps.discover.outputs.pairs }}
       runs: ${{ steps.inputs.outputs.runs }}
       timeout_sec: ${{ steps.inputs.outputs.timeout_sec }}
+      sandbox_concurrency: ${{ steps.inputs.outputs.sandbox_concurrency }}
       filter_changed: ${{ steps.inputs.outputs.filter_changed }}
       do_merge: ${{ steps.inputs.outputs.do_merge }}
     steps:
@@ -88,20 +93,23 @@ jobs:
             experiment_suite="${{ inputs.experiment_suite }}"
             runs="${{ inputs.runs }}"
             timeout_sec="${{ inputs.timeout_sec }}"
+            sandbox_concurrency="${{ inputs.sandbox_concurrency }}"
           elif [ "${{ github.event_name }}" = "schedule" ]; then
             experiments_override=""
             eval_id=""
             suite="regression"
             experiment_suite="regression"
             runs="2"
             timeout_sec="720"
+            sandbox_concurrency="250"
           else
             experiments_override=""
             eval_id=""
             suite="benchmark,regression"
             experiment_suite="benchmark,no-skills,regression"
             runs="2"
             timeout_sec="720"
+            sandbox_concurrency="250"
           fi
 
           suite_json="$(jq -Rc 'split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))' <<< "$suite")"
@@ -130,6 +138,7 @@ jobs:
             echo "experiment_suite=$experiment_suite_json"
             echo "runs=$runs"
             echo "timeout_sec=$timeout_sec"
+            echo "sandbox_concurrency=$sandbox_concurrency"
             echo "filter_changed=$filter_changed"
             echo "do_merge=$do_merge"
           } >> "$GITHUB_OUTPUT"
@@ -252,19 +261,20 @@ jobs:
     needs: prepare
     if: needs.prepare.outputs.pairs != '[]'
     runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        include: ${{ fromJSON(needs.prepare.outputs.pairs) }}
+    # 60 min gives ~2x margin over the slowest observed run:
+    # https://github.com/supabase/evals/actions/runs/31802584549
+    timeout-minutes: 60
     env:
       ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
       AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }}
+      VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}
+      VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }}
+      VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }}
     steps:
       - name: Checkout
         uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3
         with:
-          submodules: recursive
           ref: ${{ github.event_name == 'pull_request' && github.head_ref || github.ref }}
 
       - name: Install pnpm
@@ -291,35 +301,35 @@ jobs:
           } > .env
 
       - name: Run evals
+        env:
+          EVAL_PAIRS: ${{ needs.prepare.outputs.pairs }}
+          EVAL_REVISION: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
         shell: bash
         run: |
           set -euo pipefail
 
-          pnpm eval -- \
-            --experiment "${{ matrix.experiment }}" \
-            --experiment-suite "${{ matrix.experiment_suite }}" \
-            --eval "${{ matrix.eval_id }}" \
+          pnpm --filter @supabase-evals/framework eval:vercel -- \
+            --pairs-json "$EVAL_PAIRS" \
+            --revision "$EVAL_REVISION" \
             --runs "${{ needs.prepare.outputs.runs }}" \
-            --timeout-sec "${{ needs.prepare.outputs.timeout_sec }}"
-
-          # A skipped experiment (e.g. missing API key) exits 0 without writing results.
-          if [ ! -f "results/${{ matrix.experiment }}/${{ matrix.eval_id }}.json" ]; then
-            echo "::error::no result written for ${{ matrix.experiment }} x ${{ matrix.eval_id }} (experiment skipped?)"
-            exit 1
-          fi
+            --timeout-sec "${{ needs.prepare.outputs.timeout_sec }}" \
+            --concurrency "${{ needs.prepare.outputs.sandbox_concurrency }}"
 
       - name: Upload raw results
+        if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: raw-results-${{ matrix.experiment }}__${{ matrix.eval_id }}
-          path: |
-            results/${{ matrix.experiment }}/
-            !results/**/node_modules/**
+          name: raw-results
+          path: results/downloaded/
+          overwrite: true
+          if-no-files-found: warn
           retention-days: 3
 
   publish-results:
     needs: [prepare, run-evals]
-    if: needs.prepare.outputs.pairs != '[]'
+    # Runs even after run-evals times out or partially fails, since results
+    # are written per pair and export-results skips any pair that never finished.
+    if: "always() && needs.run-evals.result != 'skipped' && needs.prepare.outputs.pairs != '[]'"
     runs-on: ubuntu-latest
     steps:
       - name: Generate GitHub App token
@@ -338,7 +348,6 @@ jobs:
       - name: Checkout
         uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3
         with:
-          submodules: recursive
           ref: ${{ github.event_name == 'pull_request' && github.head_ref || github.ref }}
           token: ${{ steps.generate-token.outputs.token || github.token }}
 
@@ -355,22 +364,56 @@ jobs:
         run: pnpm install --frozen-lockfile
 
       - name: Download raw results
-        # actions/download-artifact only nests a downloaded artifact under a
-        # directory named after it when there's more than one to disambiguate;
-        # with exactly one matching pair it dumps the contents flat into
-        # `path`, silently breaking the per-experiment glob below. Naming the
-        # destination directory ourselves side-steps that inconsistency.
-        env:
-          GH_TOKEN: ${{ github.token }}
+        # If every pair failed, run-evals never uploads a raw-results artifact
+        # at all, so this action errors. Audit pair results below already
+        # handles a missing/empty results/downloaded correctly (every pair
+        # reports missing), so don't let a total failure hide behind this.
+        continue-on-error: true
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: raw-results
+          path: results/downloaded
+
+      - name: Audit pair results
+        id: audit
         shell: bash
         run: |
           set -euo pipefail
 
-          mkdir -p downloaded-results
           pairs='${{ needs.prepare.outputs.pairs }}'
-          while IFS= read -r name; do
-            gh run download ${{ github.run_id }} --name "$name" --dir "downloaded-results/$name"
-          done < <(jq -r '.[] | "raw-results-\(.experiment)__\(.eval_id)"' <<< "$pairs" | sort -u)
+          total="$(jq 'length' <<< "$pairs")"
+          missing=()
+          while IFS= read -r key; do
+            [ -d "results/downloaded/raw-results-$key" ] || missing+=("$key")
+          done < <(jq -r '.[] | "\(.experiment)__\(.eval_id)"' <<< "$pairs")
+
+          echo "count=${#missing[@]}" >> "$GITHUB_OUTPUT"
+
+          if [ "${#missing[@]}" -eq "$total" ]; then
+            echo "::warning::No results published: all $total eval pairs never finished (run-evals likely hit its timeout or failed partway). This run needs a follow-up to complete the refresh."
+            {
+              echo "## :warning: No results published"
+              echo ""
+              echo "All $total eval pairs never finished, likely because \`run-evals\` hit its timeout or failed partway through:"
+              echo ""
+              echo '```'
+              printf '%s\n' "${missing[@]}"
+              echo '```'
+            } >> "$GITHUB_STEP_SUMMARY"
+          elif [ "${#missing[@]}" -gt 0 ]; then
+            echo "::warning::Publishing PARTIAL results: ${#missing[@]} of $total eval pairs never finished (run-evals likely hit its timeout or failed partway). This run needs a follow-up to complete the refresh."
+            {
+              echo "## :warning: Partial results published"
+              echo ""
+              echo "${#missing[@]} of $total eval pairs never finished, likely because \`run-evals\` hit its timeout or failed partway through. Only the completed pairs below were published:"
+              echo ""
+              echo '```'
+              printf '%s\n' "${missing[@]}"
+              echo '```'
+              echo ""
+              echo "Re-run this workflow to fill in the missing pairs."
+            } >> "$GITHUB_STEP_SUMMARY"
+          fi
 
       - name: Export results
         shell: bash
@@ -381,7 +424,7 @@ jobs:
           pairs='${{ needs.prepare.outputs.pairs }}'
           while IFS= read -r experiment; do
             mkdir -p "results/$experiment"
-            for artifact_dir in "downloaded-results/raw-results-${experiment}__"*/; do
+            for artifact_dir in "results/downloaded/raw-results-${experiment}__"*/; do
               [ -d "$artifact_dir" ] && cp -R "$artifact_dir"/. "results/$experiment"/
             done
           done < <(jq -r '.[].experiment' <<< "$pairs" | sort -u)
@@ -462,7 +505,21 @@ jobs:
 
       - name: Merge scheduled results pull request
         # The app is on the ruleset's bypass list, so no review is required.
-        if: github.event_name == 'schedule' && steps.cpr.outputs.pull-request-number
+        # Skips the merge when results are partial, so it stays open for review.
+        if: >-
+          github.event_name == 'schedule' &&
+          steps.cpr.outputs.pull-request-number &&
+          steps.audit.outputs.count == '0'
         env:
           GH_TOKEN: ${{ steps.generate-token.outputs.token }}
         run: gh pr merge "${{ steps.cpr.outputs.pull-request-number }}" --squash --delete-branch
+
+      - name: Require complete results
+        # Runs last so partial results still publish, but the job still fails.
+        if: >-
+          always() &&
+          steps.audit.outcome == 'success' &&
+          steps.audit.outputs.count != '0'
+        run: |
+          echo "::error::${{ steps.audit.outputs.count }} eval pair(s) never finished. This run needs a follow-up to complete the refresh."
+          exit 1

apps/framework/harness/run-eval.ts

@@ -21,7 +21,9 @@ import {
 } from '@supabase-evals/sandbox';
 import {
   normalizeExperimentName,
+  positiveInteger,
   readExperimentSuiteFilters,
+  readFlag,
   readRepeatedFlag,
   readSuiteFilters,
 } from '../lib/cli-args.js';
@@ -72,9 +74,15 @@ const SELECTED_EXPERIMENT_SUITE =
   EXPERIMENT_SUITE_FILTERS.length === 1
     ? EXPERIMENT_SUITE_FILTERS[0]
     : undefined;
-const RUNS = Number(readFlag('runs') ?? 1);
-const TIMEOUT_SEC = Number(readFlag('timeout-sec') ?? 720);
-const CONCURRENCY = Number(readFlag('concurrency') ?? 1);
+const RUNS = positiveInteger(readFlag(rawArgs, 'runs') ?? '1', 'runs');
+const TIMEOUT_SEC = positiveInteger(
+  readFlag(rawArgs, 'timeout-sec') ?? '720',
+  'timeout-sec'
+);
+const CONCURRENCY = positiveInteger(
+  readFlag(rawArgs, 'concurrency') ?? '1',
+  'concurrency'
+);
 const STOP_ON_PASS = !args.has('--run-all-attempts');
 const DEBUG = args.has('--debug');
 
@@ -91,22 +99,6 @@ async function loadExperiments() {
   return out;
 }
 
-function readFlag(name: string): string | undefined {
-  const prefix = `--${name}=`;
-  const inline = rawArgs.find((arg) => arg.startsWith(prefix));
-  if (inline) return inline.slice(prefix.length);
-
-  const idx = rawArgs.indexOf(`--${name}`);
-  if (idx !== -1) {
-    const value = rawArgs[idx + 1];
-    if (!value || value.startsWith('--')) {
-      throw new Error(`--${name} requires a value`);
-    }
-    return value;
-  }
-  return undefined;
-}
-
 /**
  * Resolve the run mode. The sandbox (local-stack) is needed when the agent
  * uses the Supabase CLI (`interface: cli`) — including bootstrap scenarios that

apps/framework/lib/cli-args.test.ts

@@ -0,0 +1,37 @@
+import { describe, expect, it } from 'vitest';
+import { positiveInteger, readFlag } from './cli-args.js';
+
+describe('readFlag', () => {
+  it('reads flags in both --name value and --name=value form', () => {
+    expect(readFlag(['--runs', '3'], 'runs')).toBe('3');
+    expect(readFlag(['--runs=3'], 'runs')).toBe('3');
+    expect(readFlag(['--other', 'x'], 'runs')).toBeUndefined();
+    expect(() => readFlag(['--runs'], 'runs')).toThrow(
+      '--runs requires a value'
+    );
+    expect(() => readFlag(['--runs', '--other'], 'runs')).toThrow(
+      '--runs requires a value'
+    );
+    expect(() => readFlag(['--runs='], 'runs')).toThrow(
+      '--runs requires a value'
+    );
+  });
+});
+
+describe('positiveInteger', () => {
+  it('rejects non-positive-integer CLI options', () => {
+    expect(positiveInteger('3', 'runs')).toBe(3);
+    expect(() => positiveInteger('0', 'runs')).toThrow(
+      '--runs must be a positive integer'
+    );
+    expect(() => positiveInteger('-1', 'runs')).toThrow(
+      '--runs must be a positive integer'
+    );
+    expect(() => positiveInteger('1.5', 'runs')).toThrow(
+      '--runs must be a positive integer'
+    );
+    expect(() => positiveInteger('abc', 'runs')).toThrow(
+      '--runs must be a positive integer'
+    );
+  });
+});

apps/framework/lib/cli-args.ts

@@ -4,6 +4,34 @@ import {
   type EvalSuite,
   type ExperimentSuite,
 } from '@supabase-evals/core/eval-metadata';
+import { z } from 'zod';
+
+const positiveIntegerSchema = z.coerce.number().int().min(1);
+
+/** Parses a positive integer CLI option. */
+export function positiveInteger(value: string, name: string): number {
+  const parsed = positiveIntegerSchema.safeParse(value);
+  if (!parsed.success) throw new Error(`--${name} must be a positive integer`);
+  return parsed.data;
+}
+
+/** Reads one CLI flag in either `--name value` or `--name=value` form. */
+export function readFlag(rawArgs: string[], name: string): string | undefined {
+  const prefix = `--${name}=`;
+  const inline = rawArgs.find((arg) => arg.startsWith(prefix));
+  if (inline) {
+    const value = inline.slice(prefix.length);
+    if (!value) throw new Error(`--${name} requires a value`);
+    return value;
+  }
+  const index = rawArgs.indexOf(`--${name}`);
+  if (index === -1) return undefined;
+  const value = rawArgs[index + 1];
+  if (!value || value.startsWith('--')) {
+    throw new Error(`--${name} requires a value`);
+  }
+  return value;
+}
 
 export function splitList(value: string): string[] {
   return value