diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 78af4ef6..413060f2 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -27,6 +27,10 @@ on: description: "Timeout per attempt in seconds" required: true default: "720" + sandbox_concurrency: + description: "Maximum Vercel Sandboxes running at once" + required: true + default: "250" merge: description: "Merge into existing results instead of overwriting (graft new experiment/eval pairs)" type: boolean @@ -72,6 +76,7 @@ jobs: pairs: ${{ steps.discover.outputs.pairs }} runs: ${{ steps.inputs.outputs.runs }} timeout_sec: ${{ steps.inputs.outputs.timeout_sec }} + sandbox_concurrency: ${{ steps.inputs.outputs.sandbox_concurrency }} filter_changed: ${{ steps.inputs.outputs.filter_changed }} do_merge: ${{ steps.inputs.outputs.do_merge }} steps: @@ -88,6 +93,7 @@ jobs: experiment_suite="${{ inputs.experiment_suite }}" runs="${{ inputs.runs }}" timeout_sec="${{ inputs.timeout_sec }}" + sandbox_concurrency="${{ inputs.sandbox_concurrency }}" elif [ "${{ github.event_name }}" = "schedule" ]; then experiments_override="" eval_id="" @@ -95,6 +101,7 @@ jobs: experiment_suite="regression" runs="2" timeout_sec="720" + sandbox_concurrency="250" else experiments_override="" eval_id="" @@ -102,6 +109,7 @@ jobs: experiment_suite="benchmark,no-skills,regression" runs="2" timeout_sec="720" + sandbox_concurrency="250" fi suite_json="$(jq -Rc 'split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))' <<< "$suite")" @@ -130,6 +138,7 @@ jobs: echo "experiment_suite=$experiment_suite_json" echo "runs=$runs" echo "timeout_sec=$timeout_sec" + echo "sandbox_concurrency=$sandbox_concurrency" echo "filter_changed=$filter_changed" echo "do_merge=$do_merge" } >> "$GITHUB_OUTPUT" @@ -252,19 +261,20 @@ jobs: needs: prepare if: needs.prepare.outputs.pairs != '[]' runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.prepare.outputs.pairs) }} + # 60 min gives ~2x margin over the slowest observed run: + # https://github.com/supabase/evals/actions/runs/31802584549 + timeout-minutes: 60 env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }} + VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} + VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }} + VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} steps: - name: Checkout uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3 with: - submodules: recursive ref: ${{ github.event_name == 'pull_request' && github.head_ref || github.ref }} - name: Install pnpm @@ -291,35 +301,35 @@ jobs: } > .env - name: Run evals + env: + EVAL_PAIRS: ${{ needs.prepare.outputs.pairs }} + EVAL_REVISION: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} shell: bash run: | set -euo pipefail - pnpm eval -- \ - --experiment "${{ matrix.experiment }}" \ - --experiment-suite "${{ matrix.experiment_suite }}" \ - --eval "${{ matrix.eval_id }}" \ + pnpm --filter @supabase-evals/framework eval:vercel -- \ + --pairs-json "$EVAL_PAIRS" \ + --revision "$EVAL_REVISION" \ --runs "${{ needs.prepare.outputs.runs }}" \ - --timeout-sec "${{ needs.prepare.outputs.timeout_sec }}" - - # A skipped experiment (e.g. missing API key) exits 0 without writing results. - if [ ! -f "results/${{ matrix.experiment }}/${{ matrix.eval_id }}.json" ]; then - echo "::error::no result written for ${{ matrix.experiment }} x ${{ matrix.eval_id }} (experiment skipped?)" - exit 1 - fi + --timeout-sec "${{ needs.prepare.outputs.timeout_sec }}" \ + --concurrency "${{ needs.prepare.outputs.sandbox_concurrency }}" - name: Upload raw results + if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: raw-results-${{ matrix.experiment }}__${{ matrix.eval_id }} - path: | - results/${{ matrix.experiment }}/ - !results/**/node_modules/** + name: raw-results + path: results/downloaded/ + overwrite: true + if-no-files-found: warn retention-days: 3 publish-results: needs: [prepare, run-evals] - if: needs.prepare.outputs.pairs != '[]' + # Runs even after run-evals times out or partially fails, since results + # are written per pair and export-results skips any pair that never finished. + if: "always() && needs.run-evals.result != 'skipped' && needs.prepare.outputs.pairs != '[]'" runs-on: ubuntu-latest steps: - name: Generate GitHub App token @@ -338,7 +348,6 @@ jobs: - name: Checkout uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3 with: - submodules: recursive ref: ${{ github.event_name == 'pull_request' && github.head_ref || github.ref }} token: ${{ steps.generate-token.outputs.token || github.token }} @@ -355,22 +364,56 @@ jobs: run: pnpm install --frozen-lockfile - name: Download raw results - # actions/download-artifact only nests a downloaded artifact under a - # directory named after it when there's more than one to disambiguate; - # with exactly one matching pair it dumps the contents flat into - # `path`, silently breaking the per-experiment glob below. Naming the - # destination directory ourselves side-steps that inconsistency. - env: - GH_TOKEN: ${{ github.token }} + # If every pair failed, run-evals never uploads a raw-results artifact + # at all, so this action errors. Audit pair results below already + # handles a missing/empty results/downloaded correctly (every pair + # reports missing), so don't let a total failure hide behind this. + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: raw-results + path: results/downloaded + + - name: Audit pair results + id: audit shell: bash run: | set -euo pipefail - mkdir -p downloaded-results pairs='${{ needs.prepare.outputs.pairs }}' - while IFS= read -r name; do - gh run download ${{ github.run_id }} --name "$name" --dir "downloaded-results/$name" - done < <(jq -r '.[] | "raw-results-\(.experiment)__\(.eval_id)"' <<< "$pairs" | sort -u) + total="$(jq 'length' <<< "$pairs")" + missing=() + while IFS= read -r key; do + [ -d "results/downloaded/raw-results-$key" ] || missing+=("$key") + done < <(jq -r '.[] | "\(.experiment)__\(.eval_id)"' <<< "$pairs") + + echo "count=${#missing[@]}" >> "$GITHUB_OUTPUT" + + if [ "${#missing[@]}" -eq "$total" ]; then + echo "::warning::No results published: all $total eval pairs never finished (run-evals likely hit its timeout or failed partway). This run needs a follow-up to complete the refresh." + { + echo "## :warning: No results published" + echo "" + echo "All $total eval pairs never finished, likely because \`run-evals\` hit its timeout or failed partway through:" + echo "" + echo '```' + printf '%s\n' "${missing[@]}" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + elif [ "${#missing[@]}" -gt 0 ]; then + echo "::warning::Publishing PARTIAL results: ${#missing[@]} of $total eval pairs never finished (run-evals likely hit its timeout or failed partway). This run needs a follow-up to complete the refresh." + { + echo "## :warning: Partial results published" + echo "" + echo "${#missing[@]} of $total eval pairs never finished, likely because \`run-evals\` hit its timeout or failed partway through. Only the completed pairs below were published:" + echo "" + echo '```' + printf '%s\n' "${missing[@]}" + echo '```' + echo "" + echo "Re-run this workflow to fill in the missing pairs." + } >> "$GITHUB_STEP_SUMMARY" + fi - name: Export results shell: bash @@ -381,7 +424,7 @@ jobs: pairs='${{ needs.prepare.outputs.pairs }}' while IFS= read -r experiment; do mkdir -p "results/$experiment" - for artifact_dir in "downloaded-results/raw-results-${experiment}__"*/; do + for artifact_dir in "results/downloaded/raw-results-${experiment}__"*/; do [ -d "$artifact_dir" ] && cp -R "$artifact_dir"/. "results/$experiment"/ done done < <(jq -r '.[].experiment' <<< "$pairs" | sort -u) @@ -462,7 +505,21 @@ jobs: - name: Merge scheduled results pull request # The app is on the ruleset's bypass list, so no review is required. - if: github.event_name == 'schedule' && steps.cpr.outputs.pull-request-number + # Skips the merge when results are partial, so it stays open for review. + if: >- + github.event_name == 'schedule' && + steps.cpr.outputs.pull-request-number && + steps.audit.outputs.count == '0' env: GH_TOKEN: ${{ steps.generate-token.outputs.token }} run: gh pr merge "${{ steps.cpr.outputs.pull-request-number }}" --squash --delete-branch + + - name: Require complete results + # Runs last so partial results still publish, but the job still fails. + if: >- + always() && + steps.audit.outcome == 'success' && + steps.audit.outputs.count != '0' + run: | + echo "::error::${{ steps.audit.outputs.count }} eval pair(s) never finished. This run needs a follow-up to complete the refresh." + exit 1 diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index c9b4e255..f2a00641 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -21,7 +21,9 @@ import { } from '@supabase-evals/sandbox'; import { normalizeExperimentName, + positiveInteger, readExperimentSuiteFilters, + readFlag, readRepeatedFlag, readSuiteFilters, } from '../lib/cli-args.js'; @@ -72,9 +74,15 @@ const SELECTED_EXPERIMENT_SUITE = EXPERIMENT_SUITE_FILTERS.length === 1 ? EXPERIMENT_SUITE_FILTERS[0] : undefined; -const RUNS = Number(readFlag('runs') ?? 1); -const TIMEOUT_SEC = Number(readFlag('timeout-sec') ?? 720); -const CONCURRENCY = Number(readFlag('concurrency') ?? 1); +const RUNS = positiveInteger(readFlag(rawArgs, 'runs') ?? '1', 'runs'); +const TIMEOUT_SEC = positiveInteger( + readFlag(rawArgs, 'timeout-sec') ?? '720', + 'timeout-sec' +); +const CONCURRENCY = positiveInteger( + readFlag(rawArgs, 'concurrency') ?? '1', + 'concurrency' +); const STOP_ON_PASS = !args.has('--run-all-attempts'); const DEBUG = args.has('--debug'); @@ -91,22 +99,6 @@ async function loadExperiments() { return out; } -function readFlag(name: string): string | undefined { - const prefix = `--${name}=`; - const inline = rawArgs.find((arg) => arg.startsWith(prefix)); - if (inline) return inline.slice(prefix.length); - - const idx = rawArgs.indexOf(`--${name}`); - if (idx !== -1) { - const value = rawArgs[idx + 1]; - if (!value || value.startsWith('--')) { - throw new Error(`--${name} requires a value`); - } - return value; - } - return undefined; -} - /** * Resolve the run mode. The sandbox (local-stack) is needed when the agent * uses the Supabase CLI (`interface: cli`) — including bootstrap scenarios that diff --git a/apps/framework/lib/cli-args.test.ts b/apps/framework/lib/cli-args.test.ts new file mode 100644 index 00000000..8b2f3a1a --- /dev/null +++ b/apps/framework/lib/cli-args.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from 'vitest'; +import { positiveInteger, readFlag } from './cli-args.js'; + +describe('readFlag', () => { + it('reads flags in both --name value and --name=value form', () => { + expect(readFlag(['--runs', '3'], 'runs')).toBe('3'); + expect(readFlag(['--runs=3'], 'runs')).toBe('3'); + expect(readFlag(['--other', 'x'], 'runs')).toBeUndefined(); + expect(() => readFlag(['--runs'], 'runs')).toThrow( + '--runs requires a value' + ); + expect(() => readFlag(['--runs', '--other'], 'runs')).toThrow( + '--runs requires a value' + ); + expect(() => readFlag(['--runs='], 'runs')).toThrow( + '--runs requires a value' + ); + }); +}); + +describe('positiveInteger', () => { + it('rejects non-positive-integer CLI options', () => { + expect(positiveInteger('3', 'runs')).toBe(3); + expect(() => positiveInteger('0', 'runs')).toThrow( + '--runs must be a positive integer' + ); + expect(() => positiveInteger('-1', 'runs')).toThrow( + '--runs must be a positive integer' + ); + expect(() => positiveInteger('1.5', 'runs')).toThrow( + '--runs must be a positive integer' + ); + expect(() => positiveInteger('abc', 'runs')).toThrow( + '--runs must be a positive integer' + ); + }); +}); diff --git a/apps/framework/lib/cli-args.ts b/apps/framework/lib/cli-args.ts index 23f4e36d..77eaf1e5 100644 --- a/apps/framework/lib/cli-args.ts +++ b/apps/framework/lib/cli-args.ts @@ -4,6 +4,34 @@ import { type EvalSuite, type ExperimentSuite, } from '@supabase-evals/core/eval-metadata'; +import { z } from 'zod'; + +const positiveIntegerSchema = z.coerce.number().int().min(1); + +/** Parses a positive integer CLI option. */ +export function positiveInteger(value: string, name: string): number { + const parsed = positiveIntegerSchema.safeParse(value); + if (!parsed.success) throw new Error(`--${name} must be a positive integer`); + return parsed.data; +} + +/** Reads one CLI flag in either `--name value` or `--name=value` form. */ +export function readFlag(rawArgs: string[], name: string): string | undefined { + const prefix = `--${name}=`; + const inline = rawArgs.find((arg) => arg.startsWith(prefix)); + if (inline) { + const value = inline.slice(prefix.length); + if (!value) throw new Error(`--${name} requires a value`); + return value; + } + const index = rawArgs.indexOf(`--${name}`); + if (index === -1) return undefined; + const value = rawArgs[index + 1]; + if (!value || value.startsWith('--')) { + throw new Error(`--${name} requires a value`); + } + return value; +} export function splitList(value: string): string[] { return value diff --git a/apps/framework/package.json b/apps/framework/package.json index c726bc37..5b282780 100644 --- a/apps/framework/package.json +++ b/apps/framework/package.json @@ -4,12 +4,14 @@ "version": "0.0.1", "type": "module", "scripts": { - "check": "pnpm typecheck && pnpm test:framework", + "check": "pnpm typecheck && pnpm test:framework && pnpm test:vercel-runner", "eval": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts", "eval:dry": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --dry", "eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke", + "eval:vercel": "node --env-file=../../.env --import tsx/esm scripts/run-vercel-evals.ts", "typecheck": "tsc --noEmit", "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts", + "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts lib/cli-args.test.ts", "export-results": "node --import tsx/esm scripts/export-results.ts", "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts", "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts" @@ -18,21 +20,25 @@ "@ai-sdk/anthropic": "catalog:", "@ai-sdk/mcp": "catalog:", "@ai-sdk/openai": "catalog:", - "@supabase-evals/platform-lite": "workspace:*", "@electric-sql/pglite": "catalog:", "@supabase-evals/core": "workspace:*", + "@supabase-evals/platform-lite": "workspace:*", "@supabase-evals/sandbox": "workspace:*", + "@supabase/lite": "catalog:", "@supabase/supabase-js": "catalog:", "@testing-library/jest-dom": "^6.9.1", "@testing-library/react": "^16.3.2", + "@vercel/sandbox": "^3.0.0", "@vitejs/plugin-react": "catalog:", "ai": "catalog:", "happy-dom": "^20.9.0", - "@supabase/lite": "catalog:", + "p-limit": "^7.3.1", + "p-retry": "^8.0.0", "react": "^19.2.5", "react-dom": "^19.2.5", "vite": "catalog:", - "vitest": "catalog:" + "vitest": "catalog:", + "zod": "catalog:" }, "devDependencies": { "@types/node": "catalog:", diff --git a/apps/framework/scripts/run-vercel-evals.test.ts b/apps/framework/scripts/run-vercel-evals.test.ts new file mode 100644 index 00000000..538c4407 --- /dev/null +++ b/apps/framework/scripts/run-vercel-evals.test.ts @@ -0,0 +1,108 @@ +import { APIError } from '@vercel/sandbox'; +import { describe, expect, it } from 'vitest'; +import { + isRetryableSandboxCreateError, + isTerminalSandboxCreateError, + parsePairs, + runBounded, + tagValue, +} from './run-vercel-evals.js'; + +describe('Vercel eval controller', () => { + it('bounds concurrent work and lets independent failures settle', async () => { + let active = 0; + let maximum = 0; + const results = await runBounded([1, 2, 3, 4], 2, async (item) => { + active += 1; + maximum = Math.max(maximum, active); + await new Promise((resolve) => setTimeout(resolve, 10)); + active -= 1; + if (item === 2) throw new Error('terminal'); + return item; + }); + + expect(maximum).toBe(2); + expect(results.map((result) => result.status)).toEqual([ + 'fulfilled', + 'rejected', + 'fulfilled', + 'fulfilled', + ]); + }); + + it('sanitizes Sandbox names and tags to the allowed charset', () => { + expect(tagValue('openai-gpt-5.4-nano')).toBe('openai-gpt-5-4-nano'); + expect(tagValue('Build CLI / Bootstrap App!')).toBe( + 'build-cli-bootstrap-app-' + ); + expect(tagValue('a'.repeat(100))).toHaveLength(64); + }); + + it('validates pair input before starting Sandboxes', () => { + expect( + parsePairs( + JSON.stringify([ + { + eval_id: 'eval-1', + experiment: 'experiment-1', + experiment_suite: 'benchmark', + eval_suite: 'benchmark', + }, + ]) + ) + ).toHaveLength(1); + expect(() => parsePairs('[{"eval_id":"eval-1"}]')).toThrow( + 'each pair must contain' + ); + }); + + it('retries sandbox creation only on 429s and 5xx API responses', () => { + const apiError = (status: number) => + new APIError(new Response(null, { status })); + expect(isRetryableSandboxCreateError(apiError(429), 1)).toBe(true); + expect(isRetryableSandboxCreateError(apiError(500), 1)).toBe(true); + expect(isRetryableSandboxCreateError(apiError(401), 1)).toBe(false); + expect(isRetryableSandboxCreateError(apiError(400), 1)).toBe(false); + }); + + it('retries a real network error through the full attempt budget', () => { + const networkError = new TypeError('fetch failed', { + cause: Object.assign(new Error('read ECONNRESET'), { + code: 'ECONNRESET', + }), + }); + expect(isRetryableSandboxCreateError(networkError, 1)).toBe(true); + expect(isRetryableSandboxCreateError(networkError, 12)).toBe(true); + }); + + it('caps an unrecognized error to a couple of attempts', () => { + const mystery = new Error('something we have never seen'); + expect(isRetryableSandboxCreateError(mystery, 1)).toBe(true); + expect(isRetryableSandboxCreateError(mystery, 2)).toBe(true); + expect(isRetryableSandboxCreateError(mystery, 3)).toBe(false); + }); + + it('marks definitive 4xx API responses as terminal', () => { + const apiError = (status: number) => + new APIError(new Response(null, { status })); + + expect(isTerminalSandboxCreateError(apiError(401))).toBe(true); + expect(isTerminalSandboxCreateError(apiError(400))).toBe(true); + }); + + it('does not mark retryable or unknown errors as terminal', () => { + const apiError = (status: number) => + new APIError(new Response(null, { status })); + const networkError = new TypeError('fetch failed', { + cause: Object.assign(new Error('read ECONNRESET'), { + code: 'ECONNRESET', + }), + }); + const mystery = new Error('something we have never seen'); + + expect(isTerminalSandboxCreateError(apiError(429))).toBe(false); + expect(isTerminalSandboxCreateError(apiError(500))).toBe(false); + expect(isTerminalSandboxCreateError(networkError)).toBe(false); + expect(isTerminalSandboxCreateError(mystery)).toBe(false); + }); +}); diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts new file mode 100644 index 00000000..0f21eb0f --- /dev/null +++ b/apps/framework/scripts/run-vercel-evals.ts @@ -0,0 +1,643 @@ +#!/usr/bin/env tsx + +import { APIError, Sandbox } from '@vercel/sandbox'; +import { execFile, execFileSync } from 'node:child_process'; +import { mkdtempSync, mkdirSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { promisify } from 'node:util'; +import pLimit from 'p-limit'; +import pRetry, { AbortError } from 'p-retry'; +import { z } from 'zod'; +import { positiveInteger, readFlag } from '../lib/cli-args.js'; + +const ROOT = fileURLToPath(new URL('../../../', import.meta.url)); +const execFileAsync = promisify(execFile); +/** Base for sandbox URLs printed during runs */ +const SANDBOX_DASHBOARD_URL = + 'https://vercel.com/supabase/evals-runner/sandboxes'; +const AGENT_ENV_NAMES = [ + 'ANTHROPIC_API_KEY', + 'OPENAI_API_KEY', + 'AI_GATEWAY_API_KEY', +]; +/** + * Slack for the non-agent work inside `pnpm eval` (supabase start, resets, + * scoring, export). Cold image pulls alone can take ~10 min. + */ +const EVAL_TIMEOUT_BUFFER_MS = 25 * 60 * 1_000; +/** + * Covers the sandbox steps outside the eval command timer (setup before it, + * pack/download after) so there's buffer time after eval timeout to read logs + * before the platform kills the sandbox. + */ +const SANDBOX_TIMEOUT_BUFFER_MS = 10 * 60 * 1_000; + +const evalPairSchema = z.object({ + eval_id: z.string(), + experiment: z.string(), + experiment_suite: z.string(), + eval_suite: z.string(), +}); +export type EvalPair = z.infer; + +interface RunnerOptions { + pairs: EvalPair[]; + revision: string; + repoUrl: string; + outputDir: string; + runs: number; + timeoutSec: number; + concurrency: number; + vcpus: number; +} + +interface PairOptions extends RunnerOptions { + pair: EvalPair; + attempt: number; +} + +interface SandboxCommandOptions { + cmd: string; + args?: string[]; + cwd?: string; + env?: Record; + sudo?: boolean; + timeoutMs?: number; +} + +class SandboxCommandError extends Error { + constructor(step: string, exitCode: number, output: string) { + const detail = output.trim().slice(-4_000); + super(`${step} exited with code ${exitCode}${detail ? `\n${detail}` : ''}`); + this.name = 'SandboxCommandError'; + } +} + +/** Runs every item while keeping at most `concurrency` promises active. */ +export async function runBounded( + items: readonly T[], + concurrency: number, + run: (item: T) => Promise +): Promise[]> { + const limit = pLimit(concurrency); + return Promise.allSettled(items.map((item) => limit(run, item))); +} + +/** Runs all pairs and reports failures only after independent work finishes. */ +async function runPairs(options: RunnerOptions): Promise { + const credentials = vercelCredentialsFromEnv(); + + const results = await runBounded( + options.pairs, + options.concurrency, + async (pair) => { + try { + await pRetry( + (attempt) => + runPairOnce( + { + ...options, + pair, + attempt, + }, + credentials + ), + { + retries: 2, + factor: 2, + minTimeout: 5_000, + maxTimeout: 30_000, + randomize: true, + onFailedAttempt: ({ + error, + attemptNumber, + retriesLeft, + retryDelay, + }) => { + const retrying = retriesLeft > 0; + console.warn( + `${pairLabel(pair)} attempt ${attemptNumber} failed${retrying ? `, retrying in ${Math.round(retryDelay / 1000)}s` : ', not retrying'}: ${firstLine(errorMessage(error))}` + ); + }, + } + ); + console.log(`SANDBOX OK ${pairLabel(pair)}`); + } catch (error) { + console.error( + `SANDBOX FAILED ${pairLabel(pair)}: ${errorMessage(error)}` + ); + throw error; + } + } + ); + + const failures: string[] = []; + for (let index = 0; index < results.length; index += 1) { + const pair = options.pairs[index]; + const result = results[index]; + if (!pair || !result || result.status === 'fulfilled') continue; + failures.push(`${pairLabel(pair)}: ${errorMessage(result.reason)}`); + } + + if (failures.length > 0) { + throw new AggregateError( + failures.map((message) => new Error(message)), + `${failures.length} Sandbox eval pair(s) failed` + ); + } +} + +/** Runs one pair in a fresh Sandbox and downloads its complete result tree. */ +async function runPairOnce( + options: PairOptions, + credentials: VercelCredentials +): Promise { + const { pair } = options; + const label = pairLabel(pair); + let sandbox: Sandbox | undefined; + + try { + sandbox = await createSandbox(label, { + ...credentials, + name: sandboxName(pair), + runtime: 'node24', + source: { + type: 'git', + url: options.repoUrl, + revision: options.revision, + depth: 1, + }, + resources: { vcpus: options.vcpus }, + timeout: + options.runs * options.timeoutSec * 1_000 + + EVAL_TIMEOUT_BUFFER_MS + + SANDBOX_TIMEOUT_BUFFER_MS, + persistent: false, + tags: { + runner: 'supabase-evals', + run: process.env.GITHUB_RUN_ID ?? 'local', + experiment: tagValue(pair.experiment), + eval: tagValue(pair.eval_id), + attempt: String(options.attempt), + }, + }); + console.log( + `${label} attempt ${options.attempt}: ${SANDBOX_DASHBOARD_URL}/${sandbox.name}` + ); + + await runSandboxCommand(sandbox, label, 'initialize submodules', { + cmd: 'git', + args: [ + '-c', + 'url.https://github.com/.insteadOf=git@github.com:', + 'submodule', + 'update', + '--init', + '--recursive', + ], + cwd: sandbox.cwd, + timeoutMs: 2 * 60 * 1_000, + }); + await runSandboxCommand(sandbox, label, 'install Docker', { + cmd: 'dnf', + args: ['install', '-y', '-q', 'docker'], + sudo: true, + timeoutMs: 3 * 60 * 1_000, + }); + await sandbox.runCommand({ + cmd: 'dockerd', + sudo: true, + detached: true, + }); + await runSandboxCommand(sandbox, label, 'start Docker', { + cmd: 'bash', + args: [ + '-c', + 'for i in $(seq 60); do docker info >/dev/null 2>&1 && chmod 666 /var/run/docker.sock && exit 0; sleep 1; done; echo "dockerd not ready" >&2; exit 1', + ], + sudo: true, + timeoutMs: 90_000, + }); + // Installs the pnpm version pinned by packageManager in the checked-out + // root package.json so the two can't drift. + await runSandboxCommand(sandbox, label, 'install pnpm', { + cmd: 'bash', + args: [ + '-c', + `npm install --global "$(node -p 'require("./package.json").packageManager')"`, + ], + cwd: sandbox.cwd, + sudo: true, + timeoutMs: 2 * 60 * 1_000, + }); + await runSandboxCommand(sandbox, label, 'install dependencies', { + cmd: 'pnpm', + args: ['install', '--frozen-lockfile'], + cwd: sandbox.cwd, + timeoutMs: 6 * 60 * 1_000, + }); + + await sandbox.writeFiles([ + { + path: '.env', + content: `${agentEnvironment()}\n`, + }, + ]); + console.log(`${label} run eval`); + await runSandboxCommand( + sandbox, + label, + 'run eval', + { + cmd: 'pnpm', + args: [ + 'eval', + '--', + '--experiment', + pair.experiment, + '--experiment-suite', + pair.experiment_suite, + '--eval', + pair.eval_id, + '--runs', + String(options.runs), + '--timeout-sec', + String(options.timeoutSec), + ], + cwd: sandbox.cwd, + timeoutMs: + options.runs * options.timeoutSec * 1_000 + EVAL_TIMEOUT_BUFFER_MS, + }, + true + ); + await runSandboxCommand(sandbox, label, 'validate result', { + cmd: 'test', + args: ['-f', `results/${pair.experiment}/${pair.eval_id}.json`], + cwd: sandbox.cwd, + timeoutMs: 30_000, + }); + await runSandboxCommand(sandbox, label, 'pack results', { + cmd: 'tar', + args: [ + '--exclude=*/node_modules', + '-czf', + '/tmp/eval-results.tgz', + '-C', + `results/${pair.experiment}`, + '.', + ], + cwd: sandbox.cwd, + timeoutMs: 3 * 60 * 1_000, + }); + await downloadResults(sandbox, pair, options.outputDir); + } catch (error) { + if (error instanceof Error) throw error; + throw new Error(String(error)); + } finally { + if (sandbox) await cleanupSandbox(sandbox, label); + } +} + +/** + * Retries Sandbox.create() through the vCPU-provisioning rate limit. + * The SDK's own retry gives up after ~3 attempts or a >20s Retry-After, + * which isn't enough to ride out a large burst, so this layer takes over + * with a much bigger budget and honors the same Retry-After header. + */ +async function createSandbox( + label: string, + createOptions: Parameters[0] +): Promise { + try { + return await pRetry(() => Sandbox.create(createOptions), { + retries: 12, + minTimeout: 0, + shouldRetry: ({ error, attemptNumber }) => + isRetryableSandboxCreateError(error, attemptNumber), + onFailedAttempt: async ({ error, attemptNumber, retriesLeft }) => { + const retrying = + retriesLeft > 0 && + isRetryableSandboxCreateError(error, attemptNumber); + if (isUnrecognizedSandboxCreateError(error)) { + console.warn( + `${label} sandbox create hit an unrecognized error type (capped at ${UNKNOWN_ERROR_RETRY_LIMIT} attempts) - consider adding it to isRetryableSandboxCreateError: ${errorMessage(error)}` + ); + } + const retryAfter = getRetryAfterMs(error); + // Jitter so on top of server's `Retry-After` so concurrent pairs hitting + // the same rate limit don't all retry in lockstep. + const wait = retryAfter + ? retryAfter * (1 + Math.random() * 0.4) + : Math.min(2 ** attemptNumber * 1_000, 20_000) * + (0.8 + Math.random() * 0.4); + console.warn( + `${label} sandbox create attempt ${attemptNumber} failed${retrying ? `, retrying in ${Math.round(wait / 1_000)}s` : ', not retrying'}: ${errorMessage(error)}` + ); + if (retrying) await new Promise((resolve) => setTimeout(resolve, wait)); + }, + }); + } catch (error) { + // AbortError stops any pRetry call it bubbles through, so this also + // keeps runPairs' outer retry from re-attempting a doomed sandbox create. + if (isTerminalSandboxCreateError(error) && error instanceof Error) { + throw new AbortError(error); + } + throw error; + } +} + +const UNKNOWN_ERROR_RETRY_LIMIT = 2; + +const NETWORK_ERROR_CODES = new Set([ + 'ECONNRESET', + 'ECONNREFUSED', + 'ETIMEDOUT', + 'ENOTFOUND', + 'EAI_AGAIN', + 'EPIPE', +]); + +/** + * Detects a real network fault surfaced through fetch's TypeError wrapper. + * https://nodejs.org/api/errors.html#nodejs-error-codes + */ +function isNetworkError(error: unknown): boolean { + return ( + error instanceof TypeError && + error.cause instanceof Error && + 'code' in error.cause && + NETWORK_ERROR_CODES.has(String(error.cause.code)) + ); +} + +/** + * Based on the SDK's retry policy for API responses (429s and 5xx): + * https://github.com/vercel/sandbox/blob/bf2bc66003fc89cf07a1346a7ea63951747cbec6/packages/vercel-sandbox/src/api-client/with-retry.ts#L10-L17 + * + * Other known network faults get the full retry budget. Anything unrecognized + * gets a few retries and warning on job since we can't tell if it's transient. + */ +export function isRetryableSandboxCreateError( + error: unknown, + attemptNumber: number +): boolean { + if (error instanceof APIError) { + const { status } = error.response; + return status === 429 || status >= 500; + } + if (isNetworkError(error)) return true; + return attemptNumber <= UNKNOWN_ERROR_RETRY_LIMIT; +} + +/** True for a non-APIError that isn't a known network error. */ +function isUnrecognizedSandboxCreateError(error: unknown): boolean { + return !(error instanceof APIError) && !isNetworkError(error); +} + +/** True when no amount of retrying at any level could fix this. */ +export function isTerminalSandboxCreateError(error: unknown): boolean { + return error instanceof APIError && !isRetryableSandboxCreateError(error, 1); +} + +/** + * Reads the Sandbox API's Retry-After header (seconds), converted to milliseconds. + * The SDK's own retry layer reads the same header on 429s: + * https://github.com/vercel/sandbox/blob/bf2bc66003fc89cf07a1346a7ea63951747cbec6/packages/vercel-sandbox/src/api-client/with-retry.ts#L56 + */ +function getRetryAfterMs(error: unknown): number | undefined { + if (!(error instanceof APIError)) return undefined; + const seconds = Number(error.response.headers.get('Retry-After')); + return seconds > 0 ? seconds * 1_000 : undefined; +} + +/** Runs a detached command, streams logs best-effort, and waits for its exit code. */ +async function runSandboxCommand( + sandbox: Sandbox, + label: string, + step: string, + options: SandboxCommandOptions, + logOutput = false +): Promise { + const command = await sandbox.runCommand({ + ...options, + detached: true, + }); + const result = await pRetry(() => command.wait(), { + retries: 5, + factor: 2, + minTimeout: 1_000, + maxTimeout: 10_000, + onFailedAttempt: ({ error, attemptNumber }) => { + console.warn( + `${label} ${step} wait failed (attempt ${attemptNumber}): ${errorMessage(error)}` + ); + }, + }); + if (result.exitCode === 0) { + if (logOutput) { + const output = await result.output('both').catch(() => ''); + if (output.trim()) process.stdout.write(`${label} ${output}`); + } + return; + } + + const output = await result.output('both').catch(() => ''); + throw new SandboxCommandError(step, result.exitCode, output); +} + +/** Downloads and extracts one pair into the aggregate artifact directory. */ +async function downloadResults( + sandbox: Sandbox, + pair: EvalPair, + outputDir: string +): Promise { + const staging = mkdtempSync(join(tmpdir(), 'vercel-eval-results-')); + const archive = join(staging, 'results.tgz'); + const destination = join(outputDir, artifactDirectory(pair)); + try { + const downloaded = await sandbox.downloadFile( + { path: '/tmp/eval-results.tgz' }, + { path: archive }, + { mkdirRecursive: true } + ); + if (!downloaded) throw new Error('results archive was missing'); + rmSync(destination, { recursive: true, force: true }); + mkdirSync(destination, { recursive: true }); + await execFileAsync('tar', ['-xzf', archive, '-C', destination]); + console.log(`${pairLabel(pair)} results downloaded to ${destination}`); + } finally { + rmSync(staging, { recursive: true, force: true }); + } +} + +/** Stops and deletes a Sandbox while preserving the pair's original outcome. */ +async function cleanupSandbox(sandbox: Sandbox, label: string): Promise { + try { + await sandbox.stop(); + console.log(`${label} sandbox ${sandbox.name} stopped`); + } catch (error) { + console.warn(`${label} sandbox stop failed: ${errorMessage(error)}`); + } + try { + await sandbox.delete(); + console.log(`${label} sandbox ${sandbox.name} deleted`); + } catch (error) { + console.warn(`${label} sandbox delete failed: ${errorMessage(error)}`); + } +} + +/** Parses and validates the pair list supplied by GitHub Actions. */ +export function parsePairs(value: string): EvalPair[] { + const json: unknown = JSON.parse(value); + if (!Array.isArray(json) || json.length === 0) { + throw new Error('--pairs-json must be a non-empty JSON array'); + } + const parsed = z.array(evalPairSchema).safeParse(json); + if (!parsed.success) { + throw new Error( + 'each pair must contain eval_id, experiment, experiment_suite, and eval_suite strings' + ); + } + return parsed.data; +} + +/** Returns an environment variable or a useful configuration error. */ +function requireEnv(name: string, hint: string): string { + const value = process.env[name]; + if (!value) throw new Error(`${name} is not set: ${hint}`); + return value; +} + +type VercelCredentials = ReturnType; + +/** Returns explicit credentials because the SDK does not infer all local vars. */ +function vercelCredentialsFromEnv(): { + token: string; + teamId: string; + projectId: string; +} { + return { + token: requireEnv('VERCEL_TOKEN', 'missing Vercel token'), + teamId: requireEnv('VERCEL_TEAM_ID', 'missing Vercel team ID'), + projectId: requireEnv('VERCEL_PROJECT_ID', 'missing Vercel project ID'), + }; +} + +/** Serializes configured provider keys into the repo-root `.env` file. */ +function agentEnvironment(): string { + const lines: string[] = []; + for (const name of AGENT_ENV_NAMES) { + const value = process.env[name]; + if (value) lines.push(`${name}=${value}`); + } + return lines.join('\n'); +} + +/** Converts the origin remote into a Vercel-compatible HTTPS clone URL. */ +function repositoryUrl(): string { + return execFileSync('git', ['remote', 'get-url', 'origin'], { + cwd: ROOT, + encoding: 'utf8', + }) + .trim() + .replace(/^git@github\.com:/, 'https://github.com/'); +} + +/** Reads the checked-out commit used when no explicit revision is supplied. */ +function currentRevision(): string { + return execFileSync('git', ['rev-parse', 'HEAD'], { + cwd: ROOT, + encoding: 'utf8', + }).trim(); +} + +/** Builds the stable per-pair directory expected by the publish job. */ +function artifactDirectory(pair: EvalPair): string { + return `raw-results-${pair.experiment}__${pair.eval_id}`; +} + +/** Formats a pair consistently in interleaved controller output. */ +function pairLabel(pair: EvalPair): string { + return `[${pair.experiment} x ${pair.eval_id}]`; +} + +/** Produces a unique dashboard-safe Sandbox name. */ +function sandboxName(pair: EvalPair): string { + const suffix = Math.random().toString(36).slice(2, 8); + return `${tagValue(pair.experiment).slice(0, 35)}--${tagValue(pair.eval_id).slice(0, 45)}--${suffix}`; +} + +/** Sanitizes metadata for Sandbox names and tags. */ +export function tagValue(value: string): string { + return value + .toLowerCase() + .replaceAll(/[^a-z0-9-]+/g, '-') + .slice(0, 64); +} + +const apiErrorBodySchema = z.object({ + error: z.object({ code: z.string(), message: z.string() }), +}); + +/** Converts unknown thrown values into readable diagnostics. */ +function errorMessage(error: unknown): string { + if (error instanceof APIError) { + const body = apiErrorBodySchema.safeParse(error.json); + const detail = body.success + ? `${body.data.error.code}: ${body.data.error.message}` + : error.message; + return `HTTP ${error.response.status} ${detail}`; + } + return error instanceof Error ? error.message : String(error); +} + +/** Keeps retry notices readable when the final summary carries full output. */ +function firstLine(value: string): string { + return value.split('\n', 1)[0] ?? value; +} + +/** Parses CLI inputs and starts the controller. */ +async function main(): Promise { + const rawArgs = process.argv.slice(2).filter((arg) => arg !== '--'); + const pairsValue = readFlag(rawArgs, 'pairs-json') ?? process.env.EVAL_PAIRS; + if (!pairsValue) throw new Error('--pairs-json or EVAL_PAIRS is required'); + + const options: RunnerOptions = { + pairs: parsePairs(pairsValue), + revision: readFlag(rawArgs, 'revision') ?? currentRevision(), + repoUrl: readFlag(rawArgs, 'repo-url') ?? repositoryUrl(), + outputDir: resolve( + ROOT, + readFlag(rawArgs, 'output-dir') ?? 'results/downloaded' + ), + runs: positiveInteger(readFlag(rawArgs, 'runs') ?? '2', 'runs'), + timeoutSec: positiveInteger( + readFlag(rawArgs, 'timeout-sec') ?? '720', + 'timeout-sec' + ), + concurrency: positiveInteger( + readFlag(rawArgs, 'concurrency') ?? '4', + 'concurrency' + ), + vcpus: positiveInteger(readFlag(rawArgs, 'vcpus') ?? '4', 'vcpus'), + }; + + console.log( + `${options.pairs.length} pair(s), concurrency=${options.concurrency}, runs=${options.runs}, timeout=${options.timeoutSec}s, revision=${options.revision.slice(0, 8)}` + ); + for (const pair of options.pairs) console.log(`PLAN ${pairLabel(pair)}`); + if (rawArgs.includes('--dry-run')) return; + await runPairs(options); +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + main().catch((error) => { + console.error(errorMessage(error)); + process.exitCode = 1; + }); +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a16e5712..6e34baa1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -126,6 +126,9 @@ importers: '@testing-library/react': specifier: ^16.3.2 version: 16.3.2(@testing-library/dom@10.4.1)(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(react-dom@19.2.7(react@19.2.7))(react@19.2.7) + '@vercel/sandbox': + specifier: ^3.0.0 + version: 3.0.0 '@vitejs/plugin-react': specifier: 'catalog:' version: 5.2.0(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)) @@ -135,6 +138,12 @@ importers: happy-dom: specifier: ^20.9.0 version: 20.10.2 + p-limit: + specifier: ^7.3.1 + version: 7.3.1 + p-retry: + specifier: ^8.0.0 + version: 8.0.0 react: specifier: ^19.2.5 version: 19.2.7 @@ -147,6 +156,9 @@ importers: vitest: specifier: 'catalog:' version: 4.1.8(@opentelemetry/api@1.9.1)(@types/node@22.19.20)(happy-dom@20.10.2)(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)) + zod: + specifier: 'catalog:' + version: 4.4.3 devDependencies: '@types/node': specifier: 'catalog:' @@ -595,28 +607,24 @@ packages: engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] - libc: [musl] '@biomejs/cli-linux-arm64@1.9.4': resolution: {integrity: sha512-fJIW0+LYujdjUgJJuwesP4EjIBl/N/TcOX3IvIHJQNsAqvV2CHIogsmA94BPG6jZATS4Hi+xv4SkBBQSt1N4/g==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] - libc: [glibc] '@biomejs/cli-linux-x64-musl@1.9.4': resolution: {integrity: sha512-gEhi/jSBhZ2m6wjV530Yy8+fNqG8PAinM3oV7CyO+6c3CEh16Eizm21uHVsyVBEB6RIM8JHIl6AGYCv6Q6Q9Tg==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] - libc: [musl] '@biomejs/cli-linux-x64@1.9.4': resolution: {integrity: sha512-lRCJv/Vi3Vlwmbd6K+oQ0KhLHMAysN8lXoCI7XeHlxaajk06u7G+UsFSO01NAs5iYuWKmVZjmiOzJ0OJmGsMwg==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] - libc: [glibc] '@biomejs/cli-win32-arm64@1.9.4': resolution: {integrity: sha512-tlbhLk+WXZmgwoIKwHIHEBZUwxml7bRJgk0X2sPyNR3S93cdRq6XulAZRQJ17FYGGzWne0fgrXBKpl7l4M87Hg==} @@ -1932,79 +1940,66 @@ packages: resolution: {integrity: sha512-Q8CBCCQtDFrYtXoeUXSrnFXKOnyUhx6bz+SkL6A0E7V8kAiCJ5pamq1WtbfpVGhR5TSpXY6ak3avmDc5fHTyJA==} cpu: [arm] os: [linux] - libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.61.1': resolution: {integrity: sha512-nwnhk1581l0FBVellGcVCAT0Oi06onEA3WB53sf01VO3I0UPBkMH9sXONYME2K0ovXcNayJfNtHfm6mpJElatQ==} cpu: [arm] os: [linux] - libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.61.1': resolution: {integrity: sha512-x5Xr49hwt3hdW75UOZm3395YwwzPyauktslv29KpWL/T+vVAzoT3azLcTWv0eMciBNrx+DYjH4paehHoLpPvpg==} cpu: [arm64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.61.1': resolution: {integrity: sha512-unMS3H73DpaoPyyEVPjGKleM/s0mkmsauTENpw4INQY8y4+IuLNjkueQ5QCtC0D3N38Y38yhAU8OoZ20S2Tm6w==} cpu: [arm64] os: [linux] - libc: [musl] '@rollup/rollup-linux-loong64-gnu@4.61.1': resolution: {integrity: sha512-zNZzGRnAhwjFEYmvphJRV5XaQGjs62cCmeYYHUT//NbvEnHauw+I85nGG+SiVg5ld4GX8D1IbKIX+ozITQnhMQ==} cpu: [loong64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-loong64-musl@4.61.1': resolution: {integrity: sha512-LdpWGL8X209B2SIvWjqlc8VZgM6PKfontSerGepuldQmHYrAOtnMCXeJkxXGbC+PPZVOuu5czJo7fNV6aeW8rQ==} cpu: [loong64] os: [linux] - libc: [musl] '@rollup/rollup-linux-ppc64-gnu@4.61.1': resolution: {integrity: sha512-EC5kTtNaNGOmbMGqar8dvJy6y/hg99GAwjfBz++pxZhQATXGcRjd6c5en5wcbru0vkRmiMGsQKdMJOOf6sza4g==} cpu: [ppc64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-ppc64-musl@4.61.1': resolution: {integrity: sha512-8hiwp6D4acEcNK78I4rP0/XtS1sknWIAMJBPdR4l6zUtyTm5KiTDr5bXmWt4foY7nAN7AThDHgkLIEZOWKbzWw==} cpu: [ppc64] os: [linux] - libc: [musl] '@rollup/rollup-linux-riscv64-gnu@4.61.1': resolution: {integrity: sha512-10dh/h/BqA7DuMPWSxkR8uks18FRwnwOEqr5zOTEl+NOwP/OMzKX8OFR/Of9xxDA7D5qef1Nzar5WDD2kCCr1g==} cpu: [riscv64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.61.1': resolution: {integrity: sha512-YKJ5lg35DP17gcAOggnihe+APw9HLyj1Xn7gsmGumBJAUDa6NGXNixJzmkWLhcK9TOuuyQjdamzvJefkO7qHZQ==} cpu: [riscv64] os: [linux] - libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.61.1': resolution: {integrity: sha512-Mlil5G2Jj6a7B3LWGctg+XPL9vdXYuzCtNXfxOQ0nPjc2m6ueUktocPGH9bnAM0bNRKb/bAWTujUU7IJQdQA+g==} cpu: [s390x] os: [linux] - libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.61.1': resolution: {integrity: sha512-bVWIOIk6pV01p4CdUbPP7CJ/434z+OooYjDuFcR+44N35YvKUC66G8MGnvcWx5mWKW3g61J+t74l3Kj15Kwn2Q==} cpu: [x64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-x64-musl@4.61.1': resolution: {integrity: sha512-qy5pBvZbqNFheBz61R1rzsezjm0J7O2oNGoWtGoY89SZYLUfxAJTBAqDChqAIdB4rCiIbi9nF7yZ83GnNiLwSw==} cpu: [x64] os: [linux] - libc: [musl] '@rollup/rollup-openbsd-x64@4.61.1': resolution: {integrity: sha512-E83TXjI4zm0+5f2qO+UOudaCYIhYwpJ5jq6YCZNIZ+6CbfhKrkAGezeiASBL9ElxAxFsRS9ZhESv8mfnj6TKeg==} @@ -2199,28 +2194,24 @@ packages: engines: {node: '>= 20'} cpu: [arm64] os: [linux] - libc: [glibc] '@tailwindcss/oxide-linux-arm64-musl@4.3.0': resolution: {integrity: sha512-Z6sukiQsngnWO+l39X4pPbiWT81IC+PLKF+PHxIlyZbGNb9MODfYlXEVlFvej5BOZInWX01kVyzeLvHsXhfczQ==} engines: {node: '>= 20'} cpu: [arm64] os: [linux] - libc: [musl] '@tailwindcss/oxide-linux-x64-gnu@4.3.0': resolution: {integrity: sha512-DRNdQRpSGzRGfARVuVkxvM8Q12nh19l4BF/G7zGA1oe+9wcC6saFBHTISrpIcKzhiXtSrlSrluCfvMuledoCTQ==} engines: {node: '>= 20'} cpu: [x64] os: [linux] - libc: [glibc] '@tailwindcss/oxide-linux-x64-musl@4.3.0': resolution: {integrity: sha512-Z0IADbDo8bh6I7h2IQMx601AdXBLfFpEdUotft86evd/8ZPflZe9COPO8Q1vw+pfLWIUo9zN/JGZvwuAJqduqg==} engines: {node: '>= 20'} cpu: [x64] os: [linux] - libc: [musl] '@tailwindcss/oxide-wasm32-wasi@4.3.0': resolution: {integrity: sha512-HNZGOUxEmElksYR7S6sC5jTeNGpobAsy9u7Gu0AskJ8/20FR9GqebUyB+HBcU/ax6BHuiuJi+Oda4B+YX6H1yA==} @@ -2436,6 +2427,9 @@ packages: resolution: {integrity: sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==} engines: {node: '>= 20'} + '@vercel/sandbox@3.0.0': + resolution: {integrity: sha512-1pF3id7LIG2GfjkEAZW+ZngMDdywJw7aFLOyIlry/lj8v3b4GMn3WCuBbvG4N4wTmYUfFzHaVHfdwVpqALEFkw==} + '@vitejs/plugin-react@5.2.0': resolution: {integrity: sha512-YmKkfhOAi3wsB1PhJq5Scj3GXMn3WvtQ/JC0xoopuHoXSdmtdStOpFrYaT1kie2YgFBcIe64ROzMYRjCrYOdYw==} engines: {node: ^20.19.0 || >=22.12.0} @@ -2471,6 +2465,9 @@ packages: '@vitest/utils@4.1.8': resolution: {integrity: sha512-uOJamYALNhfJ6iolExyQM40yIQwDqYnkKtQ5VCiSe17E33H0aQ/u+1GlRuz4LZBk6Mm3sg90G9hEbmEt37C1Zg==} + '@workflow/serde@4.1.0-beta.2': + resolution: {integrity: sha512-8kkeoQKLDaKXefjV5dbhBj2aErfKp1Mc4pb6tj8144cF+Em5SPbyMbyLCHp+BVrFfFVCBluCtMx+jjvaFVZGww==} + accepts@2.0.0: resolution: {integrity: sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==} engines: {node: '>= 0.6'} @@ -2559,6 +2556,17 @@ packages: resolution: {integrity: sha512-6t10qk83GOG8p0vKmaCr8eiilZwO171AvbROMtvvNiwrTly62t+7XkA8RdIIVbpMhCASAsxgAzdRSwh6nw/5Dg==} engines: {node: '>=4'} + async-retry@1.3.3: + resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==} + + b4a@1.8.1: + resolution: {integrity: sha512-aiqre1Nr0B/6DgE2N5vwTc+2/oQZ4Wh1t4NznYY4E00y8LCt6NqdRv81so00oo27D8MVKTpUa/MwUUtBLXCoDw==} + peerDependencies: + react-native-b4a: '*' + peerDependenciesMeta: + react-native-b4a: + optional: true + balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} @@ -2566,6 +2574,14 @@ packages: resolution: {integrity: sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==} engines: {node: 18 || 20 || >=22} + bare-events@2.9.1: + resolution: {integrity: sha512-Z0oHEHAFDZkffN8Qc39zNZjQlMDkPJRyyyZieU1VH7u8c5S+qHZ2S8ixdKIAxEjfHO7FJxXmJWgteOghVanIsg==} + peerDependencies: + bare-abort-controller: '*' + peerDependenciesMeta: + bare-abort-controller: + optional: true + baseline-browser-mapping@2.10.35: resolution: {integrity: sha512-honAfLBde0HAFLdNyBEfuuENkF6zR+ozxqxa/2zJKHBe1qzLqyTSeRKpdPEHAP03rlDGyQOPnCSxnVpVqQo9Mg==} engines: {node: '>=6.0.0'} @@ -2959,6 +2975,9 @@ packages: resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} engines: {node: '>= 0.6'} + events-universal@1.0.1: + resolution: {integrity: sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==} + eventsource-parser@3.1.0: resolution: {integrity: sha512-kJezFj9YFAMLeORyi7aCLxLbD5/qWMQnoMVlVPyHIll7lgRJCc3JVln9Vgl9nwQi0YkMnhdGTMNn7CkRRAptMg==} engines: {node: '>=18.0.0'} @@ -3041,6 +3060,9 @@ packages: fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + fast-fifo@1.3.2: + resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} + fast-glob@3.3.3: resolution: {integrity: sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==} engines: {node: '>=8.6.0'} @@ -3316,6 +3338,10 @@ packages: resolution: {integrity: sha512-qP1vozQRI+BMOPcjFzrjXuQvdak2pHNUMZoeG2eRbiSqyvbEf/wQtEOTOX1guk6E3t36RkaqiSt8A/6YElNxLQ==} engines: {node: '>=12'} + is-network-error@1.3.2: + resolution: {integrity: sha512-PhBY86zaxNZUuWP6h13Vu5oFe0XY6/UlKzQnYFELzGVHygP3MxmvTfYSG7GN3aIab/iWudSMgjSnG9Dq+nHrgA==} + engines: {node: '>=16'} + is-number@7.0.0: resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} engines: {node: '>=0.12.0'} @@ -3426,6 +3452,9 @@ packages: jsonfile@6.2.1: resolution: {integrity: sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==} + jsonlines@0.1.1: + resolution: {integrity: sha512-ekDrAGso79Cvf+dtm+mL8OBI2bmAOt3gssYs833De/C9NmIpWDWyUO4zPgB5x2/OhY366dkhgfPMYfwZF7yOZA==} + keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} @@ -3502,28 +3531,24 @@ packages: engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] - libc: [glibc] lightningcss-linux-arm64-musl@1.32.0: resolution: {integrity: sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==} engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] - libc: [musl] lightningcss-linux-x64-gnu@1.32.0: resolution: {integrity: sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] - libc: [glibc] lightningcss-linux-x64-musl@1.32.0: resolution: {integrity: sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] - libc: [musl] lightningcss-win32-arm64-msvc@1.32.0: resolution: {integrity: sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==} @@ -3762,14 +3787,26 @@ packages: resolution: {integrity: sha512-weP+BZ8MVNnlCm8c0Qdc1WSWq4Qn7I+9CJGm7Qali6g44e/PUzbjNqJX5NJ9ljlNMosfJvg1fKEGILklK9cwnw==} engines: {node: '>=18'} + os-paths@4.4.0: + resolution: {integrity: sha512-wrAwOeXp1RRMFfQY8Sy7VaGVmPocaLwSFOYCGKSyo8qmJ+/yaafCl5BCA1IQZWqFSRBrKDYFeR9d/VyQzfH/jg==} + engines: {node: '>= 6.0'} + p-limit@3.1.0: resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} engines: {node: '>=10'} + p-limit@7.3.1: + resolution: {integrity: sha512-0trZaiG7Y7kN/Egy9a8j47t9osC0Tch4PaIWd9yGF6bvmlk7muExRvGNYb8sXBwEKMoNKsbNN9P8EefuQekE4Q==} + engines: {node: '>=20'} + p-locate@5.0.0: resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} engines: {node: '>=10'} + p-retry@8.0.0: + resolution: {integrity: sha512-kFVqH1HxOHp8LupNsOys7bSV09VYTRLxarH/mokO4Rqhk6wGi70E0jh4VzvVGXfEVNggHoHLAMWsQqHyU1Ey9A==} + engines: {node: '>=22'} + parent-module@1.0.1: resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} engines: {node: '>=6'} @@ -4093,6 +4130,10 @@ packages: resolution: {integrity: sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==} engines: {node: '>=18'} + retry@0.13.1: + resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} + engines: {node: '>= 4'} + reusify@1.1.0: resolution: {integrity: sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} @@ -4216,6 +4257,9 @@ packages: resolution: {integrity: sha512-UhDfHmA92YAlNnCfhmq0VeNL5bDbiZGg7sZ2IvPsXubGkiNa9EC+tUTsjBRsYUAz87btI6/1wf4XoVvQ3uRnmQ==} engines: {node: '>=18'} + streamx@2.28.0: + resolution: {integrity: sha512-1Yowhzjf0ivGMrTIkY9hav5TxobO9qIVqUE41fiCGMGgc3CLlf4MY+9AHmZqBWgDTue0fY9zWjYFVyf6Diuobw==} + strfy-js@3.2.2: resolution: {integrity: sha512-hUgJ5k2PR1ivhq4uObxnin5j6GcOr0Y0N1lzi3z6SRhxNqu4rzpDfyoC2ToUAyM8yXNXM0zs6f4KIiqj8NqheQ==} @@ -4277,6 +4321,12 @@ packages: resolution: {integrity: sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==} engines: {node: '>=6'} + tar-stream@3.1.7: + resolution: {integrity: sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==} + + text-decoder@1.2.7: + resolution: {integrity: sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==} + tiny-invariant@1.3.3: resolution: {integrity: sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==} @@ -4360,6 +4410,10 @@ packages: undici-types@7.24.6: resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} + undici@7.29.0: + resolution: {integrity: sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==} + engines: {node: '>=20.18.1'} + unicorn-magic@0.3.0: resolution: {integrity: sha512-+QBBXBCvifc56fsbuxZQ6Sic3wqqc3WWaqxs58gvJrcOuN83HGTCwz3oS5phzU9LthRNE9VrJCFCLUgHeeFnfA==} engines: {node: '>=18'} @@ -4546,6 +4600,14 @@ packages: resolution: {integrity: sha512-g/eziiSUNBSsdDJtCLB8bdYEUMj4jR7AGeUo96p/3dTafgjHhpF4RiCFPiRILwjQoDXx5MqkBr4fwWtR3Ky4Wg==} engines: {node: '>=20'} + xdg-app-paths@5.1.0: + resolution: {integrity: sha512-RAQ3WkPf4KTU1A8RtFx3gWywzVKe00tfOPFfl2NDGqbIFENQO4kqAJp7mhQjNj/33W5x5hiWWUdyfPq/5SU3QA==} + engines: {node: '>=6'} + + xdg-portable@7.3.0: + resolution: {integrity: sha512-sqMMuL1rc0FmMBOzCpd0yuy9trqF2yTTVe+E9ogwCSWQCdDEtQUwrZPT6AxqtsFGRNxycgncbP/xmOOSPw5ZUw==} + engines: {node: '>= 6.0'} + xtend@4.0.2: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} @@ -4564,6 +4626,10 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} + yocto-queue@1.2.2: + resolution: {integrity: sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==} + engines: {node: '>=12.20'} + yocto-spinner@1.2.0: resolution: {integrity: sha512-Yw0hUB6UA3o4YUgKy3oSe9a4cxoaZ9sBfYDw+JSxo6Id0KoJGoxzPA24qqUXYKBWABs/zDSGTz9kww7t3F0XGw==} engines: {node: '>=18.19'} @@ -6601,6 +6667,23 @@ snapshots: '@vercel/oidc@3.2.0': {} + '@vercel/sandbox@3.0.0': + dependencies: + '@vercel/oidc': 3.2.0 + '@workflow/serde': 4.1.0-beta.2 + async-retry: 1.3.3 + jose: 6.2.3 + jsonlines: 0.1.1 + ms: 2.1.3 + picocolors: 1.1.1 + tar-stream: 3.1.7 + undici: 7.29.0 + xdg-app-paths: 5.1.0 + zod: 4.4.3 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + '@vitejs/plugin-react@5.2.0(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4))': dependencies: '@babel/core': 7.29.7 @@ -6662,6 +6745,8 @@ snapshots: convert-source-map: 2.0.0 tinyrainbow: 3.1.0 + '@workflow/serde@4.1.0-beta.2': {} + accepts@2.0.0: dependencies: mime-types: 3.0.2 @@ -6739,10 +6824,18 @@ snapshots: dependencies: tslib: 2.8.1 + async-retry@1.3.3: + dependencies: + retry: 0.13.1 + + b4a@1.8.1: {} + balanced-match@1.0.2: {} balanced-match@4.0.4: {} + bare-events@2.9.1: {} + baseline-browser-mapping@2.10.35: {} bcryptjs@3.0.3: {} @@ -7142,6 +7235,12 @@ snapshots: etag@1.8.1: {} + events-universal@1.0.1: + dependencies: + bare-events: 2.9.1 + transitivePeerDependencies: + - bare-abort-controller + eventsource-parser@3.1.0: {} eventsource@3.0.7: @@ -7256,6 +7355,8 @@ snapshots: fast-deep-equal@3.1.3: {} + fast-fifo@1.3.2: {} + fast-glob@3.3.3: dependencies: '@nodelib/fs.stat': 2.0.5 @@ -7506,6 +7607,8 @@ snapshots: is-interactive@2.0.0: {} + is-network-error@1.3.2: {} + is-number@7.0.0: {} is-obj@3.0.0: {} @@ -7582,6 +7685,8 @@ snapshots: optionalDependencies: graceful-fs: 4.2.11 + jsonlines@0.1.1: {} + keyv@4.5.4: dependencies: json-buffer: 3.0.1 @@ -7843,14 +7948,24 @@ snapshots: string-width: 7.2.0 strip-ansi: 7.2.0 + os-paths@4.4.0: {} + p-limit@3.1.0: dependencies: yocto-queue: 0.1.0 + p-limit@7.3.1: + dependencies: + yocto-queue: 1.2.2 + p-locate@5.0.0: dependencies: p-limit: 3.1.0 + p-retry@8.0.0: + dependencies: + is-network-error: 1.3.2 + parent-module@1.0.1: dependencies: callsites: 3.1.0 @@ -8169,6 +8284,8 @@ snapshots: onetime: 7.0.0 signal-exit: 4.1.0 + retry@0.13.1: {} + reusify@1.1.0: {} rollup@4.61.1: @@ -8362,6 +8479,15 @@ snapshots: stdin-discarder@0.2.2: {} + streamx@2.28.0: + dependencies: + events-universal: 1.0.1 + fast-fifo: 1.3.2 + text-decoder: 1.2.7 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + strfy-js@3.2.2: dependencies: minimatch: 10.2.5 @@ -8412,6 +8538,21 @@ snapshots: tapable@2.3.3: {} + tar-stream@3.1.7: + dependencies: + b4a: 1.8.1 + fast-fifo: 1.3.2 + streamx: 2.28.0 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + + text-decoder@1.2.7: + dependencies: + b4a: 1.8.1 + transitivePeerDependencies: + - react-native-b4a + tiny-invariant@1.3.3: {} tinybench@2.9.0: {} @@ -8487,6 +8628,8 @@ snapshots: undici-types@7.24.6: {} + undici@7.29.0: {} + unicorn-magic@0.3.0: {} universalify@2.0.1: {} @@ -8644,6 +8787,14 @@ snapshots: is-wsl: 3.1.1 powershell-utils: 0.1.0 + xdg-app-paths@5.1.0: + dependencies: + xdg-portable: 7.3.0 + + xdg-portable@7.3.0: + dependencies: + os-paths: 4.4.0 + xtend@4.0.2: {} yallist@3.1.1: {} @@ -8654,6 +8805,8 @@ snapshots: yocto-queue@0.1.0: {} + yocto-queue@1.2.2: {} + yocto-spinner@1.2.0: dependencies: yoctocolors: 2.1.2