feat: parallel eval attempts

.github/workflows/eval-refresh.yml

@@ -305,17 +305,31 @@ jobs:
           # Optional warm-boot snapshot to reuse across runs (built out-of-band).
           # Unset -> cold boots, so this can never regress the default path.
           EVAL_SNAPSHOT_ID: ${{ vars.EVAL_SNAPSHOT_ID }}
+          # When "true", run each attempt in its own sandbox concurrently and
+          # pass an eval if any attempt passes (any-pass), instead of one
+          # sandbox running the attempts sequentially with stop-on-pass.
+          EVAL_PARALLEL_ATTEMPTS: ${{ vars.EVAL_PARALLEL_ATTEMPTS }}
+          RUNS: ${{ needs.prepare.outputs.runs }}
+          CONCURRENCY: ${{ needs.prepare.outputs.sandbox_concurrency }}
         shell: bash
         run: |
           set -euo pipefail
 
+          concurrency="$CONCURRENCY"
           args=(
             --pairs-json "$EVAL_PAIRS"
             --revision "$EVAL_REVISION"
-            --runs "${{ needs.prepare.outputs.runs }}"
+            --runs "$RUNS"
             --timeout-sec "${{ needs.prepare.outputs.timeout_sec }}"
-            --concurrency "${{ needs.prepare.outputs.sandbox_concurrency }}"
           )
+          if [ "${EVAL_PARALLEL_ATTEMPTS:-}" = "true" ]; then
+            args+=(--parallel-attempts)
+            # Size concurrency to pairs x runs so every attempt runs in one wave
+            # (the halving only shows up when attempts don't queue behind pairs).
+            npairs="$(jq 'length' <<< "$EVAL_PAIRS")"
+            concurrency=$(( npairs * RUNS ))
+          fi
+          args+=(--concurrency "$concurrency")
           if [ -n "${EVAL_SNAPSHOT_ID:-}" ]; then
             args+=(--snapshot-id "$EVAL_SNAPSHOT_ID")
           fi

apps/framework/scripts/run-vercel-evals.test.ts

@@ -1,6 +1,20 @@
+import {
+  mkdirSync,
+  mkdtempSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
 import { APIError } from '@vercel/sandbox';
-import { describe, expect, it } from 'vitest';
-import { parsePairs, runBounded } from './run-vercel-evals.js';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import {
+  aggregateAttempts,
+  parsePairs,
+  runBounded,
+  type EvalPair,
+} from './run-vercel-evals.js';
 import { isRetryableSandboxCreateError, tagValue } from './vercel-sandbox.js';
 
 describe('Vercel eval controller', () => {
@@ -63,3 +77,82 @@ describe('Vercel eval controller', () => {
     );
   });
 });
+
+describe('aggregateAttempts (any-pass)', () => {
+  const pair: EvalPair = {
+    eval_id: 'e1',
+    experiment: 'exp1',
+    experiment_suite: 'benchmark',
+    eval_suite: 'benchmark',
+  };
+  let root: string;
+
+  beforeEach(() => {
+    root = mkdtempSync(join(tmpdir(), 'agg-test-'));
+  });
+  afterEach(() => rmSync(root, { recursive: true, force: true }));
+
+  const attemptDir = (
+    name: string,
+    result: { passed: boolean; marker: string } | null
+  ): string => {
+    const dir = join(root, name);
+    mkdirSync(dir, { recursive: true });
+    if (result) {
+      writeFileSync(
+        join(dir, `${pair.eval_id}.json`),
+        JSON.stringify({ ...result, attempts: 1, checks: [] })
+      );
+    }
+    return dir;
+  };
+  const output = () =>
+    JSON.parse(
+      readFileSync(
+        join(root, 'out', `raw-results-${pair.experiment}__${pair.eval_id}`, `${pair.eval_id}.json`),
+        'utf8'
+      )
+    ) as { passed: boolean; attempts: number; marker: string };
+
+  it('passes if any attempt passed and keeps the passing tree', () => {
+    aggregateAttempts(
+      pair,
+      [
+        attemptDir('a1', { passed: false, marker: 'fail' }),
+        attemptDir('a2', { passed: true, marker: 'pass' }),
+      ],
+      join(root, 'out')
+    );
+    expect(output()).toMatchObject({ passed: true, attempts: 2, marker: 'pass' });
+  });
+
+  it('fails only when every attempt failed', () => {
+    aggregateAttempts(
+      pair,
+      [
+        attemptDir('a1', { passed: false, marker: 'a1' }),
+        attemptDir('a2', { passed: false, marker: 'a2' }),
+      ],
+      join(root, 'out')
+    );
+    expect(output()).toMatchObject({ passed: false, attempts: 2 });
+  });
+
+  it('counts only attempts that produced a result file', () => {
+    aggregateAttempts(
+      pair,
+      [
+        attemptDir('a1', { passed: true, marker: 'pass' }),
+        attemptDir('a2', null),
+      ],
+      join(root, 'out')
+    );
+    expect(output()).toMatchObject({ passed: true, attempts: 1, marker: 'pass' });
+  });
+
+  it('throws when no attempt produced a result file', () => {
+    expect(() =>
+      aggregateAttempts(pair, [attemptDir('a1', null)], join(root, 'out'))
+    ).toThrow('no attempt produced a result');
+  });
+});