fix(website): keep the post-promotion e2e run off the local fixture runtime (#983)

The deploy job re-runs the ordinary Website suite against production with
only BASE_URL set, and #982 let that step run for the first time since the
custom-target specs landed. Two failure clusters surfaced, both invisible to
PR CI by construction because only the push-only deploy job runs the suite
against a remote origin:

- Twelve custom-runtime-target cases dial a fixture runtime on 127.0.0.1:4399
  and the local example apps. Those exist only because the Playwright config
  starts them, and it starts nothing when BASE_URL is set, so every case
  failed with ECONNREFUSED after the site had already promoted. The config now
  ignores the fixture-driven specs whenever it starts no local server,
  including production-smoke mode.
- The reduced-motion check held the runtime in its configuring state by
  refusing http://localhost:4300. Against the deployed site the frame loads
  from the production runtime origin, the handshake completes within half a
  second, and the loader is gone before the assertion. The route now matches
  the runtime frame by the session params Run mode stamps on every runtime
  URL, which holds the loader on screen locally and in production alike.

Verified by running the full suite against https://threadplane.ai exactly as
the deploy step does: 105 passed, 0 failed (was 104 passed, 13 failed).

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>

Author: blove

apps/website/e2e/workspace-shell.spec.ts

@@ -730,7 +730,15 @@ test.describe('workspace shell', () => {
     page,
   }) => {
     await page.emulateMedia({ reducedMotion: 'reduce' });
-    await page.route('http://localhost:4300/**', (request) => request.abort());
+    // The loader is on screen only while the runtime is still being
+    // configured, so refuse the runtime frame itself. Match it by the session
+    // params Run mode stamps on every runtime URL rather than by host: against
+    // the deployed site the frame loads from the production runtime origin,
+    // and a `localhost:4300` route lets it reach ready before the assertion.
+    await page.route(
+      (url) => url.searchParams.has('cockpit_cap'),
+      (route) => route.abort()
+    );
     await page.setViewportSize({ width: 390, height: 844 });
     await page.goto(`${streamingDocsPath}?mode=run`);
     await page.getByRole('button', { name: 'Open navigation' }).click();

apps/website/playwright.config.ts

@@ -22,29 +22,43 @@ export const createWebsitePlaywrightConfig = (
   const reuseExistingServer =
     environment['PLAYWRIGHT_REUSE_EXISTING_SERVER'] === 'true';
 
+  // The public-copy gate crawls every sitemap route. Against a prebuilt
+  // production server that is seconds; against `next dev` each route compiles
+  // on demand, which is far too slow to belong in the ordinary suite. It runs
+  // in production mode only, where its answers are the ones that matter.
+  const modeIgnores: readonly string[] = productionSmoke
+    ? ['**/public-copy.spec.ts']
+    : productionMode
+    ? [
+        '**/platform-production-smoke.spec.ts',
+        '**/custom-runtime-bfcache.spec.ts',
+      ]
+    : bfcacheRuntimeTest
+    ? ['**/platform-production-smoke.spec.ts', '**/public-copy.spec.ts']
+    : [
+        '**/platform-production-smoke.spec.ts',
+        '**/custom-runtime-bfcache.spec.ts',
+        '**/public-copy.spec.ts',
+      ];
+  // The custom-target specs drive the fixture runtime on 127.0.0.1:4399 and
+  // the local example apps, which exist only because this config starts them.
+  // A run against a deployed BASE_URL — the post-promotion verification, the
+  // production smoke — starts nothing, so every case would dial a fixture that
+  // is not there and fail with ECONNREFUSED after the site already promoted.
+  const fixtureDrivenSpecs: readonly string[] = [
+    '**/custom-runtime-targets.spec.ts',
+    '**/custom-runtime-bfcache.spec.ts',
+  ];
+  const testIgnore = shouldStartLocalServer
+    ? [...modeIgnores]
+    : [...new Set([...modeIgnores, ...fixtureDrivenSpecs])];
+
   return defineConfig({
     testDir: './e2e',
     testMatch: bfcacheRuntimeTest
       ? '**/custom-runtime-bfcache.spec.ts'
       : undefined,
-    // The public-copy gate crawls every sitemap route. Against a prebuilt
-    // production server that is seconds; against `next dev` each route compiles
-    // on demand, which is far too slow to belong in the ordinary suite. It runs
-    // in production mode only, where its answers are the ones that matter.
-    testIgnore: productionSmoke
-      ? '**/public-copy.spec.ts'
-      : productionMode
-      ? [
-          '**/platform-production-smoke.spec.ts',
-          '**/custom-runtime-bfcache.spec.ts',
-        ]
-      : bfcacheRuntimeTest
-      ? ['**/platform-production-smoke.spec.ts', '**/public-copy.spec.ts']
-      : [
-          '**/platform-production-smoke.spec.ts',
-          '**/custom-runtime-bfcache.spec.ts',
-          '**/public-copy.spec.ts',
-        ],
+    testIgnore,
     fullyParallel: true,
     // Match the cockpit configs: 2 retries on CI to absorb transient Next.js
     // dev-server startup flake; 0 locally for fast feedback.

apps/website/src/playwright-config.spec.ts

@@ -75,8 +75,49 @@ describe('Website Playwright configuration', () => {
     expect(config.webServer).toBeUndefined();
     // The smoke job hits the deployed site, so it runs every spec except the
     // public-copy gate, which exists to check a locally built production
-    // server before the code is deployed at all.
-    expect(config.testIgnore).toBe('**/public-copy.spec.ts');
+    // server before the code is deployed at all, and the custom-target specs,
+    // which drive a fixture runtime this config did not start.
+    expect(config.testIgnore).toEqual([
+      '**/public-copy.spec.ts',
+      '**/custom-runtime-targets.spec.ts',
+      '**/custom-runtime-bfcache.spec.ts',
+    ]);
+  });
+
+  it('skips the fixture-driven specs when BASE_URL points at a deployed site', () => {
+    // The deploy job re-runs the ordinary suite against production with only
+    // BASE_URL set. No local server starts in that mode, so the custom-target
+    // specs would dial a fixture on 127.0.0.1:4399 that does not exist and fail
+    // every case with ECONNREFUSED — after the site was already promoted.
+    const config = createWebsitePlaywrightConfig({
+      BASE_URL: 'https://threadplane.ai',
+    });
+
+    expect(config.webServer).toBeUndefined();
+    expect(config.use).toEqual(
+      expect.objectContaining({ baseURL: 'https://threadplane.ai' })
+    );
+    expect(config.testIgnore).toEqual([
+      '**/platform-production-smoke.spec.ts',
+      '**/custom-runtime-bfcache.spec.ts',
+      '**/public-copy.spec.ts',
+      '**/custom-runtime-targets.spec.ts',
+    ]);
+  });
+
+  it('holds the runtime frame by its session params rather than the local host', () => {
+    // The reduced-motion check needs the runtime to stay in its connecting
+    // state so the loader is on screen. Refusing `http://localhost:4300` only
+    // does that against the local example app; against the deployed site the
+    // frame loads from the production runtime origin, the handshake completes,
+    // and the loader is gone before the assertion runs.
+    const shell = readFileSync(
+      resolve(__dirname, '../e2e/workspace-shell.spec.ts'),
+      'utf8'
+    );
+
+    expect(shell).not.toContain("page.route('http://localhost:4300/**'");
+    expect(shell).toContain("url.searchParams.has('cockpit_cap')");
   });
 
   it('starts Website, all migrated runtime apps under custom-runtime E2E, and the fixture', () => {