diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4d91b9f..dc28d32 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -76,13 +76,37 @@ jobs: run: | node dist/bin/cli.js - - name: Test with asset - if: ${{ matrix.os == 'ubuntu-latest' }} + - name: Test with asset (and V8 code cache acceptance) run: | cd test - node ../dist/bin/cli.js -a asset.txt --no-bundle sample.cjs + # Build a single-platform host binary with an embedded V8 code + # cache (the exact config from issue #28). Assert the cache is + # actually embedded AND that it is accepted at runtime (no + # "Code cache data rejected" warning). macos-latest is arm64, + # which is where the cache rejection originally reproduced. + build_log=$(node ../dist/bin/cli.js -a asset.txt --no-bundle sample.cjs 2>&1) + echo "$build_log" + if ! echo "$build_log" | grep -q "with code cache"; then + echo "FAIL: host binary was built without an embedded V8 code cache" + exit 1 + fi + set -- dist-bin/sample-* + bin=$1 + echo "Running $bin" + set +e + actual=$("$bin" 2>stderr.txt) + rc=$? + set -e + echo "----- stderr -----"; cat stderr.txt; echo "------------------" + if [ "$rc" -ne 0 ]; then + echo "FAIL: binary exited with code $rc" + exit 1 + fi + if grep -q "Code cache data rejected" stderr.txt; then + echo "FAIL: V8 rejected the embedded code cache at runtime (issue #28)" + exit 1 + fi expected=$(cat asset.txt) - actual=$(./dist-bin/sample-linux-x64) [ "$actual" = "$expected" ] artifacts: diff --git a/.lore.md b/.lore.md index 4d51406..26fea22 100644 --- a/.lore.md +++ b/.lore.md @@ -4,6 +4,8 @@ ### Gotcha +* **SEA "Code cache data rejected" = V8 flag-hash mismatch between cache generator and consumer (signed vs unsigned on macOS)**: Node SEA's V8 code cache (\`useCodeCache\`) is only accepted at runtime when the consuming process has the *identical* V8 flag-hash (\`FlagList::Hash()\`) as the process that generated it. Proven via \`vm.compileFunction({produceCachedData})\` then consuming under a differing flag — \`cachedDataRejected\` flips true for \`--no-lazy\`, \`--jitless\`, \`--no-turbofan\`, \`--interpreted-frames-native-stack\`, \`--no-flush-bytecode\`, etc. (same code path as embedding.js's "Code cache data rejected" warning). The generation API (\`GenerateCodeCache\` in node_sea.cc → \`contextify::CompileFunction\` with \`ScriptOrigin(filename,0,0,true)\`) and consumption (\`CompileFunctionForCJSLoader\`, same 5 CJS params) match — host-defined options are NOT part of the cache key (confirmed by Node's own comment). The ROOT CAUSE in fossilize (#28): \`getNodeBinaryFromCache\` UNSIGNS the consumer binary copy on darwin/win (\`unsign()\`/\`signatureSet\`, node-util.ts) but the blob was generated by the raw *signed* official download (\`targetNodeBinary\`, no \`targetPath\`). On Apple Silicon, the hardened-runtime/JIT entitlements (\`allow-jit\`, \`disable-executable-page-protection\` in entitlements.plist) change V8's code-memory strategy → different flag-hash → rejection. Linux never reproduces (plain copy, no unsign, identical flags). FIX: base blob is always \`useCodeCache:false\`; the host code-cache blob is generated lazily inside \`createBinaryForPlatform\` by signing the prepared (stripped) host binary IN PLACE exactly like the final executable (extracted \`signBinary()\` helper, used for both cache-gen and final sign), running \`--experimental-sea-config\`, then \`unsignBinaryInPlace()\` to restore the unsigned state postject needs before injection. The two sign operations are fundamental and cannot be deduplicated (cache must be produced in the signed state BEFORE inject; final signature must cover the POST-inject blob). \`--no-code-cache\` flag disables it. Verified by darwin-arm64 CI smoke test asserting "(with code cache)" in build log AND no "Code cache data rejected" in stderr. TWO GOTCHAS found while optimizing: (1) postject does NOT require an unsigned Mach-O — it accepts a signed one and STRIPS the signature during injection (verified: signed 108MB binary → inject → unsigned, ~1MB smaller); but we still explicitly unsign to stay on Node's documented unsign→inject→sign flow. (2) On darwin/win, \`getNodeBinaryFromCache\` writes the unsigned binary via \`fs.writeFile\` at 0o644 (NO execute bit; linux \`copyFile\` preserves 0o755) — so running it directly for cache-gen needs an explicit \`fs.chmod(…, 0o755)\` first or it fails with EACCES. + * **Craft auto-version picks current package.json version — re-run with explicit version if already published**: Trap: Craft's \`version: auto\` reads package.json and detects no new conventional commits → creates a publish issue for the version already on npm (e.g. 0.5.0 when 0.5.0 is already published). Looks like it should bump, but it doesn't if package.json already matches latest npm. Fix: close the stale publish issue, then re-trigger the release workflow with an explicit version string (e.g. \`0.6.0\`). Craft will then create the correct release branch and publish issue. diff --git a/src/app.ts b/src/app.ts index f1f654d..1bb2e7d 100644 --- a/src/app.ts +++ b/src/app.ts @@ -72,6 +72,12 @@ const command = buildCommand({ brief: "Do not bundle the entrypoint using esbuild", optional: false, }, + noCodeCache: { + kind: "boolean", + brief: + "Do not embed a V8 startup code cache in the host-platform binary", + optional: true, + }, sign: { kind: "boolean", brief: "Skip signing for macOS and Windows", diff --git a/src/impl.ts b/src/impl.ts index 3d1e7f6..44fd8d5 100644 --- a/src/impl.ts +++ b/src/impl.ts @@ -6,7 +6,11 @@ import { promisify } from "node:util"; import * as esbuild from "esbuild"; import { inject } from "postject"; import type { LocalContext } from "./context"; -import { getNodeBinary, resolveNodeVersion } from "./node-util"; +import { + getNodeBinary, + resolveNodeVersion, + unsignBinaryInPlace, +} from "./node-util"; import pLimit from "p-limit"; export interface FossilizeOptions { @@ -19,6 +23,7 @@ export interface FossilizeOptions { readonly cacheDir: string; readonly noCache?: boolean; readonly noBundle: boolean; + readonly noCodeCache?: boolean; readonly sign: boolean; readonly holePunch: boolean; readonly concurrencyLimit: number; @@ -63,6 +68,75 @@ async function run(cmd: string, ...args: string[]): Promise { return output.stdout; } +// Apply the macOS code signature (ad-hoc or full identity) to a binary. +// This intentionally does NOT notarize — it is the smallest unit of work that +// puts the binary into its final signing state. It is extracted so the exact +// same signature can be applied during code-cache generation (on the prepared +// host binary, before injection) and to the final executable: V8 only accepts +// a code cache when the consuming binary runs with the same hardened-runtime / +// JIT entitlements (and therefore the same V8 flag-hash) as the binary that +// generated it. See issue #28. +// No-op on non-darwin platforms (linux is unsigned; Windows signing is the +// user's responsibility). +async function signBinary( + binaryPath: string, + platform: string, + sign: boolean +): Promise { + if (!platform.startsWith("darwin")) { + return; + } + const entitlements = fileURLToPath( + import.meta.resolve("../entitlements.plist") + ); + if (!sign) { + // Ad-hoc sign with entitlements — minimum required for Apple Silicon + // execution. Use native codesign on macOS, rcodesign elsewhere. + if (process.platform === "darwin") { + await run( + "codesign", + "--sign", + "-", + "--force", + "--entitlements", + entitlements, + binaryPath + ); + } else { + await run( + "rcodesign", + "sign", + "--code-signature-flags", + "runtime", + "--entitlements-xml-path", + entitlements, + binaryPath + ); + } + return; + } + const { APPLE_TEAM_ID, APPLE_CERT_PATH, APPLE_CERT_PASSWORD } = process.env; + if (!APPLE_TEAM_ID || !APPLE_CERT_PATH || !APPLE_CERT_PASSWORD) { + throw new Error( + "Missing required environment variables for macOS signing (at least one of APPLE_TEAM_ID, APPLE_CERT_PATH, APPLE_CERT_PASSWORD)" + ); + } + await run( + "rcodesign", + "sign", + "--team-name", + APPLE_TEAM_ID, + "--p12-file", + APPLE_CERT_PATH, + "--p12-password", + APPLE_CERT_PASSWORD, + "--for-notarization", + "-e", + entitlements, + binaryPath + ); +} + export default async function ( this: LocalContext, flags: FossilizeOptions, @@ -147,21 +221,19 @@ export default async function ( } } - // Determine if any target matches the build host — code cache is only - // valid for the same CPU architecture, so we generate two blobs when - // cross-compiling: one with code cache (host platform) and one without. - const hostIsTarget = platforms.includes(currentPlatform); - const needsCrossBlob = platforms.length > 1 || !hostIsTarget; - + // The base blob never carries a V8 code cache. Code cache is both + // CPU-architecture- AND signing-state-specific: V8 rejects it at runtime + // unless the consuming binary runs with the same flag-hash as the binary + // that produced it. We therefore generate the host platform's code-cache + // blob lazily inside createBinaryForPlatform() using a copy of the prepared + // (unsigned, stripped) host binary signed exactly like the final executable. + // See issue #28. const seaConfig: SEAConfig = { main: jsBundlePath, output: blobPath, disableExperimentalSEAWarning: true, useSnapshot: false, - // Enable code cache when building only for the host platform. - // When cross-compiling, the base blob is generated without code cache; - // a second blob with code cache is created for the host platform below. - useCodeCache: hostIsTarget && !needsCrossBlob, + useCodeCache: false, }; if (flags.assetManifest) { const manifest = JSON.parse( @@ -204,26 +276,26 @@ export default async function ( ); await run(targetNodeBinary, "--experimental-sea-config", seaConfigPath); - // When cross-compiling AND the host platform is a target, generate a - // second blob with V8 code cache enabled. Code cache pre-compiles the - // JS into bytecode, saving ~15% startup time — but the bytecode is - // CPU-architecture-specific, so it only works for the host platform. - const codeCacheBlobPath = `${blobPath}.codecache`; - let hasCodeCacheBlob = false; - if (hostIsTarget && needsCrossBlob) { - const codeCacheConfig: SEAConfig = { - ...seaConfig, - useCodeCache: true, - output: codeCacheBlobPath, - }; - const codeCacheConfigPath = `${seaConfigPath}.codecache`; - await fs.writeFile(codeCacheConfigPath, JSON.stringify(codeCacheConfig)); - console.log(`Generating code-cache blob for host platform (${currentPlatform})...`); - await run(targetNodeBinary, "--experimental-sea-config", codeCacheConfigPath); - await fs.rm(codeCacheConfigPath); - hasCodeCacheBlob = true; + // Fail fast when signing is requested for darwin targets but the required + // environment variables are missing — avoids a confusing intermediate + // "could not generate code cache" warning before the real crash. + if (flags.sign && platforms.some((p) => p.startsWith("darwin"))) { + const { APPLE_TEAM_ID, APPLE_CERT_PATH, APPLE_CERT_PASSWORD } = + process.env; + if (!APPLE_TEAM_ID || !APPLE_CERT_PATH || !APPLE_CERT_PASSWORD) { + throw new Error( + "Missing required environment variables for macOS signing " + + "(at least one of APPLE_TEAM_ID, APPLE_CERT_PATH, APPLE_CERT_PASSWORD)" + ); + } } + // Path for the host platform's code-cache blob. It is generated lazily + // inside createBinaryForPlatform() once the prepared host binary exists, so + // that the cache is produced by a binary in the same signing state as the + // final executable (otherwise V8 rejects it — see issue #28). + const codeCacheBlobPath = `${blobPath}.codecache`; + const createBinaryForPlatform = async (platform: string): Promise => { const outputPath = path.join(flags.outDir, outputName); console.log(`Creating binary for ${platform} (${outputPath})...`); @@ -254,12 +326,59 @@ export default async function ( } } - // Use the code-cache blob for the host platform, base blob for others - const blobForPlatform = (hasCodeCacheBlob && platform === currentPlatform) - ? codeCacheBlobPath - : blobPath; - const cacheLabel = blobForPlatform === codeCacheBlobPath ? " (with code cache)" : ""; - console.log(`Injecting blob into node executable: ${fossilizedBinary}${cacheLabel}`); + // The host platform gets a V8 code cache for faster startup (~15%). Code + // cache is CPU-arch- AND signing-state-specific, so it must be generated by + // a binary in the same state the final executable will run in. We sign the + // prepared (stripped) host binary in place — exactly as the final binary + // will be signed — generate the cache with it, then strip that signature + // again so postject injects into an unsigned binary (the final signature is + // applied after inject + hole-punch). Generating with a differently-signed + // binary makes V8 reject the cache at runtime ("Code cache data + // rejected"). See #28. + let blobForPlatform = blobPath; + if (platform === currentPlatform && !flags.noCodeCache) { + try { + // The freshly written binary is 0o644 on darwin/win (it was rewritten + // to strip the official signature) — make it executable before we run + // it to generate the cache. + await fs.chmod(fossilizedBinary, 0o755); + await signBinary(fossilizedBinary, platform, flags.sign); + const codeCacheConfig: SEAConfig = { + ...seaConfig, + useCodeCache: true, + output: codeCacheBlobPath, + }; + const codeCacheConfigPath = `${seaConfigPath}.codecache`; + await fs.writeFile(codeCacheConfigPath, JSON.stringify(codeCacheConfig)); + console.log( + `Generating code-cache blob for host platform (${currentPlatform})...` + ); + await run( + fossilizedBinary, + "--experimental-sea-config", + codeCacheConfigPath + ); + await fs.rm(codeCacheConfigPath, { force: true }); + blobForPlatform = codeCacheBlobPath; + } catch (err) { + console.warn( + ` Warning: could not generate V8 code cache for ${platform}, ` + + `falling back to no code cache (non-fatal): ${ + (err as Error).message + }` + ); + blobForPlatform = blobPath; + } finally { + // Restore the unsigned state postject expects before injection + // (no-op if the binary was never signed). + await unsignBinaryInPlace(fossilizedBinary, platform).catch(() => {}); + } + } + const cacheLabel = + blobForPlatform === codeCacheBlobPath ? " (with code cache)" : ""; + console.log( + `Injecting blob into node executable: ${fossilizedBinary}${cacheLabel}` + ); await inject( fossilizedBinary, "NODE_SEA_BLOB", @@ -274,7 +393,7 @@ export default async function ( } ); console.log("Created executable", fossilizedBinary); - fs.chmod(fossilizedBinary, 0o755); + await fs.chmod(fossilizedBinary, 0o755); // Hole-punch unused ICU data before signing so the signature covers the // final bytes. Must run after SEA injection (ICU blob lives in the Node @@ -293,32 +412,9 @@ export default async function ( if (platform.startsWith("darwin")) { // Ad-hoc sign with entitlements — minimum required for Apple Silicon // execution. Without at least ad-hoc signing, the kernel refuses to - // run the binary. Use native codesign on macOS, rcodesign elsewhere. - const entitlements = fileURLToPath( - import.meta.resolve("../entitlements.plist") - ); + // run the binary. try { - if (process.platform === "darwin") { - await run( - "codesign", - "--sign", - "-", - "--force", - "--entitlements", - entitlements, - fossilizedBinary - ); - } else { - await run( - "rcodesign", - "sign", - "--code-signature-flags", - "runtime", - "--entitlements-xml-path", - entitlements, - fossilizedBinary - ); - } + await signBinary(fossilizedBinary, platform, false); console.log(`Ad-hoc signed ${fossilizedBinary}`); } catch { console.warn( @@ -339,32 +435,9 @@ export default async function ( } if (platform.startsWith("darwin")) { - const { - APPLE_TEAM_ID, - APPLE_CERT_PATH, - APPLE_CERT_PASSWORD, - APPLE_API_KEY_PATH, - } = process.env; - if (!APPLE_TEAM_ID || !APPLE_CERT_PATH || !APPLE_CERT_PASSWORD) { - throw new Error( - "Missing required environment variables for macOS signing (at least one of APPLE_TEAM_ID, APPLE_CERT_PATH, APPLE_CERT_PASSWORD)" - ); - } + const { APPLE_API_KEY_PATH } = process.env; console.log(`Signing ${fossilizedBinary}...`); - await run( - "rcodesign", - "sign", - "--team-name", - APPLE_TEAM_ID, - "--p12-file", - APPLE_CERT_PATH, - "--p12-password", - APPLE_CERT_PASSWORD, - "--for-notarization", - "-e", - fileURLToPath(import.meta.resolve("../entitlements.plist")), - fossilizedBinary - ); + await signBinary(fossilizedBinary, platform, true); if (!APPLE_API_KEY_PATH) { console.warn( "Missing required environment variable for macOS notarization, you won't be able to notarize this binary which will annoy people trying to run it." @@ -391,9 +464,9 @@ export default async function ( limit(() => createBinaryForPlatform(platform)) ) ); - const cleanups = [fs.rm(seaConfigPath), fs.rm(blobPath)]; - if (hasCodeCacheBlob) { - cleanups.push(fs.rm(codeCacheBlobPath)); - } - await Promise.all(cleanups); + await Promise.all([ + fs.rm(seaConfigPath, { force: true }), + fs.rm(blobPath, { force: true }), + fs.rm(codeCacheBlobPath, { force: true }), + ]); } diff --git a/src/node-util.ts b/src/node-util.ts index 9ceddfc..24f91d8 100644 --- a/src/node-util.ts +++ b/src/node-util.ts @@ -63,6 +63,33 @@ async function getNodeBinaryFromCache( return targetFile; } +/** + * Strip an embedded code signature from a binary in place. Used to return a + * binary we temporarily signed (to generate a matching V8 code cache) back to + * the unsigned state postject expects before injection. No-op on Linux and + * when the binary carries no signature. + */ +export async function unsignBinaryInPlace( + filePath: string, + platform: string +): Promise { + if (!platform.startsWith("darwin") && !platform.startsWith("win")) { + return; + } + const buffer = await fs.readFile(filePath); + const unsigned: ArrayBufferLike | null = platform.startsWith("win") + ? signatureSet(buffer, null) + : unsign(buffer.buffer); + // `null` means there was no signature to strip — nothing to do. + if (unsigned) { + // Preserve the original file mode — fs.writeFile defaults to 0o666 + // (masked by umask → typically 0o644), which would lose the execute bit. + const { mode } = await fs.stat(filePath); + await fs.writeFile(filePath, Buffer.from(unsigned)); + await fs.chmod(filePath, mode); + } +} + const NODE_VERSIONS_INDEX_URL = "https://nodejs.org/download/release/index.json"; const NODE_VERSION_REGEX = /^v?(\d+)(?:\.(\d+))?(?:\.(\d+))?$/i;